Fix JDoc

[tests] Update user agent

.github/ISSUE_TEMPLATE/bug_report.yml

@@ -0,0 +1,93 @@
+name: Bug report
+description: Create a bug report to help us improve
+labels: [bug]
+body:
+  - type: markdown
+    attributes:
+      value: |
+        Thank you for helping to make the NewPipe Extractor better by reporting a bug. :hugs:
+
+        Please fill in as much information as possible about your bug so that we don't have to play "information ping-pong" and can help you immediately.
+
+  - type: checkboxes
+    id: checklist
+    attributes:
+      label: "Checklist"
+      options:
+        - label: "I am able to reproduce the bug with the latest version given here: [CLICK THIS LINK](https://github.com/TeamNewPipe/NewPipeExtractor/releases/latest)."
+          required: true
+        - label: "I am aware that this issue is being opened for the NewPipe Extractor, NOT the [app](https://github.com/TeamNewPipe/NewPipe), and my bug report will be dismissed otherwise."
+          required: true
+        - label: "I made sure that there are *no existing issues* - [open](https://github.com/TeamNewPipe/NewPipeExtractor/issues) or [closed](https://github.com/TeamNewPipe/NewPipeExtractor/issues?q=is%3Aissue+is%3Aclosed) - which I could contribute my information to."
+          required: true
+        - label: "I have taken the time to fill in all the required details. I understand that the bug report will be dismissed otherwise."
+          required: true
+        - label: "This issue contains only one bug."
+          required: true
+        - label: "I have read and understood the [contribution guidelines](https://github.com/TeamNewPipe/NewPipe/blob/dev/.github/CONTRIBUTING.md)."
+          required: true
+
+  - type: input
+    id: extractor-version
+    attributes:
+     label: Affected version
+     description: "In which NewPipe Extractor version did you encounter the bug?"
+     placeholder: "x.xx.x"
+    validations:
+      required: true
+
+  - type: textarea
+    id: steps-to-reproduce
+    attributes:
+      label: Steps to reproduce the bug
+      description: |
+        What did you do for the bug to show up?
+
+        If you can't cause the bug to show up again reliably (and hence don't have a proper set of steps to give us), please still try to give as many details as possible on how you think you encountered the bug.
+      placeholder: |
+        1. Init NewPipe with 'NewPipe.init(...)'
+        2. Create a StreamExtractor for xyz: 'StreamExtractor e = YouTube.getStreamExtractor(xyz.com)'
+        3. Get the description 'e.getDescription()'
+    validations:
+      required: true
+
+  - type: textarea
+    id: expected-behavior
+    attributes:
+      label: Expected behavior
+      description: |
+        Tell us what you expect to happen.
+
+  - type: textarea
+    id: actual-behavior
+    attributes:
+      label: Actual behavior
+      description: |
+        Tell us what happens with the steps given above.
+
+  - type: textarea
+    id: screen-media
+    attributes:
+      label: Screenshots/Screen recordings
+      description: |
+        A picture or video is worth a thousand words.
+
+        If applicable, add screenshots or a screen recording to help explain your problem.
+        GitHub supports uploading them directly in the text box.
+        If your file is too big for Github to accept, try to compress it (ZIP-file) or feel free to paste a link to an image/video hoster here instead.
+
+  - type: textarea
+    id: logs
+    attributes:
+      label: Logs
+      description: |
+        If your bug includes a log you think we need to see, paste it here.
+
+  - type: textarea
+    id: additional-information
+    attributes:
+      label: Additional information
+      description: |
+        Any other information you'd like to include, for instance that
+        * your cat disabled your network connection
+        * ...

.github/ISSUE_TEMPLATE/config.yml

@@ -0,0 +1,8 @@
+blank_issues_enabled: false
+contact_links:
+  - name: 💬 Matrix
+    url: https://matrix.to/#/#newpipe:matrix.newpipe-ev.de
+    about: Chat with us via Matrix for quick Q/A
+  - name: 💬 IRC
+    url: https://web.libera.chat/#newpipe
+    about: Chat with us via IRC for quick Q/A

.github/ISSUE_TEMPLATE/enhancement.yml

@@ -0,0 +1,50 @@
+name: Feature request
+description: Suggest an idea for this project
+labels: [enhancement]
+body:
+  - type: markdown
+    attributes:
+      value: |
+        Thank you for helping to make the NewPipe Extractor better by suggesting a feature. :hugs:
+
+        Your ideas are highly welcome! The Extractor is made for you, the downstream developers, after all.
+
+  - type: checkboxes
+    id: checklist
+    attributes:
+      label: "Checklist"
+      options:
+        - label: "I am aware that this issue is being opened for the NewPipe Extractor, NOT the [app](https://github.com/TeamNewPipe/NewPipe), and my feature request will be dismissed otherwise."
+          required: true
+        - label: "I made sure that there are *no existing issues* - [open](https://github.com/TeamNewPipe/NewPipeExtractor/issues) or [closed](https://github.com/TeamNewPipe/NewPipeExtractor/issues?q=is%3Aissue+is%3Aclosed) - which I could contribute my information to."
+          required: true
+        - label: "I have taken the time to fill in all the required details. I understand that the feature request will be dismissed otherwise."
+          required: true
+        - label: "This issue contains only one feature request."
+          required: true
+        - label: "I have read and understood the [contribution guidelines](https://github.com/TeamNewPipe/NewPipe/blob/dev/.github/CONTRIBUTING.md)."
+          required: true
+
+  - type: textarea
+    id: feature-description
+    attributes:
+      label: Feature description
+      description: |
+        Explain how you want the Extractor's behavior to change to suit your needs.
+    validations:
+      required: true
+
+  - type: textarea
+    id: why-is-the-feature-requested
+    attributes:
+      label: Why do you want this feature?
+      description: |
+        Describe any problem or limitation you come across while using the Extractor which would be solved by this feature.
+    validations:
+      required: true
+
+  - type: textarea
+    id: additional-information
+    attributes:
+      label: Additional information
+      description: Any other information you'd like to include, for instance sketches, mockups, pictures of cats, etc.

.github/workflows/ci.yml

@@ -20,13 +20,13 @@ jobs:
       - uses: actions/checkout@v4
 
       - name: set up JDK 11
-        uses: actions/setup-java@v3
+        uses: actions/setup-java@v4
         with:
           java-version: '11'
           distribution: 'temurin'
 
       - name: Cache Gradle dependencies
-        uses: actions/cache@v3
+        uses: actions/cache@v4
         with:
           path: ~/.gradle/caches
           key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle') }}
@@ -44,7 +44,7 @@ jobs:
           fi
 
       - name: Upload test reports when failure occurs
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         if: failure()
         with:
           name: NewPipeExtractor-test-reports

.github/workflows/docs.yml

@@ -16,13 +16,13 @@ jobs:
       - uses: actions/checkout@v4
 
       - name: set up JDK 11
-        uses: actions/setup-java@v3
+        uses: actions/setup-java@v4
         with:
           java-version: '11'
           distribution: 'temurin'
 
       - name: Cache Gradle dependencies
-        uses: actions/cache@v3
+        uses: actions/cache@v4
         with:
           path: ~/.gradle/caches
           key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle') }}
@@ -32,7 +32,7 @@ jobs:
         run: ./gradlew aggregatedJavadocs
 
       - name: Deploy JavaDocs
-        uses: peaceiris/actions-gh-pages@v3
+        uses: peaceiris/actions-gh-pages@v4
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
           publish_dir: ./build/docs

README.md

@@ -1,6 +1,6 @@
 # NewPipe Extractor
 
-[![CI](https://github.com/TeamNewPipe/NewPipeExtractor/actions/workflows/ci.yml/badge.svg?branch=dev&event=schedule)](https://github.com/TeamNewPipe/NewPipeExtractor/actions/workflows/ci.yml) [![JIT Pack Badge](https://jitpack.io/v/TeamNewPipe/NewPipeExtractor.svg)](https://jitpack.io/#TeamNewPipe/NewPipeExtractor) [JDoc](https://teamnewpipe.github.io/NewPipeExtractor/javadoc/) • [Documentation](https://teamnewpipe.github.io/documentation/)
+[![CI](https://github.com/TeamNewPipe/NewPipeExtractor/actions/workflows/ci.yml/badge.svg?branch=dev&event=schedule)](https://github.com/TeamNewPipe/NewPipeExtractor/actions/workflows/ci.yml) [![JIT Pack Badge](https://jitpack.io/v/teamnewpipe/NewPipeExtractor.svg)](https://jitpack.io/#teamnewpipe/NewPipeExtractor) [JDoc](https://teamnewpipe.github.io/NewPipeExtractor/javadoc/) • [Documentation](https://teamnewpipe.github.io/documentation/)
 
 NewPipe Extractor is a library for extracting things from streaming sites. It is a core component of [NewPipe](https://github.com/TeamNewPipe/NewPipe), but could be used independently.
 
@@ -11,7 +11,7 @@ NewPipe Extractor is available at JitPack's Maven repo.
 If you're using Gradle, you could add NewPipe Extractor as a dependency with the following steps:
 
 1. Add `maven { url 'https://jitpack.io' }` to the `repositories` in your `build.gradle`.
-2. Add `implementation 'com.github.TeamNewPipe:NewPipeExtractor:INSERT_VERSION_HERE'` to the `dependencies` in your `build.gradle`. Replace `INSERT_VERSION_HERE` with the [latest release](https://github.com/TeamNewPipe/NewPipeExtractor/releases/latest).
+2. Add `implementation 'com.github.teamnewpipe:NewPipeExtractor:INSERT_VERSION_HERE'` to the `dependencies` in your `build.gradle`. Replace `INSERT_VERSION_HERE` with the [latest release](https://github.com/TeamNewPipe/NewPipeExtractor/releases/latest).
 3. If you are using tools to minimize your project, make sure to keep the files below, by e.g. adding the following lines to your proguard file:
  ```
 ## Rules for NewPipeExtractor
@@ -21,7 +21,7 @@ If you're using Gradle, you could add NewPipe Extractor as a dependency with the
 -dontwarn org.mozilla.javascript.tools.**
 ```
 
-**Note:** To use NewPipe Extractor in Android projects with a `minSdk` below 26, [API desugaring](https://developer.android.com/studio/write/java8-support#library-desugaring) is required. If the `minSdk` is below 19, the `desugar_jdk_libs_nio` artifact is required, which requires Android Gradle Plugin (AGP) version 7.4.0.
+**Note:** To use NewPipe Extractor in Android projects with a `minSdk` below 33, [core library desugaring](https://developer.android.com/studio/write/java8-support#library-desugaring) with the `desugar_jdk_libs_nio` artifact is required.
 
 ### Testing changes
 
@@ -30,7 +30,7 @@ To test changes quickly you can build the library locally. A good approach would
 ```groovy
 includeBuild('../NewPipeExtractor') {
     dependencySubstitution {
-        substitute module('com.github.TeamNewPipe:NewPipeExtractor') with project(':extractor')
+        substitute module('com.github.teamnewpipe:NewPipeExtractor') with project(':extractor')
     }
 }
 ```
@@ -40,7 +40,7 @@ Another approach would be to use the local Maven repository, here's a gist of ho
 1. Add `mavenLocal()` in your project `repositories` list (usually as the first entry to give priority above the others).
 2. It's _recommended_ that you change the `version` of this library (e.g. `LOCAL_SNAPSHOT`).
 3. Run gradle's `ìnstall` task to deploy this library to your local repository (using the wrapper, present in the root of this project: `./gradlew install`)
-4. Change the dependency version used in your project to match the one you chose in step 2 (`implementation 'com.github.TeamNewPipe:NewPipeExtractor:LOCAL_SNAPSHOT'`)
+4. Change the dependency version used in your project to match the one you chose in step 2 (`implementation 'com.github.teamnewpipe:NewPipeExtractor:LOCAL_SNAPSHOT'`)
 
 > Tip for Android Studio users: After you make changes and run the `install` task, use the menu option `File → "Sync with File System"` to refresh the library in your project.
 

build.gradle

@@ -8,7 +8,7 @@ allprojects {
     sourceCompatibility = JavaVersion.VERSION_11
     targetCompatibility = JavaVersion.VERSION_11
 
-    version 'v0.23.0'
+    version 'v0.24.3'
     group 'com.github.TeamNewPipe'
 
     repositories {
@@ -28,8 +28,8 @@ allprojects {
 
     ext {
         nanojsonVersion = "1d9e1aea9049fc9f85e68b43ba39fe7be1c1f751"
-        spotbugsVersion = "4.8.0"
-        junitVersion = "5.10.0"
+        spotbugsVersion = "4.8.6"
+        junitVersion = "5.11.3"
         checkstyleVersion = "10.4"
     }
 }

checkstyle/checkstyle.xml

@@ -38,6 +38,8 @@
     <module name="LineLength">
       <property name="max" value="100"/>
       <property name="fileExtensions" value="java"/>
+        <!-- Also allow links in javadocs to be longer (the default would just cover imports) -->
+        <property name="ignorePattern" value="^((package|import) .*)|( *\* (@see )?&lt;a href ?\= ?&quot;.*&quot;&gt;)$"/>
     </module>
 
     <!-- Checks for whitespace                               -->

extractor/build.gradle

@@ -26,12 +26,12 @@ dependencies {
     implementation project(':timeago-parser')
 
     implementation "com.github.TeamNewPipe:nanojson:$nanojsonVersion"
-    implementation 'org.jsoup:jsoup:1.16.2'
+    implementation 'org.jsoup:jsoup:1.17.2'
     implementation "com.github.spotbugs:spotbugs-annotations:$spotbugsVersion"
 
     // do not upgrade to 1.7.14, since in 1.7.14 Rhino uses the `SourceVersion` class, which is not
     // available on Android (even when using desugaring), and `NoClassDefFoundError` is thrown
-    implementation 'org.mozilla:rhino:1.7.13'
+    implementation 'org.mozilla:rhino:1.7.15'
 
     checkstyle "com.puppycrawl.tools:checkstyle:$checkstyleVersion"
 
@@ -41,5 +41,5 @@ dependencies {
     testImplementation 'org.junit.jupiter:junit-jupiter-params'
 
     testImplementation "com.squareup.okhttp3:okhttp:3.12.13"
-    testImplementation 'com.google.code.gson:gson:2.10.1'
+    testImplementation 'com.google.code.gson:gson:2.11.0'
 }

extractor/src/main/java/org/schabi/newpipe/extractor/MediaFormat.java

@@ -51,7 +51,7 @@ public enum MediaFormat {
     WEBMA_OPUS(0x200, "WebM Opus", "webm", "audio/webm"),
     AIFF      (0x600, "AIFF",      "aiff", "audio/aiff"),
     /**
-     * Same as {@link MediaFormat.AIFF}, just with the shorter suffix/file extension
+     * Same as {@link MediaFormat#AIFF}, just with the shorter suffix/file extension
      */
     AIF       (0x600, "AIFF",      "aif",  "audio/aiff"),
     WAV       (0x700, "WAV",       "wav",  "audio/wav"),

extractor/src/main/java/org/schabi/newpipe/extractor/comments/CommentsInfoItem.java

@@ -13,7 +13,8 @@ import java.util.List;
 public class CommentsInfoItem extends InfoItem {
 
     private String commentId;
-    private Description commentText;
+    @Nonnull
+    private Description commentText = Description.EMPTY_DESCRIPTION;
     private String uploaderName;
     @Nonnull
     private List<Image> uploaderAvatars = List.of();
@@ -50,11 +51,12 @@ public class CommentsInfoItem extends InfoItem {
         this.commentId = commentId;
     }
 
+    @Nonnull
     public Description getCommentText() {
         return commentText;
     }
 
-    public void setCommentText(final Description commentText) {
+    public void setCommentText(@Nonnull final Description commentText) {
         this.commentText = commentText;
     }
 

extractor/src/main/java/org/schabi/newpipe/extractor/comments/CommentsInfoItemExtractor.java

@@ -45,6 +45,7 @@ public interface CommentsInfoItemExtractor extends InfoItemExtractor {
     /**
      * The text of the comment
      */
+    @Nonnull
     default Description getCommentText() throws ParsingException {
         return Description.EMPTY_DESCRIPTION;
     }

extractor/src/main/java/org/schabi/newpipe/extractor/linkhandler/ReadyChannelTabListLinkHandler.java

@@ -27,7 +27,6 @@ import java.util.List;
  *     return ((ReadyChannelTabListLinkHandler) linkHandler).getChannelTabExtractor(this);
  * }
  * </pre>
- * </p>
  */
 public class ReadyChannelTabListLinkHandler extends ListLinkHandler {
 

extractor/src/main/java/org/schabi/newpipe/extractor/localization/Localization.java

@@ -11,10 +11,12 @@ import java.util.List;
 import java.util.Locale;
 import java.util.Map;
 import java.util.Objects;
+import java.util.Optional;
 
 import javax.annotation.Nonnull;
 import javax.annotation.Nullable;
 
+
 public class Localization implements Serializable {
     public static final Localization DEFAULT = new Localization("en", "GB");
 
@@ -26,20 +28,28 @@ public class Localization implements Serializable {
     /**
      * @param localizationCodeList a list of localization code, formatted like {@link
      *                             #getLocalizationCode()}
+     * @throws IllegalArgumentException If any of the localizationCodeList is formatted incorrectly
+     * @return list of Localization objects
      */
+    @Nonnull
     public static List<Localization> listFrom(final String... localizationCodeList) {
         final List<Localization> toReturn = new ArrayList<>();
         for (final String localizationCode : localizationCodeList) {
-            toReturn.add(fromLocalizationCode(localizationCode));
+            toReturn.add(fromLocalizationCode(localizationCode)
+                    .orElseThrow(() -> new IllegalArgumentException(
+                            "Not a localization code: " + localizationCode
+                    )));
         }
         return Collections.unmodifiableList(toReturn);
     }
 
     /**
      * @param localizationCode a localization code, formatted like {@link #getLocalizationCode()}
+     * @return A Localization, if the code was valid.
      */
-    public static Localization fromLocalizationCode(final String localizationCode) {
-        return fromLocale(LocaleCompat.forLanguageTag(localizationCode));
+    @Nonnull
+    public static Optional<Localization> fromLocalizationCode(final String localizationCode) {
+        return LocaleCompat.forLanguageTag(localizationCode).map(Localization::fromLocale);
     }
 
     public Localization(@Nonnull final String languageCode, @Nullable final String countryCode) {
@@ -61,10 +71,6 @@ public class Localization implements Serializable {
         return countryCode == null ? "" : countryCode;
     }
 
-    public Locale asLocale() {
-        return new Locale(getLanguageCode(), getCountryCode());
-    }
-
     public static Localization fromLocale(@Nonnull final Locale locale) {
         return new Localization(locale.getLanguage(), locale.getCountry());
     }
@@ -72,6 +78,8 @@ public class Localization implements Serializable {
     /**
      * Return a formatted string in the form of: {@code language-Country}, or
      * just {@code language} if country is {@code null}.
+     *
+     * @return A correctly formatted localizationCode for this localization.
      */
     public String getLocalizationCode() {
         return languageCode + (countryCode == null ? "" : "-" + countryCode);

extractor/src/main/java/org/schabi/newpipe/extractor/search/SearchInfo.java

@@ -19,7 +19,7 @@ public class SearchInfo extends ListInfo<InfoItem> {
     private final String searchString;
     private String searchSuggestion;
     private boolean isCorrectedSearch;
-    private List<MetaInfo> metaInfo;
+    private List<MetaInfo> metaInfo = List.of();
 
     public SearchInfo(final int serviceId,
                       final SearchQueryHandler qIHandler,

extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampCommentsInfoItemExtractor.java

@@ -38,6 +38,7 @@ public class BandcampCommentsInfoItemExtractor implements CommentsInfoItemExtrac
         return getUploaderAvatars();
     }
 
+    @Nonnull
     @Override
     public Description getCommentText() throws ParsingException {
         return new Description(review.getString("why"), Description.PLAIN_TEXT);

extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampExtractorHelper.java

@@ -16,6 +16,7 @@ import org.schabi.newpipe.extractor.exceptions.ParsingException;
 import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
 import org.schabi.newpipe.extractor.localization.DateWrapper;
 import org.schabi.newpipe.extractor.utils.ImageSuffix;
+import org.schabi.newpipe.extractor.utils.Utils;
 
 import java.io.IOException;
 import java.nio.charset.StandardCharsets;
@@ -111,8 +112,7 @@ public final class BandcampExtractorHelper {
 
     /**
      * Fetch artist details from mobile endpoint.
-     * <a href="https://notabug.org/fynngodau/bandcampDirect/wiki/
-     * rewindBandcamp+%E2%80%93+Fetching+artist+details">
+     * <a href="https://notabug.org/fynngodau/bandcampDirect/wiki/rewindBandcamp+%E2%80%93+Fetching+artist+details">
      * More technical info.</a>
      */
     public static JsonObject getArtistDetails(final String id) throws ParsingException {
@@ -156,25 +156,34 @@ public final class BandcampExtractorHelper {
 
     /**
      * @return <code>true</code> if the given URL looks like it comes from a bandcamp custom domain
-     * or if it comes from <code>bandcamp.com</code> itself
+     * or a <code>*.bandcamp.com</code> subdomain
      */
-    public static boolean isSupportedDomain(final String url) throws ParsingException {
+    public static boolean isArtistDomain(final String url) throws ParsingException {
 
         // Accept all bandcamp.com URLs
         if (url.toLowerCase().matches("https?://.+\\.bandcamp\\.com(/.*)?")) {
             return true;
         }
 
+        // Reject non-artist bandcamp.com URLs
+        if (url.toLowerCase().matches("https?://bandcamp\\.com(/.*)?")) {
+            return false;
+        }
+
         try {
             // Test other URLs for whether they contain a footer that links to bandcamp
-            return Jsoup.parse(NewPipe.getDownloader().get(url).responseBody())
-                    .getElementById("pgFt")
-                    .getElementById("pgFt-inner")
-                    .getElementById("footer-logo-wrapper")
-                    .getElementById("footer-logo")
-                    .getElementsByClass("hiddenAccess")
-                    .text().equals("Bandcamp");
-        } catch (final NullPointerException e) {
+            return Jsoup.parse(
+                    NewPipe.getDownloader()
+                            .get(Utils.replaceHttpWithHttps(url))
+                            .responseBody()
+                    )
+                    .getElementsByClass("cart-wrapper")
+                    .get(0)
+                    .getElementsByTag("a")
+                    .get(0)
+                    .attr("href")
+                    .equals("https://bandcamp.com/cart");
+        } catch (final NullPointerException | IndexOutOfBoundsException e) {
             return false;
         } catch (final IOException | ReCaptchaException e) {
             throw new ParsingException("Could not determine whether URL is custom domain "

extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampRadioExtractor.java

@@ -24,7 +24,7 @@ import static org.schabi.newpipe.extractor.services.bandcamp.extractors.Bandcamp
 public class BandcampRadioExtractor extends KioskExtractor<StreamInfoItem> {
 
     public static final String KIOSK_RADIO = "Radio";
-    public static final String RADIO_API_URL = BASE_API_URL + "/bcweekly/1/list";
+    public static final String RADIO_API_URL = BASE_API_URL + "/bcweekly/3/list";
 
     private JsonObject json = null;
 

extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampRadioInfoItemExtractor.java

@@ -35,6 +35,12 @@ public class BandcampRadioInfoItemExtractor implements StreamInfoItemExtractor {
         return 0;
     }
 
+    @Nullable
+    @Override
+    public String getShortDescription() {
+        return show.getString("desc");
+    }
+
     @Nullable
     @Override
     public String getTextualUploadDate() {
@@ -75,8 +81,8 @@ public class BandcampRadioInfoItemExtractor implements StreamInfoItemExtractor {
 
     @Override
     public String getUploaderName() {
-        // JSON does not contain uploader name
-        return "";
+        // The "title" field contains the title of the series, e.g. "Bandcamp Weekly".
+        return show.getString("title");
     }
 
     @Override

extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/streaminfoitem/BandcampPlaylistStreamInfoItemExtractor.java

@@ -43,7 +43,12 @@ public class BandcampPlaylistStreamInfoItemExtractor extends BandcampStreamInfoI
 
     @Override
     public String getUrl() {
-        return getUploaderUrl() + track.getString("title_link");
+        final String relativeUrl = track.getString("title_link");
+        if (relativeUrl != null) {
+            return getUploaderUrl() + relativeUrl;
+        } else {
+            return null;
+        }
     }
 
     @Override
@@ -66,7 +71,7 @@ public class BandcampPlaylistStreamInfoItemExtractor extends BandcampStreamInfoI
     @Nonnull
     @Override
     public List<Image> getThumbnails() throws ParsingException {
-        if (substituteCovers.isEmpty()) {
+        if (substituteCovers.isEmpty() && getUrl() != null) {
             try {
                 final StreamExtractor extractor = service.getStreamExtractor(getUrl());
                 extractor.fetchPage();

extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/linkHandler/BandcampChannelLinkHandlerFactory.java

@@ -33,7 +33,8 @@ public final class BandcampChannelLinkHandlerFactory extends ListLinkHandlerFact
     @Override
     public String getId(final String url) throws ParsingException, UnsupportedOperationException {
         try {
-            final String response = NewPipe.getDownloader().get(url).responseBody();
+            final String response = NewPipe.getDownloader().get(Utils.replaceHttpWithHttps(url))
+                    .responseBody();
 
             // Use band data embedded in website to extract ID
             final JsonObject bandData = JsonUtils.getJsonData(response, "data-band");
@@ -90,7 +91,7 @@ public final class BandcampChannelLinkHandlerFactory extends ListLinkHandlerFact
             }
 
             // Test whether domain is supported
-            return BandcampExtractorHelper.isSupportedDomain(lowercaseUrl);
+            return BandcampExtractorHelper.isArtistDomain(lowercaseUrl);
         }
     }
 }

extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/linkHandler/BandcampCommentsLinkHandlerFactory.java

@@ -3,6 +3,7 @@ package org.schabi.newpipe.extractor.services.bandcamp.linkHandler;
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
 import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory;
 import org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampExtractorHelper;
+import org.schabi.newpipe.extractor.utils.Utils;
 
 import java.util.List;
 
@@ -24,7 +25,7 @@ public final class BandcampCommentsLinkHandlerFactory extends ListLinkHandlerFac
 
     @Override
     public String getId(final String url) throws ParsingException, UnsupportedOperationException {
-        return url;
+        return Utils.replaceHttpWithHttps(url);
     }
 
     @Override
@@ -39,7 +40,7 @@ public final class BandcampCommentsLinkHandlerFactory extends ListLinkHandlerFac
         }
 
         // Test whether domain is supported
-        return BandcampExtractorHelper.isSupportedDomain(url);
+        return BandcampExtractorHelper.isArtistDomain(url);
     }
 
     @Override
@@ -47,6 +48,6 @@ public final class BandcampCommentsLinkHandlerFactory extends ListLinkHandlerFac
                          final List<String> contentFilter,
                          final String sortFilter)
             throws ParsingException, UnsupportedOperationException {
-        return id;
+        return Utils.replaceHttpWithHttps(id);
     }
 }

extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/linkHandler/BandcampPlaylistLinkHandlerFactory.java

@@ -5,6 +5,7 @@ package org.schabi.newpipe.extractor.services.bandcamp.linkHandler;
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
 import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory;
 import org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampExtractorHelper;
+import org.schabi.newpipe.extractor.utils.Utils;
 
 import java.util.List;
 
@@ -33,7 +34,7 @@ public final class BandcampPlaylistLinkHandlerFactory extends ListLinkHandlerFac
                          final List<String> contentFilter,
                          final String sortFilter)
             throws ParsingException, UnsupportedOperationException {
-        return url;
+        return Utils.replaceHttpWithHttps(url);
     }
 
     /**
@@ -48,6 +49,6 @@ public final class BandcampPlaylistLinkHandlerFactory extends ListLinkHandlerFac
         }
 
         // Test whether domain is supported
-        return BandcampExtractorHelper.isSupportedDomain(url);
+        return BandcampExtractorHelper.isArtistDomain(url);
     }
 }

extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/linkHandler/BandcampSearchQueryHandlerFactory.java

@@ -8,7 +8,6 @@ import org.schabi.newpipe.extractor.exceptions.ParsingException;
 import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandlerFactory;
 import org.schabi.newpipe.extractor.utils.Utils;
 
-import java.io.UnsupportedEncodingException;
 import java.util.List;
 
 public final class BandcampSearchQueryHandlerFactory extends SearchQueryHandlerFactory {
@@ -28,10 +27,6 @@ public final class BandcampSearchQueryHandlerFactory extends SearchQueryHandlerF
                          final List<String> contentFilter,
                          final String sortFilter)
             throws ParsingException, UnsupportedOperationException {
-        try {
-            return BASE_URL + "/search?q=" + Utils.encodeUrlUtf8(query) + "&page=1";
-        } catch (final UnsupportedEncodingException e) {
-            throw new ParsingException("query \"" + query + "\" could not be encoded", e);
-        }
+        return BASE_URL + "/search?q=" + Utils.encodeUrlUtf8(query) + "&page=1";
     }
 }

extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/linkHandler/BandcampStreamLinkHandlerFactory.java

@@ -5,6 +5,7 @@ package org.schabi.newpipe.extractor.services.bandcamp.linkHandler;
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
 import org.schabi.newpipe.extractor.linkhandler.LinkHandlerFactory;
 import org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampExtractorHelper;
+import org.schabi.newpipe.extractor.utils.Utils;
 
 import static org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampExtractorHelper.BASE_URL;
 
@@ -49,7 +50,7 @@ public final class BandcampStreamLinkHandlerFactory extends LinkHandlerFactory {
         if (input.matches("\\d+")) {
             return BASE_URL + "/?show=" + input;
         } else {
-            return input;
+            return Utils.replaceHttpWithHttps(input);
         }
     }
 
@@ -71,6 +72,6 @@ public final class BandcampStreamLinkHandlerFactory extends LinkHandlerFactory {
         }
 
         // Test whether domain is supported
-        return BandcampExtractorHelper.isSupportedDomain(url);
+        return BandcampExtractorHelper.isArtistDomain(url);
     }
 }

extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCService.java

@@ -19,6 +19,7 @@ import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandler;
 import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandlerFactory;
 import org.schabi.newpipe.extractor.playlist.PlaylistExtractor;
 import org.schabi.newpipe.extractor.search.SearchExtractor;
+import org.schabi.newpipe.extractor.services.media_ccc.extractors.MediaCCCChannelTabExtractor;
 import org.schabi.newpipe.extractor.services.media_ccc.extractors.MediaCCCConferenceExtractor;
 import org.schabi.newpipe.extractor.services.media_ccc.extractors.MediaCCCConferenceKiosk;
 import org.schabi.newpipe.extractor.services.media_ccc.extractors.MediaCCCLiveStreamExtractor;
@@ -57,7 +58,9 @@ public class MediaCCCService extends StreamingService {
 
     @Override
     public ListLinkHandlerFactory getChannelTabLHFactory() {
-        return null;
+        // there is just one channel tab in MediaCCC, the one containing conferences, so there is
+        // no need for a specific channel tab link handler, but we can just use the channel one
+        return MediaCCCConferenceLinkHandlerFactory.getInstance();
     }
 
     @Override
@@ -86,17 +89,13 @@ public class MediaCCCService extends StreamingService {
     @Override
     public ChannelTabExtractor getChannelTabExtractor(final ListLinkHandler linkHandler) {
         if (linkHandler instanceof ReadyChannelTabListLinkHandler) {
+            // conference data has already been fetched, let the ReadyChannelTabListLinkHandler
+            // create a MediaCCCChannelTabExtractor with that data
             return ((ReadyChannelTabListLinkHandler) linkHandler).getChannelTabExtractor(this);
+        } else {
+            // conference data has not been fetched yet, so pass null instead
+            return new MediaCCCChannelTabExtractor(this, linkHandler, null);
         }
-
-        /*
-        Channel tab extractors are only supported in conferences and should only come from a
-        ReadyChannelTabListLinkHandler instance with a ChannelTabExtractorBuilder instance of the
-        conferences extractor
-
-        If that's not the case, return null in this case, so no channel tabs support
-        */
-        return null;
     }
 
     @Override

extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCChannelTabExtractor.java

@@ -0,0 +1,69 @@
+package org.schabi.newpipe.extractor.services.media_ccc.extractors;
+
+import com.grack.nanojson.JsonObject;
+
+import org.schabi.newpipe.extractor.InfoItem;
+import org.schabi.newpipe.extractor.ListExtractor;
+import org.schabi.newpipe.extractor.MultiInfoItemsCollector;
+import org.schabi.newpipe.extractor.Page;
+import org.schabi.newpipe.extractor.StreamingService;
+import org.schabi.newpipe.extractor.channel.tabs.ChannelTabExtractor;
+import org.schabi.newpipe.extractor.downloader.Downloader;
+import org.schabi.newpipe.extractor.exceptions.ExtractionException;
+import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
+import org.schabi.newpipe.extractor.services.media_ccc.extractors.infoItems.MediaCCCStreamInfoItemExtractor;
+
+import java.io.IOException;
+import java.util.Objects;
+
+import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
+
+/**
+ * MediaCCC does not really have channel tabs, but rather a list of videos for each conference,
+ * so this class just acts as a videos channel tab extractor.
+ */
+public class MediaCCCChannelTabExtractor extends ChannelTabExtractor {
+    @Nullable
+    private JsonObject conferenceData;
+
+    /**
+     * @param conferenceData will be not-null if conference data has already been fetched by
+     *                       {@link MediaCCCConferenceExtractor}. Otherwise, if this parameter is
+     *                       {@code null}, conference data will be fetched anew.
+     */
+    public MediaCCCChannelTabExtractor(final StreamingService service,
+                                       final ListLinkHandler linkHandler,
+                                       @Nullable final JsonObject conferenceData) {
+        super(service, linkHandler);
+        this.conferenceData = conferenceData;
+    }
+
+    @Override
+    public void onFetchPage(@Nonnull final Downloader downloader)
+            throws ExtractionException, IOException {
+        if (conferenceData == null) {
+            // only fetch conference data if we don't have it already
+            conferenceData = MediaCCCConferenceExtractor.fetchConferenceData(downloader, getId());
+        }
+    }
+
+    @Nonnull
+    @Override
+    public ListExtractor.InfoItemsPage<InfoItem> getInitialPage() {
+        final MultiInfoItemsCollector collector =
+                new MultiInfoItemsCollector(getServiceId());
+        Objects.requireNonNull(conferenceData) // will surely be != null after onFetchPage
+                .getArray("events")
+                .stream()
+                .filter(JsonObject.class::isInstance)
+                .map(JsonObject.class::cast)
+                .forEach(event -> collector.commit(new MediaCCCStreamInfoItemExtractor(event)));
+        return new ListExtractor.InfoItemsPage<>(collector, null);
+    }
+
+    @Override
+    public ListExtractor.InfoItemsPage<InfoItem> getPage(final Page page) {
+        return ListExtractor.InfoItemsPage.emptyPage();
+    }
+}

extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCConferenceExtractor.java

@@ -1,24 +1,20 @@
 package org.schabi.newpipe.extractor.services.media_ccc.extractors;
 
+import static org.schabi.newpipe.extractor.services.media_ccc.extractors.MediaCCCParsingHelper.getImageListFromLogoImageUrl;
+
 import com.grack.nanojson.JsonObject;
 import com.grack.nanojson.JsonParser;
 import com.grack.nanojson.JsonParserException;
 
-import org.schabi.newpipe.extractor.InfoItem;
-import org.schabi.newpipe.extractor.ListExtractor;
-import org.schabi.newpipe.extractor.MultiInfoItemsCollector;
 import org.schabi.newpipe.extractor.Image;
-import org.schabi.newpipe.extractor.Page;
 import org.schabi.newpipe.extractor.StreamingService;
 import org.schabi.newpipe.extractor.channel.ChannelExtractor;
-import org.schabi.newpipe.extractor.channel.tabs.ChannelTabExtractor;
 import org.schabi.newpipe.extractor.channel.tabs.ChannelTabs;
 import org.schabi.newpipe.extractor.downloader.Downloader;
 import org.schabi.newpipe.extractor.exceptions.ExtractionException;
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
 import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
 import org.schabi.newpipe.extractor.linkhandler.ReadyChannelTabListLinkHandler;
-import org.schabi.newpipe.extractor.services.media_ccc.extractors.infoItems.MediaCCCStreamInfoItemExtractor;
 import org.schabi.newpipe.extractor.services.media_ccc.linkHandler.MediaCCCConferenceLinkHandlerFactory;
 
 import java.io.IOException;
@@ -27,8 +23,6 @@ import java.util.List;
 
 import javax.annotation.Nonnull;
 
-import static org.schabi.newpipe.extractor.services.media_ccc.extractors.MediaCCCParsingHelper.getImageListFromLogoImageUrl;
-
 public class MediaCCCConferenceExtractor extends ChannelExtractor {
     private JsonObject conferenceData;
 
@@ -37,6 +31,19 @@ public class MediaCCCConferenceExtractor extends ChannelExtractor {
         super(service, linkHandler);
     }
 
+    static JsonObject fetchConferenceData(@Nonnull final Downloader downloader,
+                                          @Nonnull final String conferenceId)
+            throws IOException, ExtractionException {
+        final String conferenceUrl
+                = MediaCCCConferenceLinkHandlerFactory.CONFERENCE_API_ENDPOINT + conferenceId;
+        try {
+            return JsonParser.object().from(downloader.get(conferenceUrl).responseBody());
+        } catch (final JsonParserException jpe) {
+            throw new ExtractionException("Could not parse json returned by URL: " + conferenceUrl);
+        }
+    }
+
+
     @Nonnull
     @Override
     public List<Image> getAvatars() {
@@ -88,20 +95,17 @@ public class MediaCCCConferenceExtractor extends ChannelExtractor {
     @Nonnull
     @Override
     public List<ListLinkHandler> getTabs() throws ParsingException {
-        return List.of(new ReadyChannelTabListLinkHandler(getUrl(), getId(),
-                ChannelTabs.VIDEOS, new VideosTabExtractorBuilder(conferenceData)));
+        // avoid keeping a reference to MediaCCCConferenceExtractor inside the lambda
+        final JsonObject theConferenceData = conferenceData;
+        return List.of(new ReadyChannelTabListLinkHandler(getUrl(), getId(), ChannelTabs.VIDEOS,
+                (service, linkHandler) ->
+                        new MediaCCCChannelTabExtractor(service, linkHandler, theConferenceData)));
     }
 
     @Override
     public void onFetchPage(@Nonnull final Downloader downloader)
             throws IOException, ExtractionException {
-        final String conferenceUrl
-                = MediaCCCConferenceLinkHandlerFactory.CONFERENCE_API_ENDPOINT + getId();
-        try {
-            conferenceData = JsonParser.object().from(downloader.get(conferenceUrl).responseBody());
-        } catch (final JsonParserException jpe) {
-            throw new ExtractionException("Could not parse json returned by URL: " + conferenceUrl);
-        }
+        conferenceData = fetchConferenceData(downloader, getId());
     }
 
     @Nonnull
@@ -109,55 +113,4 @@ public class MediaCCCConferenceExtractor extends ChannelExtractor {
     public String getName() throws ParsingException {
         return conferenceData.getString("title");
     }
-
-    private static final class VideosTabExtractorBuilder
-            implements ReadyChannelTabListLinkHandler.ChannelTabExtractorBuilder {
-
-        private final JsonObject conferenceData;
-
-        VideosTabExtractorBuilder(final JsonObject conferenceData) {
-            this.conferenceData = conferenceData;
-        }
-
-        @Nonnull
-        @Override
-        public ChannelTabExtractor build(@Nonnull final StreamingService service,
-                                         @Nonnull final ListLinkHandler linkHandler) {
-            return new VideosChannelTabExtractor(service, linkHandler, conferenceData);
-        }
-    }
-
-    private static final class VideosChannelTabExtractor extends ChannelTabExtractor {
-        private final JsonObject conferenceData;
-
-        VideosChannelTabExtractor(final StreamingService service,
-                                  final ListLinkHandler linkHandler,
-                                  final JsonObject conferenceData) {
-            super(service, linkHandler);
-            this.conferenceData = conferenceData;
-        }
-
-        @Override
-        public void onFetchPage(@Nonnull final Downloader downloader) {
-            // Nothing to do here, as data was already fetched
-        }
-
-        @Nonnull
-        @Override
-        public ListExtractor.InfoItemsPage<InfoItem> getInitialPage() {
-            final MultiInfoItemsCollector collector =
-                    new MultiInfoItemsCollector(getServiceId());
-            conferenceData.getArray("events")
-                    .stream()
-                    .filter(JsonObject.class::isInstance)
-                    .map(JsonObject.class::cast)
-                    .forEach(event -> collector.commit(new MediaCCCStreamInfoItemExtractor(event)));
-            return new InfoItemsPage<>(collector, null);
-        }
-
-        @Override
-        public InfoItemsPage<InfoItem> getPage(final Page page) {
-            return InfoItemsPage.emptyPage();
-        }
-    }
 }

extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCStreamExtractor.java

@@ -130,7 +130,10 @@ public class MediaCCCStreamExtractor extends StreamExtractor {
                 // track with multiple languages, so there is no specific language for this stream
                 // Don't set the audio language in this case
                 if (language != null && !language.contains("-")) {
-                    builder.setAudioLocale(LocaleCompat.forLanguageTag(language));
+                    builder.setAudioLocale(LocaleCompat.forLanguageTag(language).orElseThrow(() ->
+                        new ParsingException(
+                                "Cannot convert this language to a locale: " + language)
+                    ));
                 }
 
                 // Not checking containsSimilarStream here, since MediaCCC does not provide enough

extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/linkHandler/MediaCCCConferenceLinkHandlerFactory.java

@@ -1,11 +1,17 @@
 package org.schabi.newpipe.extractor.services.media_ccc.linkHandler;
 
+import org.schabi.newpipe.extractor.channel.tabs.ChannelTabs;
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
 import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory;
 import org.schabi.newpipe.extractor.utils.Parser;
 
 import java.util.List;
 
+/**
+ * Since MediaCCC does not really have channel tabs (i.e. it only has one single "tab" with videos),
+ * this link handler acts both as the channel link handler and the channel tab link handler. That's
+ * why {@link #getAvailableContentFilter()} has been overridden.
+ */
 public final class MediaCCCConferenceLinkHandlerFactory extends ListLinkHandlerFactory {
 
     private static final MediaCCCConferenceLinkHandlerFactory INSTANCE
@@ -46,4 +52,15 @@ public final class MediaCCCConferenceLinkHandlerFactory extends ListLinkHandlerF
             return false;
         }
     }
+
+    /**
+     * @see MediaCCCConferenceLinkHandlerFactory
+     * @return MediaCCC's only channel "tab", i.e. {@link ChannelTabs#VIDEOS}
+     */
+    @Override
+    public String[] getAvailableContentFilter() {
+        return new String[]{
+                ChannelTabs.VIDEOS,
+        };
+    }
 }

extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/linkHandler/MediaCCCSearchQueryHandlerFactory.java

@@ -4,7 +4,6 @@ import org.schabi.newpipe.extractor.exceptions.ParsingException;
 import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandlerFactory;
 import org.schabi.newpipe.extractor.utils.Utils;
 
-import java.io.UnsupportedEncodingException;
 import java.util.List;
 
 public final class MediaCCCSearchQueryHandlerFactory extends SearchQueryHandlerFactory {
@@ -42,10 +41,6 @@ public final class MediaCCCSearchQueryHandlerFactory extends SearchQueryHandlerF
                          final List<String> contentFilter,
                          final String sortFilter)
             throws ParsingException, UnsupportedOperationException {
-        try {
-            return "https://media.ccc.de/public/events/search?q=" + Utils.encodeUrlUtf8(query);
-        } catch (final UnsupportedEncodingException e) {
-            throw new ParsingException("Could not create search string with query: " + query, e);
-        }
+        return "https://media.ccc.de/public/events/search?q=" + Utils.encodeUrlUtf8(query);
     }
 }

extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeCommentsInfoItemExtractor.java

@@ -77,6 +77,7 @@ public class PeertubeCommentsInfoItemExtractor implements CommentsInfoItemExtrac
         return new DateWrapper(parseDateFrom(textualUploadDate));
     }
 
+    @Nonnull
     @Override
     public Description getCommentText() throws ParsingException {
         final String htmlText = JsonUtils.getString(item, "text");

extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java

@@ -26,9 +26,11 @@ import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeStream
 import org.schabi.newpipe.extractor.stream.AudioStream;
 import org.schabi.newpipe.extractor.stream.DeliveryMethod;
 import org.schabi.newpipe.extractor.stream.Description;
+import org.schabi.newpipe.extractor.stream.Frameset;
 import org.schabi.newpipe.extractor.stream.Stream;
 import org.schabi.newpipe.extractor.stream.StreamExtractor;
 import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
+import org.schabi.newpipe.extractor.stream.StreamSegment;
 import org.schabi.newpipe.extractor.stream.StreamType;
 import org.schabi.newpipe.extractor.stream.SubtitlesStream;
 import org.schabi.newpipe.extractor.stream.VideoStream;
@@ -36,7 +38,6 @@ import org.schabi.newpipe.extractor.utils.JsonUtils;
 import org.schabi.newpipe.extractor.utils.Utils;
 
 import java.io.IOException;
-import java.io.UnsupportedEncodingException;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
@@ -317,8 +318,67 @@ public class PeertubeStreamExtractor extends StreamExtractor {
     }
 
     @Nonnull
-    private String getRelatedItemsUrl(@Nonnull final List<String> tags)
-            throws UnsupportedEncodingException {
+    @Override
+    public List<StreamSegment> getStreamSegments() throws ParsingException {
+        final List<StreamSegment> segments = new ArrayList<>();
+        final JsonObject segmentsJson;
+        try {
+            segmentsJson = fetchSubApiContent("chapters");
+        } catch (final IOException | ReCaptchaException e) {
+            throw new ParsingException("Could not get stream segments", e);
+        }
+        if (segmentsJson != null && segmentsJson.has("chapters")) {
+            final JsonArray segmentsArray = segmentsJson.getArray("chapters");
+            for (int i = 0; i < segmentsArray.size(); i++) {
+                final JsonObject segmentObject = segmentsArray.getObject(i);
+                segments.add(new StreamSegment(
+                        segmentObject.getString("title"),
+                        segmentObject.getInt("timecode")));
+            }
+        }
+
+        return segments;
+    }
+
+    @Nonnull
+    @Override
+    public List<Frameset> getFrames() throws ExtractionException {
+        final List<Frameset> framesets = new ArrayList<>();
+        final JsonObject storyboards;
+        try {
+            storyboards = fetchSubApiContent("storyboards");
+        } catch (final IOException | ReCaptchaException e) {
+            throw new ExtractionException("Could not get frames", e);
+        }
+        if (storyboards != null && storyboards.has("storyboards")) {
+            final JsonArray storyboardsArray = storyboards.getArray("storyboards");
+            for (final Object storyboard : storyboardsArray) {
+                if (storyboard instanceof JsonObject) {
+                    final JsonObject storyboardObject = (JsonObject) storyboard;
+                    final String url = storyboardObject.getString("storyboardPath");
+                    final int width = storyboardObject.getInt("spriteWidth");
+                    final int height = storyboardObject.getInt("spriteHeight");
+                    final int totalWidth = storyboardObject.getInt("totalWidth");
+                    final int totalHeight = storyboardObject.getInt("totalHeight");
+                    final int framesPerPageX = totalWidth / width;
+                    final int framesPerPageY = totalHeight / height;
+                    final int count = framesPerPageX * framesPerPageY;
+                    final int durationPerFrame = storyboardObject.getInt("spriteDuration") * 1000;
+
+                    framesets.add(new Frameset(
+                            // there is only one composite image per video containing all frames
+                            List.of(baseUrl + url),
+                            width, height, count,
+                            durationPerFrame, framesPerPageX, framesPerPageY));
+                }
+            }
+        }
+
+        return framesets;
+    }
+
+    @Nonnull
+    private String getRelatedItemsUrl(@Nonnull final List<String> tags) {
         final String url = baseUrl + PeertubeSearchQueryHandlerFactory.SEARCH_ENDPOINT_VIDEOS;
         final StringBuilder params = new StringBuilder();
         params.append("start=0&count=8&sort=-createdAt");
@@ -636,6 +696,41 @@ public class PeertubeStreamExtractor extends StreamExtractor {
         }
     }
 
+    /**
+     * Fetch content from a sub-API of the video.
+     * @param subPath the API subpath after the video id,
+     *                e.g. "storyboards" for "/api/v1/videos/{id}/storyboards"
+     * @return the {@link JsonObject} of the sub-API or null if the API does not exist
+     * which is the case if the instance has an outdated PeerTube version.
+     * @throws ParsingException if the API response could not be parsed to a {@link JsonObject}
+     * @throws IOException if the API response could not be fetched
+     * @throws ReCaptchaException if the API response is a reCaptcha
+     */
+    @Nullable
+    private JsonObject fetchSubApiContent(@Nonnull final String subPath)
+            throws ParsingException, IOException, ReCaptchaException {
+        final String apiUrl = baseUrl + PeertubeStreamLinkHandlerFactory.VIDEO_API_ENDPOINT
+                + getId() + "/" + subPath;
+        final Response response = getDownloader().get(apiUrl);
+        if (response == null) {
+            throw new ParsingException("Could not get segments from API.");
+        }
+        if (response.responseCode() == 400) {
+            // Chapter or segments support was added with PeerTube v6.0.0
+            // This instance does not support it yet.
+            return null;
+        }
+        if (response.responseCode() != 200) {
+            throw new ParsingException("Could not get segments from API. Response code: "
+                    + response.responseCode());
+        }
+        try {
+            return JsonParser.object().from(response.responseBody());
+        } catch (final JsonParserException e) {
+            throw new ParsingException("Could not parse json data for segments", e);
+        }
+    }
+
     @Nonnull
     @Override
     public String getName() throws ParsingException {

extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeChannelLinkHandlerFactory.java

@@ -5,6 +5,9 @@ import org.schabi.newpipe.extractor.exceptions.ParsingException;
 import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory;
 import org.schabi.newpipe.extractor.utils.Parser;
 
+import javax.annotation.Nonnull;
+import java.net.MalformedURLException;
+import java.net.URL;
 import java.util.List;
 
 public final class PeertubeChannelLinkHandlerFactory extends ListLinkHandlerFactory {
@@ -12,6 +15,7 @@ public final class PeertubeChannelLinkHandlerFactory extends ListLinkHandlerFact
     private static final PeertubeChannelLinkHandlerFactory INSTANCE
             = new PeertubeChannelLinkHandlerFactory();
     private static final String ID_PATTERN = "((accounts|a)|(video-channels|c))/([^/?&#]*)";
+    private static final String ID_URL_PATTERN = "/((accounts|a)|(video-channels|c))/([^/?&#]*)";
     public static final String API_ENDPOINT = "/api/v1/";
 
     private PeertubeChannelLinkHandlerFactory() {
@@ -23,7 +27,7 @@ public final class PeertubeChannelLinkHandlerFactory extends ListLinkHandlerFact
 
     @Override
     public String getId(final String url) throws ParsingException, UnsupportedOperationException {
-        return fixId(Parser.matchGroup(ID_PATTERN, url, 0));
+        return fixId(Parser.matchGroup(ID_URL_PATTERN, url, 0));
     }
 
     @Override
@@ -51,8 +55,13 @@ public final class PeertubeChannelLinkHandlerFactory extends ListLinkHandlerFact
 
     @Override
     public boolean onAcceptUrl(final String url) {
-        return url.contains("/accounts/") || url.contains("/a/")
-                || url.contains("/video-channels/") || url.contains("/c/");
+        try {
+            new URL(url);
+            return url.contains("/accounts/") || url.contains("/a/")
+                    || url.contains("/video-channels/") || url.contains("/c/");
+        } catch (final MalformedURLException e) {
+            return false;
+        }
     }
 
     /**
@@ -67,12 +76,13 @@ public final class PeertubeChannelLinkHandlerFactory extends ListLinkHandlerFact
      * @param id the id to fix
      * @return the fixed id
      */
-    private String fixId(final String id) {
-        if (id.startsWith("a/")) {
-            return "accounts" + id.substring(1);
-        } else if (id.startsWith("c/")) {
-            return "video-channels" + id.substring(1);
+    private String fixId(@Nonnull final String id) {
+        final String cleanedId = id.startsWith("/") ? id.substring(1) : id;
+        if (cleanedId.startsWith("a/")) {
+            return "accounts" + cleanedId.substring(1);
+        } else if (cleanedId.startsWith("c/")) {
+            return "video-channels" + cleanedId.substring(1);
         }
-        return id;
+        return cleanedId;
     }
 }

extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeCommentsLinkHandlerFactory.java

@@ -5,6 +5,8 @@ import org.schabi.newpipe.extractor.exceptions.FoundAdException;
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
 import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory;
 
+import java.net.MalformedURLException;
+import java.net.URL;
 import java.util.List;
 
 public final class PeertubeCommentsLinkHandlerFactory extends ListLinkHandlerFactory {
@@ -27,7 +29,12 @@ public final class PeertubeCommentsLinkHandlerFactory extends ListLinkHandlerFac
 
     @Override
     public boolean onAcceptUrl(final String url) throws FoundAdException {
-        return url.contains("/videos/") || url.contains("/w/");
+        try {
+            new URL(url);
+            return url.contains("/videos/") || url.contains("/w/");
+        } catch (final MalformedURLException e) {
+            return false;
+        }
     }
 
     @Override

extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubePlaylistLinkHandlerFactory.java

@@ -6,6 +6,8 @@ import org.schabi.newpipe.extractor.exceptions.ParsingException;
 import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory;
 import org.schabi.newpipe.extractor.utils.Parser;
 
+import java.net.MalformedURLException;
+import java.net.URL;
 import java.util.List;
 
 public final class PeertubePlaylistLinkHandlerFactory extends ListLinkHandlerFactory {
@@ -52,9 +54,10 @@ public final class PeertubePlaylistLinkHandlerFactory extends ListLinkHandlerFac
     @Override
     public boolean onAcceptUrl(final String url) {
         try {
+            new URL(url);
             getId(url);
             return true;
-        } catch (final ParsingException e) {
+        } catch (final ParsingException | MalformedURLException e) {
             return false;
         }
     }

extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeSearchQueryHandlerFactory.java

@@ -5,7 +5,6 @@ import org.schabi.newpipe.extractor.exceptions.ParsingException;
 import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandlerFactory;
 import org.schabi.newpipe.extractor.utils.Utils;
 
-import java.io.UnsupportedEncodingException;
 import java.util.List;
 
 public final class PeertubeSearchQueryHandlerFactory extends SearchQueryHandlerFactory {
@@ -49,21 +48,17 @@ public final class PeertubeSearchQueryHandlerFactory extends SearchQueryHandlerF
                          final String sortFilter,
                          final String baseUrl)
             throws ParsingException, UnsupportedOperationException {
-        try {
-            final String endpoint;
-            if (contentFilters.isEmpty()
-                    || contentFilters.get(0).equals(VIDEOS)
-                    || contentFilters.get(0).equals(SEPIA_VIDEOS)) {
-                endpoint = SEARCH_ENDPOINT_VIDEOS;
-            } else if (contentFilters.get(0).equals(CHANNELS)) {
-                endpoint = SEARCH_ENDPOINT_CHANNELS;
-            } else {
-                endpoint = SEARCH_ENDPOINT_PLAYLISTS;
-            }
-            return baseUrl + endpoint + "?search=" + Utils.encodeUrlUtf8(searchString);
-        } catch (final UnsupportedEncodingException e) {
-            throw new ParsingException("Could not encode query", e);
+        final String endpoint;
+        if (contentFilters.isEmpty()
+                || contentFilters.get(0).equals(VIDEOS)
+                || contentFilters.get(0).equals(SEPIA_VIDEOS)) {
+            endpoint = SEARCH_ENDPOINT_VIDEOS;
+        } else if (contentFilters.get(0).equals(CHANNELS)) {
+            endpoint = SEARCH_ENDPOINT_CHANNELS;
+        } else {
+            endpoint = SEARCH_ENDPOINT_PLAYLISTS;
         }
+        return baseUrl + endpoint + "?search=" + Utils.encodeUrlUtf8(searchString);
     }
 
     @Override

extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeStreamLinkHandlerFactory.java

@@ -6,6 +6,9 @@ import org.schabi.newpipe.extractor.exceptions.ParsingException;
 import org.schabi.newpipe.extractor.linkhandler.LinkHandlerFactory;
 import org.schabi.newpipe.extractor.utils.Parser;
 
+import java.net.MalformedURLException;
+import java.net.URL;
+
 public final class PeertubeStreamLinkHandlerFactory extends LinkHandlerFactory {
 
     private static final PeertubeStreamLinkHandlerFactory INSTANCE
@@ -47,9 +50,10 @@ public final class PeertubeStreamLinkHandlerFactory extends LinkHandlerFactory {
             return false;
         }
         try {
+            new URL(url);
             getId(url);
             return true;
-        } catch (final ParsingException e) {
+        } catch (final ParsingException | MalformedURLException e) {
             return false;
         }
     }

extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeTrendingLinkHandlerFactory.java

@@ -4,6 +4,8 @@ import org.schabi.newpipe.extractor.ServiceList;
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
 import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory;
 
+import java.net.MalformedURLException;
+import java.net.URL;
 import java.util.List;
 import java.util.Map;
 
@@ -21,7 +23,7 @@ public final class PeertubeTrendingLinkHandlerFactory extends ListLinkHandlerFac
             KIOSK_TRENDING, "%s/api/v1/videos?sort=-trending",
             KIOSK_MOST_LIKED, "%s/api/v1/videos?sort=-likes",
             KIOSK_RECENT, "%s/api/v1/videos?sort=-publishedAt",
-            KIOSK_LOCAL, "%s/api/v1/videos?sort=-publishedAt&filter=local");
+            KIOSK_LOCAL, "%s/api/v1/videos?sort=-publishedAt&isLocal=true");
 
     private PeertubeTrendingLinkHandlerFactory() {
     }
@@ -69,8 +71,13 @@ public final class PeertubeTrendingLinkHandlerFactory extends ListLinkHandlerFac
 
     @Override
     public boolean onAcceptUrl(final String url) {
-        return url.contains("/videos?") || url.contains("/videos/trending")
-                || url.contains("/videos/most-liked") || url.contains("/videos/recently-added")
-                || url.contains("/videos/local");
+        try {
+            new URL(url);
+            return url.contains("/videos?") || url.contains("/videos/trending")
+                    || url.contains("/videos/most-liked") || url.contains("/videos/recently-added")
+                    || url.contains("/videos/local");
+        } catch (final MalformedURLException e) {
+            return false;
+        }
     }
 }

extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java

@@ -45,6 +45,7 @@ import java.util.Collections;
 import java.util.List;
 import java.util.Map;
 import java.util.stream.Collectors;
+import java.util.regex.Pattern;
 
 public final class SoundcloudParsingHelper {
     // CHECKSTYLE:OFF
@@ -88,6 +89,10 @@ public final class SoundcloudParsingHelper {
     private static String clientId;
     public static final String SOUNDCLOUD_API_V2_URL = "https://api-v2.soundcloud.com/";
 
+    private static final Pattern ON_URL_PATTERN = Pattern.compile(
+        "^https?://on.soundcloud.com/[0-9a-zA-Z]+$"
+    );
+
     private SoundcloudParsingHelper() {
     }
 
@@ -185,8 +190,21 @@ public final class SoundcloudParsingHelper {
      */
     public static String resolveIdWithWidgetApi(final String urlString) throws IOException,
             ParsingException {
-        // Remove the tailing slash from URLs due to issues with the SoundCloud API
         String fixedUrl = urlString;
+
+        // if URL is an on.soundcloud link, do a request to resolve the redirect
+
+        if (ON_URL_PATTERN.matcher(fixedUrl).find()) {
+            try {
+                fixedUrl = NewPipe.getDownloader().head(fixedUrl).latestUrl();
+                // remove tracking params which are in the query string
+                fixedUrl = fixedUrl.split("\\?")[0];
+            } catch (final ExtractionException e) {
+                throw new ParsingException("Could not follow on.soundcloud.com redirect", e);
+            }
+        }
+
+        // Remove the tailing slash from URLs due to issues with the SoundCloud API
         if (fixedUrl.charAt(fixedUrl.length() - 1) == '/') {
             fixedUrl = fixedUrl.substring(0, fixedUrl.length() - 1);
         }

extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudCommentsInfoItemExtractor.java

@@ -29,6 +29,7 @@ public class SoundcloudCommentsInfoItemExtractor implements CommentsInfoItemExtr
         return Objects.toString(json.getLong("id"), null);
     }
 
+    @Nonnull
     @Override
     public Description getCommentText() {
         return new Description(json.getString("body"), Description.PLAIN_TEXT);

extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudSearchExtractor.java

@@ -23,7 +23,6 @@ import org.schabi.newpipe.extractor.search.SearchExtractor;
 import org.schabi.newpipe.extractor.utils.Parser;
 
 import java.io.IOException;
-import java.io.UnsupportedEncodingException;
 import java.net.MalformedURLException;
 import java.net.URL;
 import java.util.Collections;
@@ -159,7 +158,7 @@ public class SoundcloudSearchExtractor extends SearchExtractor {
     private int getOffsetFromUrl(final String url) throws ParsingException {
         try {
             return Integer.parseInt(Parser.compatParseMap(new URL(url).getQuery()).get("offset"));
-        } catch (MalformedURLException | UnsupportedEncodingException e) {
+        } catch (final MalformedURLException e) {
             throw new ParsingException("Could not get offset from page URL", e);
         }
     }

extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudStreamExtractor.java

@@ -39,7 +39,6 @@ import org.schabi.newpipe.extractor.stream.VideoStream;
 import org.schabi.newpipe.extractor.utils.Utils;
 
 import java.io.IOException;
-import java.io.UnsupportedEncodingException;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
@@ -317,14 +316,6 @@ public class SoundcloudStreamExtractor extends StreamExtractor {
         }
     }
 
-    private static String urlEncode(final String value) {
-        try {
-            return Utils.encodeUrlUtf8(value);
-        } catch (final UnsupportedEncodingException e) {
-            throw new IllegalStateException(e);
-        }
-    }
-
     @Override
     public List<VideoStream> getVideoStreams() {
         return Collections.emptyList();
@@ -344,9 +335,8 @@ public class SoundcloudStreamExtractor extends StreamExtractor {
     @Override
     public StreamInfoItemsCollector getRelatedItems() throws IOException, ExtractionException {
         final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
-
-        final String apiUrl = SOUNDCLOUD_API_V2_URL + "tracks/" + urlEncode(getId())
-                + "/related?client_id=" + urlEncode(clientId());
+        final String apiUrl = SOUNDCLOUD_API_V2_URL + "tracks/" + Utils.encodeUrlUtf8(getId())
+                + "/related?client_id=" + Utils.encodeUrlUtf8(clientId());
 
         SoundcloudParsingHelper.getStreamsFromApi(collector, apiUrl);
         return collector;

extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/linkHandler/SoundcloudSearchQueryHandlerFactory.java

@@ -10,7 +10,6 @@ import org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsingHelper;
 import org.schabi.newpipe.extractor.utils.Utils;
 
 import java.io.IOException;
-import java.io.UnsupportedEncodingException;
 import java.util.List;
 
 public final class SoundcloudSearchQueryHandlerFactory extends SearchQueryHandlerFactory {
@@ -60,9 +59,6 @@ public final class SoundcloudSearchQueryHandlerFactory extends SearchQueryHandle
             return url + "?q=" + Utils.encodeUrlUtf8(id)
                     + "&client_id=" + SoundcloudParsingHelper.clientId()
                     + "&limit=" + ITEMS_PER_PAGE + "&offset=0";
-
-        } catch (final UnsupportedEncodingException e) {
-            throw new ParsingException("Could not encode query", e);
         } catch (final ReCaptchaException e) {
             throw new ParsingException("ReCaptcha required", e);
         } catch (final IOException | ExtractionException e) {

extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/linkHandler/SoundcloudStreamLinkHandlerFactory.java

@@ -9,7 +9,8 @@ import org.schabi.newpipe.extractor.utils.Utils;
 public final class SoundcloudStreamLinkHandlerFactory extends LinkHandlerFactory {
     private static final SoundcloudStreamLinkHandlerFactory INSTANCE
             = new SoundcloudStreamLinkHandlerFactory();
-    private static final String URL_PATTERN = "^https?://(www\\.|m\\.)?soundcloud.com/[0-9a-z_-]+"
+    private static final String URL_PATTERN = "^https?://(www\\.|m\\.|on\\.)?"
+            + "soundcloud.com/[0-9a-z_-]+"
             + "/(?!(tracks|albums|sets|reposts|followers|following)/?$)[0-9a-z_-]+/?([#?].*)?$";
     private static final String API_URL_PATTERN = "^https?://api-v2\\.soundcloud.com"
             + "/(tracks|albums|sets|reposts|followers|following)/([0-9a-z_-]+)/";

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelHelper.java

@@ -4,16 +4,21 @@ import com.grack.nanojson.JsonObject;
 import com.grack.nanojson.JsonWriter;
 import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException;
 import org.schabi.newpipe.extractor.exceptions.ExtractionException;
+import org.schabi.newpipe.extractor.exceptions.ParsingException;
 import org.schabi.newpipe.extractor.localization.ContentCountry;
 import org.schabi.newpipe.extractor.localization.Localization;
 
 import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
 import java.io.IOException;
+import java.io.Serializable;
 import java.nio.charset.StandardCharsets;
 import java.util.Optional;
 
 import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.defaultAlertsCheck;
 import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonPostResponse;
+import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject;
+import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.hasArtistOrVerifiedIconBadgeAttachment;
 import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareDesktopJsonBuilder;
 import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
 
@@ -21,6 +26,19 @@ import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
  * Shared functions for extracting YouTube channel pages and tabs.
  */
 public final class YoutubeChannelHelper {
+
+    private static final String BROWSE_ENDPOINT = "browseEndpoint";
+    private static final String BROWSE_ID = "browseId";
+    private static final String CAROUSEL_HEADER_RENDERER = "carouselHeaderRenderer";
+    private static final String C4_TABBED_HEADER_RENDERER = "c4TabbedHeaderRenderer";
+    private static final String CONTENT = "content";
+    private static final String CONTENTS = "contents";
+    private static final String HEADER = "header";
+    private static final String PAGE_HEADER_VIEW_MODEL = "pageHeaderViewModel";
+    private static final String TAB_RENDERER = "tabRenderer";
+    private static final String TITLE = "title";
+    private static final String TOPIC_CHANNEL_DETAILS_RENDERER = "topicChannelDetailsRenderer";
+
     private YoutubeChannelHelper() {
     }
 
@@ -64,8 +82,8 @@ public final class YoutubeChannelHelper {
                     .getObject("webCommandMetadata")
                     .getString("webPageType", "");
 
-            final JsonObject browseEndpoint = endpoint.getObject("browseEndpoint");
-            final String browseId = browseEndpoint.getString("browseId", "");
+            final JsonObject browseEndpoint = endpoint.getObject(BROWSE_ENDPOINT);
+            final String browseId = browseEndpoint.getString(BROWSE_ID, "");
 
             if (webPageType.equalsIgnoreCase("WEB_PAGE_TYPE_BROWSE")
                     || webPageType.equalsIgnoreCase("WEB_PAGE_TYPE_CHANNEL")
@@ -140,7 +158,7 @@ public final class YoutubeChannelHelper {
         while (level < 3) {
             final byte[] body = JsonWriter.string(prepareDesktopJsonBuilder(
                                     localization, country)
-                            .value("browseId", id)
+                            .value(BROWSE_ID, id)
                             .value("params", parameters)
                             .done())
                     .getBytes(StandardCharsets.UTF_8);
@@ -159,8 +177,8 @@ public final class YoutubeChannelHelper {
                     .getObject("webCommandMetadata")
                     .getString("webPageType", "");
 
-            final String browseId = endpoint.getObject("browseEndpoint")
-                    .getString("browseId", "");
+            final String browseId = endpoint.getObject(BROWSE_ENDPOINT)
+                    .getString(BROWSE_ID, "");
 
             if (webPageType.equalsIgnoreCase("WEB_PAGE_TYPE_BROWSE")
                     || webPageType.equalsIgnoreCase("WEB_PAGE_TYPE_CHANNEL")
@@ -217,7 +235,7 @@ public final class YoutubeChannelHelper {
      * properties.
      * </p>
      */
-    public static final class ChannelHeader {
+    public static final class ChannelHeader implements Serializable {
 
         /**
          * Types of supported YouTube channel headers.
@@ -257,7 +275,7 @@ public final class YoutubeChannelHelper {
              * A {@code pageHeaderRenderer} channel header type.
              *
              * <p>
-             * This header returns only the channel's name and its avatar.
+             * This header returns only the channel's name and its avatar for system channels.
              * </p>
              */
             PAGE
@@ -278,46 +296,248 @@ public final class YoutubeChannelHelper {
          */
         public final HeaderType headerType;
 
-        private ChannelHeader(@Nonnull final JsonObject json, final HeaderType headerType) {
+        public ChannelHeader(@Nonnull final JsonObject json, final HeaderType headerType) {
             this.json = json;
             this.headerType = headerType;
         }
     }
 
     /**
-     * Get a channel header as an {@link Optional} it if exists.
+     * Get a channel header it if exists.
      *
      * @param channelResponse a full channel JSON response
-     * @return an {@link Optional} containing a {@link ChannelHeader} or an empty {@link Optional}
-     * if no supported header has been found
+     * @return a {@link ChannelHeader} or {@code null} if no supported header has been found
      */
-    @Nonnull
-    public static Optional<ChannelHeader> getChannelHeader(
+    @Nullable
+    public static ChannelHeader getChannelHeader(
             @Nonnull final JsonObject channelResponse) {
-        final JsonObject header = channelResponse.getObject("header");
+        final JsonObject header = channelResponse.getObject(HEADER);
 
-        if (header.has("c4TabbedHeaderRenderer")) {
-            return Optional.of(header.getObject("c4TabbedHeaderRenderer"))
-                    .map(json -> new ChannelHeader(json, ChannelHeader.HeaderType.C4_TABBED));
-        } else if (header.has("carouselHeaderRenderer")) {
-            return header.getObject("carouselHeaderRenderer")
-                    .getArray("contents")
+        if (header.has(C4_TABBED_HEADER_RENDERER)) {
+            return Optional.of(header.getObject(C4_TABBED_HEADER_RENDERER))
+                    .map(json -> new ChannelHeader(json, ChannelHeader.HeaderType.C4_TABBED))
+                    .orElse(null);
+        } else if (header.has(CAROUSEL_HEADER_RENDERER)) {
+            return header.getObject(CAROUSEL_HEADER_RENDERER)
+                    .getArray(CONTENTS)
                     .stream()
                     .filter(JsonObject.class::isInstance)
                     .map(JsonObject.class::cast)
-                    .filter(item -> item.has("topicChannelDetailsRenderer"))
+                    .filter(item -> item.has(TOPIC_CHANNEL_DETAILS_RENDERER))
                     .findFirst()
-                    .map(item -> item.getObject("topicChannelDetailsRenderer"))
-                    .map(json -> new ChannelHeader(json, ChannelHeader.HeaderType.CAROUSEL));
+                    .map(item -> item.getObject(TOPIC_CHANNEL_DETAILS_RENDERER))
+                    .map(json -> new ChannelHeader(json, ChannelHeader.HeaderType.CAROUSEL))
+                    .orElse(null);
         } else if (header.has("pageHeaderRenderer")) {
             return Optional.of(header.getObject("pageHeaderRenderer"))
-                    .map(json -> new ChannelHeader(json, ChannelHeader.HeaderType.PAGE));
+                    .map(json -> new ChannelHeader(json, ChannelHeader.HeaderType.PAGE))
+                    .orElse(null);
         } else if (header.has("interactiveTabbedHeaderRenderer")) {
             return Optional.of(header.getObject("interactiveTabbedHeaderRenderer"))
                     .map(json -> new ChannelHeader(json,
-                            ChannelHeader.HeaderType.INTERACTIVE_TABBED));
-        } else {
-            return Optional.empty();
+                            ChannelHeader.HeaderType.INTERACTIVE_TABBED))
+                    .orElse(null);
+        }
+
+        return null;
+    }
+
+    /**
+     * Check if a channel is verified by using its header.
+     *
+     * <p>
+     * The header is mandatory, so the verified status of age-restricted channels with a
+     * {@code channelAgeGateRenderer} cannot be checked.
+     * </p>
+     *
+     * @param channelHeader the {@link ChannelHeader} of a non age-restricted channel
+     * @return whether the channel is verified
+     */
+    public static boolean isChannelVerified(@Nonnull final ChannelHeader channelHeader) {
+        switch (channelHeader.headerType) {
+            // carouselHeaderRenderers do not contain any verification badges
+            // Since they are only shown on YouTube internal channels or on channels of large
+            // organizations broadcasting live events, we can assume the channel to be verified
+            case CAROUSEL:
+                return true;
+            case PAGE:
+                final JsonObject pageHeaderViewModel = channelHeader.json.getObject(CONTENT)
+                        .getObject(PAGE_HEADER_VIEW_MODEL);
+
+                final boolean hasCircleOrMusicIcon = hasArtistOrVerifiedIconBadgeAttachment(
+                        pageHeaderViewModel.getObject(TITLE)
+                                .getObject("dynamicTextViewModel")
+                                .getObject("text")
+                                .getArray("attachmentRuns"));
+                if (!hasCircleOrMusicIcon && pageHeaderViewModel.getObject("image")
+                        .has("contentPreviewImageViewModel")) {
+                    // If a pageHeaderRenderer has no object in which a check verified may be
+                    // contained and if it has a contentPreviewImageViewModel, it should mean
+                    // that the header is coming from a system channel, which we can assume to
+                    // be verified
+                    return true;
+                }
+
+                return hasCircleOrMusicIcon;
+            case INTERACTIVE_TABBED:
+                // If the header has an autoGenerated property, it should mean that the channel has
+                // been auto generated by YouTube: we can assume the channel to be verified in this
+                // case
+                return channelHeader.json.has("autoGenerated");
+            default:
+                return YoutubeParsingHelper.isVerified(channelHeader.json.getArray("badges"));
         }
     }
+
+    /**
+     * Get the ID of a channel from its response.
+     *
+     * <p>
+     * For {@link ChannelHeader.HeaderType#C4_TABBED c4TabbedHeaderRenderer} and
+     * {@link ChannelHeader.HeaderType#CAROUSEL carouselHeaderRenderer} channel headers, the ID is
+     * get from the header.
+     * </p>
+     *
+     * <p>
+     * For other headers or if it cannot be got, the ID from the {@code channelMetadataRenderer}
+     * in the channel response is used.
+     * </p>
+     *
+     * <p>
+     * If the ID cannot still be get, the fallback channel ID, if provided, will be used.
+     * </p>
+     *
+     * @param channelHeader     the channel header
+     * @param fallbackChannelId the fallback channel ID, which can be null
+     * @return the ID of the channel
+     * @throws ParsingException if the channel ID cannot be got from the channel header, the
+     * channel response and the fallback channel ID
+     */
+    @Nonnull
+    public static String getChannelId(
+            @Nullable final ChannelHeader channelHeader,
+            @Nonnull final JsonObject jsonResponse,
+            @Nullable final String fallbackChannelId) throws ParsingException {
+        if (channelHeader != null) {
+            switch (channelHeader.headerType) {
+                case C4_TABBED:
+                    final String channelId = channelHeader.json.getObject(HEADER)
+                            .getObject(C4_TABBED_HEADER_RENDERER)
+                            .getString("channelId", "");
+                    if (!isNullOrEmpty(channelId)) {
+                        return channelId;
+                    }
+                    final String navigationC4TabChannelId = channelHeader.json
+                            .getObject("navigationEndpoint")
+                            .getObject(BROWSE_ENDPOINT)
+                            .getString(BROWSE_ID);
+                    if (!isNullOrEmpty(navigationC4TabChannelId)) {
+                        return navigationC4TabChannelId;
+                    }
+                    break;
+                case CAROUSEL:
+                    final String navigationCarouselChannelId = channelHeader.json.getObject(HEADER)
+                            .getObject(CAROUSEL_HEADER_RENDERER)
+                            .getArray(CONTENTS)
+                            .stream()
+                            .filter(JsonObject.class::isInstance)
+                            .map(JsonObject.class::cast)
+                            .filter(item -> item.has(TOPIC_CHANNEL_DETAILS_RENDERER))
+                            .findFirst()
+                            .orElse(new JsonObject())
+                            .getObject(TOPIC_CHANNEL_DETAILS_RENDERER)
+                            .getObject("navigationEndpoint")
+                            .getObject(BROWSE_ENDPOINT)
+                            .getString(BROWSE_ID);
+                    if (!isNullOrEmpty(navigationCarouselChannelId)) {
+                        return navigationCarouselChannelId;
+                    }
+                    break;
+                default:
+                    break;
+            }
+        }
+
+        final String externalChannelId = jsonResponse.getObject("metadata")
+                .getObject("channelMetadataRenderer")
+                .getString("externalChannelId");
+        if (!isNullOrEmpty(externalChannelId)) {
+            return externalChannelId;
+        }
+
+        if (!isNullOrEmpty(fallbackChannelId)) {
+            return fallbackChannelId;
+        } else {
+            throw new ParsingException("Could not get channel ID");
+        }
+    }
+
+    @Nonnull
+    public static String getChannelName(@Nullable final ChannelHeader channelHeader,
+                                        @Nullable final JsonObject channelAgeGateRenderer,
+                                        @Nonnull final JsonObject jsonResponse)
+            throws ParsingException {
+        if (channelAgeGateRenderer != null) {
+            final String title = channelAgeGateRenderer.getString("channelTitle");
+            if (isNullOrEmpty(title)) {
+                throw new ParsingException("Could not get channel name");
+            }
+            return title;
+        }
+
+        final String metadataRendererTitle = jsonResponse.getObject("metadata")
+                .getObject("channelMetadataRenderer")
+                .getString(TITLE);
+        if (!isNullOrEmpty(metadataRendererTitle)) {
+            return metadataRendererTitle;
+        }
+
+        return Optional.ofNullable(channelHeader)
+                .map(header -> {
+                    final JsonObject channelJson = header.json;
+                    switch (header.headerType) {
+                        case PAGE:
+                            return channelJson.getObject(CONTENT)
+                                    .getObject(PAGE_HEADER_VIEW_MODEL)
+                                    .getObject(TITLE)
+                                    .getObject("dynamicTextViewModel")
+                                    .getObject("text")
+                                    .getString(CONTENT, channelJson.getString("pageTitle"));
+                        case CAROUSEL:
+                        case INTERACTIVE_TABBED:
+                            return getTextFromObject(channelJson.getObject(TITLE));
+                        case C4_TABBED:
+                        default:
+                            return channelJson.getString(TITLE);
+                    }
+                })
+                // The channel name from a microformatDataRenderer may be different from the one
+                // displayed, especially for auto-generated channels, depending on the language
+                // requested for the interface (hl parameter of InnerTube requests' payload)
+                .or(() -> Optional.ofNullable(jsonResponse.getObject("microformat")
+                        .getObject("microformatDataRenderer")
+                        .getString(TITLE)))
+                .orElseThrow(() -> new ParsingException("Could not get channel name"));
+    }
+
+    @Nullable
+    public static JsonObject getChannelAgeGateRenderer(@Nonnull final JsonObject jsonResponse) {
+        return jsonResponse.getObject(CONTENTS)
+                .getObject("twoColumnBrowseResultsRenderer")
+                .getArray("tabs")
+                .stream()
+                .filter(JsonObject.class::isInstance)
+                .map(JsonObject.class::cast)
+                .flatMap(tab -> tab.getObject(TAB_RENDERER)
+                        .getObject(CONTENT)
+                        .getObject("sectionListRenderer")
+                        .getArray(CONTENTS)
+                        .stream()
+                        .filter(JsonObject.class::isInstance)
+                        .map(JsonObject.class::cast))
+                .filter(content -> content.has("channelAgeGateRenderer"))
+                .map(content -> content.getObject("channelAgeGateRenderer"))
+                .findFirst()
+                .orElse(null);
+    }
 }

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeDescriptionHelper.java

@@ -0,0 +1,316 @@
+package org.schabi.newpipe.extractor.services.youtube;
+
+import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getUrlFromNavigationEndpoint;
+import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
+
+import com.grack.nanojson.JsonObject;
+
+import org.jsoup.nodes.Entities;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.List;
+import java.util.Stack;
+import java.util.function.Function;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
+
+public final class YoutubeDescriptionHelper {
+
+    private YoutubeDescriptionHelper() {
+    }
+
+    private static final String LINK_CLOSE = "</a>";
+    private static final String STRIKETHROUGH_OPEN = "<s>";
+    private static final String STRIKETHROUGH_CLOSE = "</s>";
+    private static final String BOLD_OPEN = "<b>";
+    private static final String BOLD_CLOSE = "</b>";
+    private static final String ITALIC_OPEN = "<i>";
+    private static final String ITALIC_CLOSE = "</i>";
+
+    // special link chips (e.g. for YT videos, YT channels or social media accounts):
+    // (u00a0) u00a0 u00a0 [/•] u00a0 <link content> u00a0 u00a0
+    private static final Pattern LINK_CONTENT_CLEANER_REGEX
+            = Pattern.compile("(?s)^ +[/•] +(.*?) +$");
+
+    /**
+     * Can be a command run, or a style run.
+     */
+    static final class Run {
+        @Nonnull final String open;
+        @Nonnull final String close;
+        final int pos;
+        @Nullable final Function<String, String> transformContent;
+        int openPosInOutput = -1;
+
+        Run(
+                @Nonnull final String open,
+                @Nonnull final String close,
+                final int pos
+        ) {
+            this(open, close, pos, null);
+        }
+
+        Run(
+                @Nonnull final String open,
+                @Nonnull final String close,
+                final int pos,
+                @Nullable final Function<String, String> transformContent
+        ) {
+            this.open = open;
+            this.close = close;
+            this.pos = pos;
+            this.transformContent = transformContent;
+        }
+
+        public boolean sameOpen(@Nonnull final Run other) {
+            return open.equals(other.open);
+        }
+    }
+
+    /**
+     * Parse a video description in the new "attributed" format, which contains the entire visible
+     * plaintext ({@code content}) and an array of {@code commandRuns} and {@code styleRuns}.
+     * Returns the formatted content in HTML format, and escapes the text to make sure there are no
+     * XSS attacks.
+     *
+     * <p>
+     * {@code commandRuns} include the links and their range in the text, while {@code styleRuns}
+     * include the styling to apply to various ranges in the text.
+     * </p>
+     *
+     * @param attributedDescription the JSON object of the attributed description
+     * @return the parsed description, in HTML format, as a string
+     */
+    @Nullable
+    public static String attributedDescriptionToHtml(
+            @Nullable final JsonObject attributedDescription
+    ) {
+        if (isNullOrEmpty(attributedDescription)) {
+            return null;
+        }
+
+        final String content = attributedDescription.getString("content");
+        if (content == null) {
+            return null;
+        }
+
+        // all run pairs must always of length at least 1, or they should be discarded,
+        // otherwise various assumptions made in runsToHtml may fail
+        final List<Run> openers = new ArrayList<>();
+        final List<Run> closers = new ArrayList<>();
+        addAllCommandRuns(attributedDescription, openers, closers);
+        addAllStyleRuns(attributedDescription, openers, closers);
+
+        // Note that sorting this way might put closers with the same close position in the wrong
+        // order with respect to their openers, causing unnecessary closes and reopens. E.g.
+        // <b>b<i>b&i</i></b> is instead generated as <b>b<i>b&i</b></i><b></b> if the </b> is
+        // encountered before the </i>. Solving this wouldn't be difficult, thanks to stable sort,
+        // but would require additional sorting steps which would just make this slower for the
+        // general case where it's unlikely there are coincident closes.
+        Collections.sort(openers, Comparator.comparingInt(run -> run.pos));
+        Collections.sort(closers, Comparator.comparingInt(run -> run.pos));
+
+        return runsToHtml(openers, closers, content);
+    }
+
+    /**
+     * Applies the formatting specified by the intervals stored in {@code openers} and {@code
+     * closers} to {@code content} in order to obtain valid HTML even when intervals overlap. For
+     * example &lt;b&gt;b&lt;i&gt;b&i&lt;/b&gt;i&lt;/i&gt; would not be valid HTML, so this function
+     * instead generates &lt;b&gt;b&lt;i&gt;b&i&lt;/i&gt;&lt;/b&gt;&lt;i&gt;i&lt;/i&gt;. Any HTML
+     * special characters in {@code rawContent} are escaped to make sure there are no XSS attacks.
+     *
+     * <p>
+     * Every opener in {@code openers} must have a corresponding closer in {@code closers}. Every
+     * corresponding (opener, closer) pair must have a length of at least one (i.e. empty intervals
+     * are not allowed).
+     * </p>
+     *
+     * @param openers    contains all of the places where a run begins, must have the same size of
+     *                   closers, must be ordered by {@link Run#pos}
+     * @param closers    contains all of the places where a run ends, must have the same size of
+     *                   openers, must be ordered by {@link Run#pos}
+     * @param rawContent the content to apply formatting to, and to escape to avoid XSS
+     * @return the formatted content in HTML
+     */
+    static String runsToHtml(
+            @Nonnull final List<Run> openers,
+            @Nonnull final List<Run> closers,
+            @Nonnull final String rawContent
+    ) {
+        final String content = rawContent.replace('\u00a0', ' ');
+        final Stack<Run> openRuns = new Stack<>();
+        final Stack<Run> tempStack = new Stack<>();
+        final StringBuilder textBuilder = new StringBuilder();
+        int currentTextPos = 0;
+        int openersIndex = 0;
+        int closersIndex = 0;
+
+        // openers and closers have the same length, but we will surely finish openers earlier than
+        // closers, since every opened interval needs to be closed at some point and there can't be
+        // empty intervals, hence check only closersIndex < closers.size()
+        while (closersIndex < closers.size()) {
+            final int minPos = openersIndex < openers.size()
+                    ? Math.min(closers.get(closersIndex).pos, openers.get(openersIndex).pos)
+                    : closers.get(closersIndex).pos;
+
+            // append piece of text until current index
+            textBuilder.append(Entities.escape(content.substring(currentTextPos, minPos)));
+            currentTextPos = minPos;
+
+            if (closers.get(closersIndex).pos == minPos) {
+                // even in case of position tie, first process closers
+                final Run closer = closers.get(closersIndex);
+                ++closersIndex;
+
+                // because of the assumptions, this while wouldn't need the !openRuns.empty()
+                // condition, because no run will close before being opened, but let's be sure
+                while (!openRuns.empty()) {
+                    final Run popped = openRuns.pop();
+                    if (popped.sameOpen(closer)) {
+                        // before closing the current run, if the run has a transformContent
+                        // function, use it to transform the content of the current run, based on
+                        // the openPosInOutput set when the current run was opened
+                        if (popped.transformContent != null && popped.openPosInOutput >= 0) {
+                            textBuilder.replace(popped.openPosInOutput, textBuilder.length(),
+                                    popped.transformContent.apply(
+                                            textBuilder.substring(popped.openPosInOutput)));
+                        }
+                        // close the run that we really need to close
+                        textBuilder.append(popped.close);
+                        break;
+                    }
+                    // we keep popping from openRuns, closing all of the runs we find,
+                    // until we find the run that we really need to close ...
+                    textBuilder.append(popped.close);
+                    tempStack.push(popped);
+                }
+                while (!tempStack.empty()) {
+                    // ... and then we reopen all of the runs that we didn't need to close
+                    // e.g. in <b>b<i>b&i</b>i</i>, when </b> is encountered, </i></b><i> is printed
+                    // instead, to make sure the HTML is valid, obtaining <b>b<i>b&i</i></b><i>i</i>
+                    final Run popped = tempStack.pop();
+                    textBuilder.append(popped.open);
+                    openRuns.push(popped);
+                }
+
+            } else {
+                // this will never be reached if openersIndex >= openers.size() because of the
+                // way minPos is calculated
+                final Run opener = openers.get(openersIndex);
+                textBuilder.append(opener.open);
+                opener.openPosInOutput = textBuilder.length(); // save for transforming later
+                openRuns.push(opener);
+                ++openersIndex;
+            }
+        }
+
+        // append last piece of text
+        textBuilder.append(Entities.escape(content.substring(currentTextPos)));
+
+        return textBuilder.toString()
+                .replace("\n", "<br>")
+                .replace("  ", " &nbsp;");
+    }
+
+    private static void addAllCommandRuns(
+            @Nonnull final JsonObject attributedDescription,
+            @Nonnull final List<Run> openers,
+            @Nonnull final List<Run> closers
+    ) {
+        attributedDescription.getArray("commandRuns")
+                .stream()
+                .filter(JsonObject.class::isInstance)
+                .map(JsonObject.class::cast)
+                .forEach(run -> {
+                    final JsonObject navigationEndpoint = run.getObject("onTap")
+                            .getObject("innertubeCommand");
+
+                    final int startIndex = run.getInt("startIndex", -1);
+                    final int length = run.getInt("length", 0);
+                    if (startIndex < 0 || length < 1 || navigationEndpoint == null) {
+                        return;
+                    }
+
+                    final String url = getUrlFromNavigationEndpoint(navigationEndpoint);
+                    if (url == null) {
+                        return;
+                    }
+
+                    final String open = "<a href=\"" + Entities.escape(url) + "\">";
+                    final Function<String, String> transformContent = getTransformContentFun(run);
+
+                    openers.add(new Run(open, LINK_CLOSE, startIndex, transformContent));
+                    closers.add(new Run(open, LINK_CLOSE, startIndex + length, transformContent));
+                });
+    }
+
+    private static Function<String, String> getTransformContentFun(final JsonObject run) {
+        final String accessibilityLabel = run.getObject("onTapOptions")
+                .getObject("accessibilityInfo")
+                .getString("accessibilityLabel", "")
+                // accessibility labels are e.g. "Instagram Channel Link: instagram_profile_name"
+                .replaceFirst(" Channel Link", "");
+
+        final Function<String, String> transformContent;
+        if (accessibilityLabel.isEmpty() || accessibilityLabel.startsWith("YouTube: ")) {
+            // if there is no accessibility label, or the link points to YouTube, cleanup the link
+            // text, see LINK_CONTENT_CLEANER_REGEX's documentation for more details
+            transformContent = (content) -> {
+                final Matcher m = LINK_CONTENT_CLEANER_REGEX.matcher(content);
+                if (m.find()) {
+                    return m.group(1);
+                }
+                return content;
+            };
+        } else {
+            // if there is an accessibility label, replace the link text with it, because on the
+            // YouTube website an ambiguous link text is next to an icon explaining which service it
+            // belongs to, but since we can't add icons, we instead use the accessibility label
+            // which contains information about the service
+            transformContent = (content) -> accessibilityLabel;
+        }
+
+        return transformContent;
+    }
+
+    private static void addAllStyleRuns(
+            @Nonnull final JsonObject attributedDescription,
+            @Nonnull final List<Run> openers,
+            @Nonnull final List<Run> closers
+    ) {
+        attributedDescription.getArray("styleRuns")
+                .stream()
+                .filter(JsonObject.class::isInstance)
+                .map(JsonObject.class::cast)
+                .forEach(run -> {
+                    final int start = run.getInt("startIndex", -1);
+                    final int length = run.getInt("length", 0);
+                    if (start < 0 || length < 1) {
+                        return;
+                    }
+                    final int end = start + length;
+
+                    if (run.has("strikethrough")) {
+                        openers.add(new Run(STRIKETHROUGH_OPEN, STRIKETHROUGH_CLOSE, start));
+                        closers.add(new Run(STRIKETHROUGH_OPEN, STRIKETHROUGH_CLOSE, end));
+                    }
+
+                    if (run.getBoolean("italic", false)) {
+                        openers.add(new Run(ITALIC_OPEN, ITALIC_CLOSE, start));
+                        closers.add(new Run(ITALIC_OPEN, ITALIC_CLOSE, end));
+                    }
+
+                    if (run.has("weightLabel")
+                            && !"FONT_WEIGHT_NORMAL".equals(run.getString("weightLabel"))) {
+                        openers.add(new Run(BOLD_OPEN, BOLD_CLOSE, start));
+                        closers.add(new Run(BOLD_OPEN, BOLD_CLOSE, end));
+                    }
+                });
+    }
+}

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeMetaInfoHelper.java

@@ -0,0 +1,218 @@
+package org.schabi.newpipe.extractor.services.youtube;
+
+import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.extractCachedUrlIfNeeded;
+import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject;
+import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObjectOrThrow;
+import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getUrlFromNavigationEndpoint;
+import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.isGoogleURL;
+import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
+import static org.schabi.newpipe.extractor.utils.Utils.replaceHttpWithHttps;
+
+import com.grack.nanojson.JsonArray;
+import com.grack.nanojson.JsonObject;
+
+import org.schabi.newpipe.extractor.MetaInfo;
+import org.schabi.newpipe.extractor.exceptions.ParsingException;
+import org.schabi.newpipe.extractor.stream.Description;
+
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Objects;
+import java.util.function.Consumer;
+import java.util.stream.Collectors;
+
+import javax.annotation.Nonnull;
+
+public final class YoutubeMetaInfoHelper {
+
+    private YoutubeMetaInfoHelper() {
+    }
+
+
+    @Nonnull
+    public static List<MetaInfo> getMetaInfo(@Nonnull final JsonArray contents)
+            throws ParsingException {
+        final List<MetaInfo> metaInfo = new ArrayList<>();
+        for (final Object content : contents) {
+            final JsonObject resultObject = (JsonObject) content;
+            if (resultObject.has("itemSectionRenderer")) {
+                for (final Object sectionContentObject
+                        : resultObject.getObject("itemSectionRenderer").getArray("contents")) {
+
+                    final JsonObject sectionContent = (JsonObject) sectionContentObject;
+                    if (sectionContent.has("infoPanelContentRenderer")) {
+                        metaInfo.add(getInfoPanelContent(sectionContent
+                                .getObject("infoPanelContentRenderer")));
+                    }
+                    if (sectionContent.has("clarificationRenderer")) {
+                        metaInfo.add(getClarificationRenderer(sectionContent
+                                .getObject("clarificationRenderer")
+                        ));
+                    }
+                    if (sectionContent.has("emergencyOneboxRenderer")) {
+                        getEmergencyOneboxRenderer(
+                                sectionContent.getObject("emergencyOneboxRenderer"),
+                                metaInfo::add
+                        );
+                    }
+                }
+            }
+        }
+        return metaInfo;
+    }
+
+    @Nonnull
+    private static MetaInfo getInfoPanelContent(@Nonnull final JsonObject infoPanelContentRenderer)
+            throws ParsingException {
+        final MetaInfo metaInfo = new MetaInfo();
+        final StringBuilder sb = new StringBuilder();
+        for (final Object paragraph : infoPanelContentRenderer.getArray("paragraphs")) {
+            if (sb.length() != 0) {
+                sb.append("<br>");
+            }
+            sb.append(getTextFromObject((JsonObject) paragraph));
+        }
+        metaInfo.setContent(new Description(sb.toString(), Description.HTML));
+        if (infoPanelContentRenderer.has("sourceEndpoint")) {
+            final String metaInfoLinkUrl = getUrlFromNavigationEndpoint(
+                    infoPanelContentRenderer.getObject("sourceEndpoint"));
+            try {
+                metaInfo.addUrl(new URL(Objects.requireNonNull(extractCachedUrlIfNeeded(
+                        metaInfoLinkUrl))));
+            } catch (final NullPointerException | MalformedURLException e) {
+                throw new ParsingException("Could not get metadata info URL", e);
+            }
+
+            final String metaInfoLinkText = getTextFromObject(
+                    infoPanelContentRenderer.getObject("inlineSource"));
+            if (isNullOrEmpty(metaInfoLinkText)) {
+                throw new ParsingException("Could not get metadata info link text.");
+            }
+            metaInfo.addUrlText(metaInfoLinkText);
+        }
+
+        return metaInfo;
+    }
+
+    @Nonnull
+    private static MetaInfo getClarificationRenderer(
+            @Nonnull final JsonObject clarificationRenderer) throws ParsingException {
+        final MetaInfo metaInfo = new MetaInfo();
+
+        final String title = getTextFromObject(clarificationRenderer
+                .getObject("contentTitle"));
+        final String text = getTextFromObject(clarificationRenderer
+                .getObject("text"));
+        if (title == null || text == null) {
+            throw new ParsingException("Could not extract clarification renderer content");
+        }
+        metaInfo.setTitle(title);
+        metaInfo.setContent(new Description(text, Description.PLAIN_TEXT));
+
+        if (clarificationRenderer.has("actionButton")) {
+            final JsonObject actionButton = clarificationRenderer.getObject("actionButton")
+                    .getObject("buttonRenderer");
+            try {
+                final String url = getUrlFromNavigationEndpoint(actionButton
+                        .getObject("command"));
+                metaInfo.addUrl(new URL(Objects.requireNonNull(extractCachedUrlIfNeeded(url))));
+            } catch (final NullPointerException | MalformedURLException e) {
+                throw new ParsingException("Could not get metadata info URL", e);
+            }
+
+            final String metaInfoLinkText = getTextFromObject(
+                    actionButton.getObject("text"));
+            if (isNullOrEmpty(metaInfoLinkText)) {
+                throw new ParsingException("Could not get metadata info link text.");
+            }
+            metaInfo.addUrlText(metaInfoLinkText);
+        }
+
+        if (clarificationRenderer.has("secondaryEndpoint") && clarificationRenderer
+                .has("secondarySource")) {
+            final String url = getUrlFromNavigationEndpoint(clarificationRenderer
+                    .getObject("secondaryEndpoint"));
+            // Ignore Google URLs, because those point to a Google search about "Covid-19"
+            if (url != null && !isGoogleURL(url)) {
+                try {
+                    metaInfo.addUrl(new URL(url));
+                    final String description = getTextFromObject(clarificationRenderer
+                            .getObject("secondarySource"));
+                    metaInfo.addUrlText(description == null ? url : description);
+                } catch (final MalformedURLException e) {
+                    throw new ParsingException("Could not get metadata info secondary URL", e);
+                }
+            }
+        }
+
+        return metaInfo;
+    }
+
+    private static void getEmergencyOneboxRenderer(
+            @Nonnull final JsonObject emergencyOneboxRenderer,
+            final Consumer<MetaInfo> addMetaInfo
+    ) throws ParsingException {
+        final List<JsonObject> supportRenderers = emergencyOneboxRenderer.values()
+                .stream()
+                .filter(o -> o instanceof JsonObject
+                        && ((JsonObject) o).has("singleActionEmergencySupportRenderer"))
+                .map(o -> ((JsonObject) o).getObject("singleActionEmergencySupportRenderer"))
+                .collect(Collectors.toList());
+
+        if (supportRenderers.isEmpty()) {
+            throw new ParsingException("Could not extract any meta info from emergency renderer");
+        }
+
+        for (final JsonObject r : supportRenderers) {
+            final MetaInfo metaInfo = new MetaInfo();
+
+            // usually an encouragement like "We are with you"
+            final String title = getTextFromObjectOrThrow(r.getObject("title"), "title");
+
+            // usually a phone number
+            final String action; // this variable is expected to start with "\n"
+            if (r.has("actionText")) {
+                action = "\n" + getTextFromObjectOrThrow(r.getObject("actionText"), "action");
+            } else if (r.has("contacts")) {
+                final JsonArray contacts = r.getArray("contacts");
+                final StringBuilder stringBuilder = new StringBuilder();
+                // Loop over contacts item from the first contact to the last one
+                for (int i = 0; i < contacts.size(); i++) {
+                    stringBuilder.append("\n");
+                    stringBuilder.append(getTextFromObjectOrThrow(contacts.getObject(i)
+                            .getObject("actionText"), "contacts.actionText"));
+                }
+                action = stringBuilder.toString();
+            } else {
+                action = "";
+            }
+
+            // usually details about the phone number
+            final String details = getTextFromObjectOrThrow(r.getObject("detailsText"), "details");
+
+            // usually the name of an association
+            final String urlText = getTextFromObjectOrThrow(r.getObject("navigationText"),
+                    "urlText");
+
+            metaInfo.setTitle(title);
+            metaInfo.setContent(new Description(details + action, Description.PLAIN_TEXT));
+            metaInfo.addUrlText(urlText);
+
+            // usually the webpage of the association
+            final String url = getUrlFromNavigationEndpoint(r.getObject("navigationEndpoint"));
+            if (url == null) {
+                throw new ParsingException("Could not extract emergency renderer url");
+            }
+
+            try {
+                metaInfo.addUrl(new URL(replaceHttpWithHttps(url)));
+            } catch (final MalformedURLException e) {
+                throw new ParsingException("Could not parse emergency renderer url", e);
+            }
+
+            addMetaInfo.accept(metaInfo);
+        }
+    }
+}

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java

@@ -32,11 +32,10 @@ import com.grack.nanojson.JsonObject;
 import com.grack.nanojson.JsonParser;
 import com.grack.nanojson.JsonParserException;
 import com.grack.nanojson.JsonWriter;
-import org.jsoup.nodes.Entities;
 
+import org.jsoup.nodes.Entities;
 import org.schabi.newpipe.extractor.Image;
 import org.schabi.newpipe.extractor.Image.ResolutionLevel;
-import org.schabi.newpipe.extractor.MetaInfo;
 import org.schabi.newpipe.extractor.downloader.Response;
 import org.schabi.newpipe.extractor.exceptions.AccountTerminatedException;
 import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException;
@@ -47,14 +46,12 @@ import org.schabi.newpipe.extractor.localization.ContentCountry;
 import org.schabi.newpipe.extractor.localization.Localization;
 import org.schabi.newpipe.extractor.playlist.PlaylistInfo;
 import org.schabi.newpipe.extractor.stream.AudioTrackType;
-import org.schabi.newpipe.extractor.stream.Description;
 import org.schabi.newpipe.extractor.utils.JsonUtils;
 import org.schabi.newpipe.extractor.utils.Parser;
 import org.schabi.newpipe.extractor.utils.RandomStringFromAlphabetGenerator;
 import org.schabi.newpipe.extractor.utils.Utils;
 
 import java.io.IOException;
-import java.io.UnsupportedEncodingException;
 import java.net.MalformedURLException;
 import java.net.URL;
 import java.nio.charset.StandardCharsets;
@@ -62,12 +59,10 @@ import java.time.LocalDate;
 import java.time.OffsetDateTime;
 import java.time.ZoneOffset;
 import java.time.format.DateTimeParseException;
-import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Locale;
 import java.util.Map;
-import java.util.Objects;
 import java.util.Optional;
 import java.util.Random;
 import java.util.Set;
@@ -104,17 +99,17 @@ public final class YoutubeParsingHelper {
      * sizes.
      *
      * <p>
-     * Sent in query parameters of the requests, <b>after</b> the API key.
+     * Sent in query parameters of the requests.
      * </p>
      **/
-    public static final String DISABLE_PRETTY_PRINT_PARAMETER = "&prettyPrint=false";
+    public static final String DISABLE_PRETTY_PRINT_PARAMETER = "prettyPrint=false";
 
     /**
      * A parameter sent by official clients named {@code contentPlaybackNonce}.
      *
      * <p>
-     * It is sent by official clients on videoplayback requests, and by all clients (except the
-     * {@code WEB} one to the player requests.
+     * It is sent by official clients on videoplayback requests and InnerTube player requests in
+     * most cases.
      * </p>
      *
      * <p>
@@ -148,17 +143,16 @@ public final class YoutubeParsingHelper {
      */
     public static final String RACY_CHECK_OK = "racyCheckOk";
 
+    /**
+     * The hardcoded client ID used for InnerTube requests with the {@code WEB} client.
+     */
+    private static final String WEB_CLIENT_ID = "1";
+
     /**
      * The client version for InnerTube requests with the {@code WEB} client, used as the last
      * fallback if the extraction of the real one failed.
      */
-    private static final String HARDCODED_CLIENT_VERSION = "2.20231208.01.00";
-
-    /**
-     * The InnerTube API key which should be used by YouTube's desktop website, used as a fallback
-     * if the extraction of the real one failed.
-     */
-    private static final String HARDCODED_KEY = "AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8";
+    private static final String HARDCODED_CLIENT_VERSION = "2.20240718.01.00";
 
     /**
      * The hardcoded client version of the Android app used for InnerTube requests with this
@@ -169,17 +163,10 @@ public final class YoutubeParsingHelper {
      * such as <a href="https://www.apkmirror.com/apk/google-inc/youtube/">APKMirror</a>.
      * </p>
      */
-    private static final String ANDROID_YOUTUBE_CLIENT_VERSION = "18.48.37";
+    private static final String ANDROID_YOUTUBE_CLIENT_VERSION = "19.28.35";
 
     /**
-     * The InnerTube API key used by the {@code ANDROID} client. Found with the help of
-     * reverse-engineering app network requests.
-     */
-    private static final String ANDROID_YOUTUBE_KEY = "AIzaSyA8eiZmM1FaDVjRy-df2KTyQ_vz_yYM39w";
-
-    /**
-     * The hardcoded client version of the iOS app used for InnerTube requests with this
-     * client.
+     * The hardcoded client version of the iOS app used for InnerTube requests with this client.
      *
      * <p>
      * It can be extracted by getting the latest release version of the app on
@@ -187,42 +174,39 @@ public final class YoutubeParsingHelper {
      * Store page of the YouTube app</a>, in the {@code What’s New} section.
      * </p>
      */
-    private static final String IOS_YOUTUBE_CLIENT_VERSION = "18.48.3";
-
-    /**
-     * The InnerTube API key used by the {@code iOS} client. Found with the help of
-     * reverse-engineering app network requests.
-     */
-    private static final String IOS_YOUTUBE_KEY = "AIzaSyB-63vPrdThhKuerbB2N_l7Kwwcxj6yUAc";
+    private static final String IOS_YOUTUBE_CLIENT_VERSION = "19.28.1";
 
     /**
      * The hardcoded client version used for InnerTube requests with the TV HTML5 embed client.
      */
     private static final String TVHTML5_SIMPLY_EMBED_CLIENT_VERSION = "2.0";
 
+    /**
+     * The hardcoded client ID used for InnerTube requests with the YouTube Music desktop client.
+     */
+    private static final String YOUTUBE_MUSIC_CLIENT_ID = "67";
+
+    /**
+     * The hardcoded client version used for InnerTube requests with the YouTube Music desktop
+     * client.
+     */
+    private static final String HARDCODED_YOUTUBE_MUSIC_CLIENT_VERSION = "1.20240715.01.00";
+
     private static String clientVersion;
-    private static String key;
 
-    private static final String[] HARDCODED_YOUTUBE_MUSIC_KEY =
-            {"AIzaSyC9XL3ZjWddXya6X74dJoCTL-WEYFDNX30", "67", "1.20231204.01.00"};
-    private static String[] youtubeMusicKey;
+    private static String youtubeMusicClientVersion;
 
-    private static boolean keyAndVersionExtracted = false;
+    private static boolean clientVersionExtracted = false;
     @SuppressWarnings("OptionalUsedAsFieldOrParameterType")
-    private static Optional<Boolean> hardcodedClientVersionAndKeyValid = Optional.empty();
+    private static Optional<Boolean> hardcodedClientVersionValid = Optional.empty();
 
     private static final String[] INNERTUBE_CONTEXT_CLIENT_VERSION_REGEXES =
             {"INNERTUBE_CONTEXT_CLIENT_VERSION\":\"([0-9\\.]+?)\"",
                     "innertube_context_client_version\":\"([0-9\\.]+?)\"",
                     "client.version=([0-9\\.]+)"};
-    private static final String[] INNERTUBE_API_KEY_REGEXES =
-            {"INNERTUBE_API_KEY\":\"([0-9a-zA-Z_-]+?)\"",
-                    "innertubeApiKey\":\"([0-9a-zA-Z_-]+?)\""};
     private static final String[] INITIAL_DATA_REGEXES =
             {"window\\[\"ytInitialData\"\\]\\s*=\\s*(\\{.*?\\});",
                     "var\\s*ytInitialData\\s*=\\s*(\\{.*?\\});"};
-    private static final String INNERTUBE_CLIENT_NAME_REGEX =
-            "INNERTUBE_CONTEXT_CLIENT_NAME\":([0-9]+?),";
 
     private static final String CONTENT_PLAYBACK_NONCE_ALPHABET =
             "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_";
@@ -235,7 +219,31 @@ public final class YoutubeParsingHelper {
      * information.
      * </p>
      */
-    private static final String IOS_DEVICE_MODEL = "iPhone15,4";
+    private static final String IOS_DEVICE_MODEL = "iPhone16,2";
+
+    /**
+     * Spoofing an iPhone 15 Pro Max running iOS 17.5.1 with the hardcoded version of the iOS app.
+     * To be used for the {@code "osVersion"} field in JSON POST requests.
+     * <p>
+     * The value of this field seems to use the following structure:
+     * "iOS major version.minor version.patch version.build version", where
+     * "patch version" is equal to 0 if it isn't set
+     * The build version corresponding to the iOS version used can be found on
+     * <a href="https://theapplewiki.com/wiki/Firmware/iPhone/17.x#iPhone_15_Pro_Max">
+     *     https://theapplewiki.com/wiki/Firmware/iPhone/17.x#iPhone_15_Pro_Max</a>
+     * </p>
+     *
+     * @see #IOS_USER_AGENT_VERSION
+     */
+    private static final String IOS_OS_VERSION = "17.5.1.21F90";
+
+    /**
+     * Spoofing an iPhone 15 running iOS 17.5.1 with the hardcoded version of the iOS app. To be
+     * used in the user agent for requests.
+     *
+     * @see #IOS_OS_VERSION
+     */
+    private static final String IOS_USER_AGENT_VERSION = "17_5_1";
 
     private static Random numberGenerator = new Random();
 
@@ -262,7 +270,7 @@ public final class YoutubeParsingHelper {
 
     private static boolean consentAccepted = false;
 
-    private static boolean isGoogleURL(final String url) {
+    public static boolean isGoogleURL(final String url) {
         final String cachedUrl = extractCachedUrlIfNeeded(url);
         try {
             final URL u = new URL(cachedUrl);
@@ -522,10 +530,10 @@ public final class YoutubeParsingHelper {
         }
     }
 
-    public static boolean areHardcodedClientVersionAndKeyValid()
+    public static boolean isHardcodedClientVersionValid()
             throws IOException, ExtractionException {
-        if (hardcodedClientVersionAndKeyValid.isPresent()) {
-            return hardcodedClientVersionAndKeyValid.get();
+        if (hardcodedClientVersionValid.isPresent()) {
+            return hardcodedClientVersionValid.get();
         }
         // @formatter:off
         final byte[] body = JsonWriter.string()
@@ -554,25 +562,25 @@ public final class YoutubeParsingHelper {
             .end().done().getBytes(StandardCharsets.UTF_8);
         // @formatter:on
 
-        final var headers = getClientHeaders("1", HARDCODED_CLIENT_VERSION);
+        final var headers = getClientHeaders(WEB_CLIENT_ID, HARDCODED_CLIENT_VERSION);
 
         // This endpoint is fetched by the YouTube website to get the items of its main menu and is
         // pretty lightweight (around 30kB)
         final Response response = getDownloader().postWithContentTypeJson(
-                YOUTUBEI_V1_URL + "guide?key=" + HARDCODED_KEY + DISABLE_PRETTY_PRINT_PARAMETER,
+                YOUTUBEI_V1_URL + "guide?" + DISABLE_PRETTY_PRINT_PARAMETER,
                 headers, body);
         final String responseBody = response.responseBody();
         final int responseCode = response.responseCode();
 
-        hardcodedClientVersionAndKeyValid = Optional.of(responseBody.length() > 5000
+        hardcodedClientVersionValid = Optional.of(responseBody.length() > 5000
                 && responseCode == 200); // Ensure to have a valid response
-        return hardcodedClientVersionAndKeyValid.get();
+        return hardcodedClientVersionValid.get();
     }
 
 
-    private static void extractClientVersionAndKeyFromSwJs()
+    private static void extractClientVersionFromSwJs()
             throws IOException, ExtractionException {
-        if (keyAndVersionExtracted) {
+        if (clientVersionExtracted) {
             return;
         }
         final String url = "https://www.youtube.com/sw.js";
@@ -581,18 +589,17 @@ public final class YoutubeParsingHelper {
         try {
             clientVersion = getStringResultFromRegexArray(response,
                     INNERTUBE_CONTEXT_CLIENT_VERSION_REGEXES, 1);
-            key = getStringResultFromRegexArray(response, INNERTUBE_API_KEY_REGEXES, 1);
         } catch (final Parser.RegexException e) {
             throw new ParsingException("Could not extract YouTube WEB InnerTube client version "
-                    + "and API key from sw.js", e);
+                    + "from sw.js", e);
         }
-        keyAndVersionExtracted = true;
+        clientVersionExtracted = true;
     }
 
-    private static void extractClientVersionAndKeyFromHtmlSearchResultsPage()
+    private static void extractClientVersionFromHtmlSearchResultsPage()
             throws IOException, ExtractionException {
-        // Don't extract the client version and the InnerTube key if it has been already extracted
-        if (keyAndVersionExtracted) {
+        // Don't extract the InnerTube client version if it has been already extracted
+        if (clientVersionExtracted) {
             return;
         }
 
@@ -626,18 +633,6 @@ public final class YoutubeParsingHelper {
                     serviceTrackingParamsStream, "ECATCHER", "client.version");
         }
 
-        try {
-            key = getStringResultFromRegexArray(html, INNERTUBE_API_KEY_REGEXES, 1);
-        } catch (final Parser.RegexException ignored) {
-        }
-
-        if (isNullOrEmpty(key)) {
-            throw new ParsingException(
-                    // CHECKSTYLE:OFF
-                    "Could not extract YouTube WEB InnerTube API key from HTML search results page");
-                    // CHECKSTYLE:ON
-        }
-
         if (clientVersion == null) {
             throw new ParsingException(
                     // CHECKSTYLE:OFF
@@ -645,7 +640,7 @@ public final class YoutubeParsingHelper {
                     // CHECKSTYLE:ON
         }
 
-        keyAndVersionExtracted = true;
+        clientVersionExtracted = true;
     }
 
     @Nullable
@@ -680,17 +675,17 @@ public final class YoutubeParsingHelper {
         // JavaScript service worker, then from HTML search results page as a fallback, to prevent
         // fingerprinting based on the client version used
         try {
-            extractClientVersionAndKeyFromSwJs();
+            extractClientVersionFromSwJs();
         } catch (final Exception e) {
-            extractClientVersionAndKeyFromHtmlSearchResultsPage();
+            extractClientVersionFromHtmlSearchResultsPage();
         }
 
-        if (keyAndVersionExtracted) {
+        if (clientVersionExtracted) {
             return clientVersion;
         }
 
         // Fallback to the hardcoded one if it is valid
-        if (areHardcodedClientVersionAndKeyValid()) {
+        if (isHardcodedClientVersionValid()) {
             clientVersion = HARDCODED_CLIENT_VERSION;
             return clientVersion;
         }
@@ -698,39 +693,6 @@ public final class YoutubeParsingHelper {
         throw new ExtractionException("Could not get YouTube WEB client version");
     }
 
-    /**
-     * Get the internal API key used by YouTube website on InnerTube requests.
-     */
-    public static String getKey() throws IOException, ExtractionException {
-        if (!isNullOrEmpty(key)) {
-            return key;
-        }
-
-        // Always extract the key used by the website, by trying first to extract it from the
-        // JavaScript service worker, then from HTML search results page as a fallback, to prevent
-        // fingerprinting based on the key and/or invalid key issues
-        try {
-            extractClientVersionAndKeyFromSwJs();
-        } catch (final Exception e) {
-            extractClientVersionAndKeyFromHtmlSearchResultsPage();
-        }
-
-        if (keyAndVersionExtracted) {
-            return key;
-        }
-
-        // Fallback to the hardcoded one if it's valid
-        if (areHardcodedClientVersionAndKeyValid()) {
-            key = HARDCODED_KEY;
-            return key;
-        }
-
-        // The ANDROID API key is also valid with the WEB client so return it if we couldn't
-        // extract the WEB API key. This can be used as a way to fingerprint the extractor in this
-        // case
-        return ANDROID_YOUTUBE_KEY;
-    }
-
     /**
      * <p>
      * <b>Only used in tests.</b>
@@ -746,10 +708,9 @@ public final class YoutubeParsingHelper {
      * tests with mocks will fail, because the mock is missing.
      * </p>
      */
-    public static void resetClientVersionAndKey() {
+    public static void resetClientVersion() {
         clientVersion = null;
-        key = null;
-        keyAndVersionExtracted = false;
+        clientVersionExtracted = false;
     }
 
     /**
@@ -761,11 +722,11 @@ public final class YoutubeParsingHelper {
         numberGenerator = random;
     }
 
-    public static boolean isHardcodedYoutubeMusicKeyValid() throws IOException,
+    public static boolean isHardcodedYoutubeMusicClientVersionValid() throws IOException,
             ReCaptchaException {
         final String url =
-                "https://music.youtube.com/youtubei/v1/music/get_search_suggestions?key="
-                        + HARDCODED_YOUTUBE_MUSIC_KEY[0] + DISABLE_PRETTY_PRINT_PARAMETER;
+                "https://music.youtube.com/youtubei/v1/music/get_search_suggestions?"
+                        + DISABLE_PRETTY_PRINT_PARAMETER;
 
         // @formatter:off
         final byte[] json = JsonWriter.string()
@@ -773,7 +734,7 @@ public final class YoutubeParsingHelper {
                 .object("context")
                     .object("client")
                         .value("clientName", "WEB_REMIX")
-                        .value("clientVersion", HARDCODED_YOUTUBE_MUSIC_KEY[2])
+                        .value("clientVersion", HARDCODED_YOUTUBE_MUSIC_CLIENT_VERSION)
                         .value("hl", "en-GB")
                         .value("gl", "GB")
                         .value("platform", "DESKTOP")
@@ -795,48 +756,40 @@ public final class YoutubeParsingHelper {
         // @formatter:on
 
         final var headers = new HashMap<>(getOriginReferrerHeaders(YOUTUBE_MUSIC_URL));
-        headers.putAll(getClientHeaders(HARDCODED_YOUTUBE_MUSIC_KEY[1],
-                HARDCODED_YOUTUBE_MUSIC_KEY[2]));
+        headers.putAll(getClientHeaders(YOUTUBE_MUSIC_CLIENT_ID,
+                HARDCODED_YOUTUBE_MUSIC_CLIENT_VERSION));
 
         final Response response = getDownloader().postWithContentTypeJson(url, headers, json);
         // Ensure to have a valid response
         return response.responseBody().length() > 500 && response.responseCode() == 200;
     }
 
-    public static String[] getYoutubeMusicKey()
+    public static String getYoutubeMusicClientVersion()
             throws IOException, ReCaptchaException, Parser.RegexException {
-        if (youtubeMusicKey != null && youtubeMusicKey.length == 3) {
-            return youtubeMusicKey;
+        if (!isNullOrEmpty(youtubeMusicClientVersion)) {
+            return youtubeMusicClientVersion;
         }
-        if (isHardcodedYoutubeMusicKeyValid()) {
-            youtubeMusicKey = HARDCODED_YOUTUBE_MUSIC_KEY;
-            return youtubeMusicKey;
+        if (isHardcodedYoutubeMusicClientVersionValid()) {
+            youtubeMusicClientVersion = HARDCODED_YOUTUBE_MUSIC_CLIENT_VERSION;
+            return youtubeMusicClientVersion;
         }
 
-        String musicClientVersion;
-        String musicKey;
-        String musicClientName;
-
         try {
             final String url = "https://music.youtube.com/sw.js";
             final var headers = getOriginReferrerHeaders(YOUTUBE_MUSIC_URL);
             final String response = getDownloader().get(url, headers).responseBody();
-            musicClientVersion = getStringResultFromRegexArray(response,
+
+            youtubeMusicClientVersion = getStringResultFromRegexArray(response,
                     INNERTUBE_CONTEXT_CLIENT_VERSION_REGEXES, 1);
-            musicKey = getStringResultFromRegexArray(response, INNERTUBE_API_KEY_REGEXES, 1);
-            musicClientName = Parser.matchGroup1(INNERTUBE_CLIENT_NAME_REGEX, response);
         } catch (final Exception e) {
             final String url = "https://music.youtube.com/?ucbcb=1";
             final String html = getDownloader().get(url, getCookieHeader()).responseBody();
 
-            musicKey = getStringResultFromRegexArray(html, INNERTUBE_API_KEY_REGEXES, 1);
-            musicClientVersion = getStringResultFromRegexArray(html,
+            youtubeMusicClientVersion = getStringResultFromRegexArray(html,
                     INNERTUBE_CONTEXT_CLIENT_VERSION_REGEXES, 1);
-            musicClientName = Parser.matchGroup1(INNERTUBE_CLIENT_NAME_REGEX, html);
         }
 
-        youtubeMusicKey = new String[] {musicKey, musicClientName, musicClientVersion};
-        return youtubeMusicKey;
+        return youtubeMusicClientVersion;
     }
 
     @Nullable
@@ -856,11 +809,7 @@ public final class YoutubeParsingHelper {
                 final String[] params = internUrl.split("&");
                 for (final String param : params) {
                     if (param.split("=")[0].equals("q")) {
-                        try {
-                            return Utils.decodeUrlUtf8(param.split("=")[1]);
-                        } catch (final UnsupportedEncodingException e) {
-                            return null;
-                        }
+                        return Utils.decodeUrlUtf8(param.split("=")[1]);
                     }
                 }
             } else if (internUrl.startsWith("http")) {
@@ -876,9 +825,15 @@ public final class YoutubeParsingHelper {
             final String canonicalBaseUrl = browseEndpoint.getString("canonicalBaseUrl");
             final String browseId = browseEndpoint.getString("browseId");
 
-            // All channel ids are prefixed with UC
-            if (browseId != null && browseId.startsWith("UC")) {
-                return "https://www.youtube.com/channel/" + browseId;
+            if (browseId != null) {
+                if (browseId.startsWith("UC")) {
+                    // All channel IDs are prefixed with UC
+                    return "https://www.youtube.com/channel/" + browseId;
+                } else if (browseId.startsWith("VL")) {
+                    // All playlist IDs are prefixed with VL, which needs to be removed from the
+                    // playlist ID
+                    return "https://www.youtube.com/playlist?list=" + browseId.substring(2);
+                }
             }
 
             if (!isNullOrEmpty(canonicalBaseUrl)) {
@@ -938,12 +893,13 @@ public final class YoutubeParsingHelper {
             return textObject.getString("simpleText");
         }
 
-        if (textObject.getArray("runs").isEmpty()) {
+        final JsonArray runs = textObject.getArray("runs");
+        if (runs.isEmpty()) {
             return null;
         }
 
         final StringBuilder textBuilder = new StringBuilder();
-        for (final Object o : textObject.getArray("runs")) {
+        for (final Object o : runs) {
             final JsonObject run = (JsonObject) o;
             String text = run.getString("text");
 
@@ -1000,84 +956,14 @@ public final class YoutubeParsingHelper {
         return text;
     }
 
-    /**
-     * Parse a video description in the new "attributed" format, which contains the entire visible
-     * plaintext ({@code content}) and an array of {@code commandRuns}.
-     *
-     * <p>
-     * The {@code commandRuns} include the links and their position in the text.
-     * </p>
-     *
-     * @param attributedDescription the JSON object of the attributed description
-     * @return the parsed description, in HTML format, as a string
-     */
-    @Nullable
-    public static String getAttributedDescription(
-            @Nullable final JsonObject attributedDescription) {
-        if (isNullOrEmpty(attributedDescription)) {
-            return null;
+    @Nonnull
+    public static String getTextFromObjectOrThrow(final JsonObject textObject, final String error)
+            throws ParsingException {
+        final String result = getTextFromObject(textObject);
+        if (result == null) {
+            throw new ParsingException("Could not extract text: " + error);
         }
-
-        final String content = attributedDescription.getString("content");
-        if (content == null) {
-            return null;
-        }
-
-        final JsonArray commandRuns = attributedDescription.getArray("commandRuns");
-
-        final StringBuilder textBuilder = new StringBuilder();
-        int textStart = 0;
-
-        for (final Object commandRun : commandRuns) {
-            if (!(commandRun instanceof JsonObject)) {
-                continue;
-            }
-
-            final JsonObject run = ((JsonObject) commandRun);
-            final int startIndex = run.getInt("startIndex", -1);
-            final int length = run.getInt("length");
-            final JsonObject navigationEndpoint = run.getObject("onTap")
-                    .getObject("innertubeCommand");
-
-            if (startIndex < 0 || length < 1 || navigationEndpoint == null) {
-                continue;
-            }
-
-            final String url = getUrlFromNavigationEndpoint(navigationEndpoint);
-
-            if (url == null) {
-                continue;
-            }
-
-            // Append text before the link
-            if (startIndex > textStart) {
-                textBuilder.append(content, textStart, startIndex);
-            }
-
-            // Trim and append link text
-            // Channel/Video format: 3xu00a0, (/ •), u00a0, <Name>, 2xu00a0
-            final String linkText = content.substring(startIndex, startIndex + length)
-                    .replace('\u00a0', ' ')
-                    .trim()
-                    .replaceFirst("^[/•] *", "");
-
-            textBuilder.append("<a href=\"")
-                    .append(Entities.escape(url))
-                    .append("\">")
-                    .append(Entities.escape(linkText))
-                    .append("</a>");
-
-            textStart = startIndex + length;
-        }
-
-        // Append the remaining text
-        if (textStart < content.length()) {
-            textBuilder.append(content.substring(textStart));
-        }
-
-        return textBuilder.toString()
-                .replaceAll("\\n", "<br>")
-                .replaceAll(" {2}", " &nbsp;");
+        return result;
     }
 
     @Nullable
@@ -1091,11 +977,12 @@ public final class YoutubeParsingHelper {
             return null;
         }
 
-        if (textObject.getArray("runs").isEmpty()) {
+        final JsonArray runs = textObject.getArray("runs");
+        if (runs.isEmpty()) {
             return null;
         }
 
-        for (final Object textPart : textObject.getArray("runs")) {
+        for (final Object textPart : runs) {
             final String url = getUrlFromNavigationEndpoint(((JsonObject) textPart)
                     .getObject("navigationEndpoint"));
             if (!isNullOrEmpty(url)) {
@@ -1224,8 +1111,8 @@ public final class YoutubeParsingHelper {
         final var headers = getYouTubeHeaders();
 
         return JsonUtils.toJsonObject(getValidJsonResponseBody(
-                getDownloader().postWithContentTypeJson(YOUTUBEI_V1_URL + endpoint + "?key="
-                        + getKey() + DISABLE_PRETTY_PRINT_PARAMETER, headers, body, localization)));
+                getDownloader().postWithContentTypeJson(YOUTUBEI_V1_URL + endpoint + "?"
+                        + DISABLE_PRETTY_PRINT_PARAMETER, headers, body, localization)));
     }
 
     public static JsonObject getJsonAndroidPostResponse(
@@ -1234,7 +1121,7 @@ public final class YoutubeParsingHelper {
             @Nonnull final Localization localization,
             @Nullable final String endPartOfUrlRequest) throws IOException, ExtractionException {
         return getMobilePostResponse(endpoint, body, localization,
-                getAndroidUserAgent(localization), ANDROID_YOUTUBE_KEY, endPartOfUrlRequest);
+                getAndroidUserAgent(localization), endPartOfUrlRequest);
     }
 
     public static JsonObject getJsonIosPostResponse(
@@ -1243,7 +1130,7 @@ public final class YoutubeParsingHelper {
             @Nonnull final Localization localization,
             @Nullable final String endPartOfUrlRequest) throws IOException, ExtractionException {
         return getMobilePostResponse(endpoint, body, localization, getIosUserAgent(localization),
-                IOS_YOUTUBE_KEY, endPartOfUrlRequest);
+                endPartOfUrlRequest);
     }
 
     private static JsonObject getMobilePostResponse(
@@ -1251,12 +1138,11 @@ public final class YoutubeParsingHelper {
             final byte[] body,
             @Nonnull final Localization localization,
             @Nonnull final String userAgent,
-            @Nonnull final String innerTubeApiKey,
             @Nullable final String endPartOfUrlRequest) throws IOException, ExtractionException {
         final var headers = Map.of("User-Agent", List.of(userAgent),
                 "X-Goog-Api-Format-Version", List.of("2"));
 
-        final String baseEndpointUrl = YOUTUBEI_V1_GAPIS_URL + endpoint + "?key=" + innerTubeApiKey
+        final String baseEndpointUrl = YOUTUBEI_V1_GAPIS_URL + endpoint + "?"
                 + DISABLE_PRETTY_PRINT_PARAMETER;
 
         return JsonUtils.toJsonObject(getValidJsonResponseBody(
@@ -1369,14 +1255,7 @@ public final class YoutubeParsingHelper {
                         .value("deviceModel", IOS_DEVICE_MODEL)
                         .value("platform", "MOBILE")
                         .value("osName", "iOS")
-                        /*
-                        The value of this field seems to use the following structure:
-                        "iOS major version.minor version.patch version.build version", where
-                        "patch version" is equal to 0 if it isn't set
-                        The build version corresponding to the iOS version used can be found on
-                        https://theapplewiki.com/wiki/Firmware/iPhone/17.x#iPhone_15
-                         */
-                        .value("osVersion", "17.1.2.21B101")
+                        .value("osVersion", IOS_OS_VERSION)
                         .value("hl", localization.getLocalizationCode())
                         .value("gl", contentCountry.getCountryCode())
                         .value("utcOffsetMinutes", 0)
@@ -1430,31 +1309,49 @@ public final class YoutubeParsingHelper {
     }
 
     @Nonnull
-    public static byte[] createDesktopPlayerBody(
+    public static JsonObject getWebPlayerResponse(
+            @Nonnull final Localization localization,
+            @Nonnull final ContentCountry contentCountry,
+            @Nonnull final String videoId) throws IOException, ExtractionException {
+        final byte[] body = JsonWriter.string(
+                        prepareDesktopJsonBuilder(localization, contentCountry)
+                                .value(VIDEO_ID, videoId)
+                                .value(CONTENT_CHECK_OK, true)
+                                .value(RACY_CHECK_OK, true)
+                                .done())
+                .getBytes(StandardCharsets.UTF_8);
+        final String url = YOUTUBEI_V1_URL + "player" + "?" + DISABLE_PRETTY_PRINT_PARAMETER
+                + "&$fields=microformat,playabilityStatus,storyboards,videoDetails";
+
+        return JsonUtils.toJsonObject(getValidJsonResponseBody(
+                getDownloader().postWithContentTypeJson(
+                        url, getYouTubeHeaders(), body, localization)));
+    }
+
+    @Nonnull
+    public static byte[] createTvHtml5EmbedPlayerBody(
             @Nonnull final Localization localization,
             @Nonnull final ContentCountry contentCountry,
             @Nonnull final String videoId,
             @Nonnull final Integer sts,
-            final boolean isTvHtml5DesktopJsonBuilder,
-            @Nonnull final String contentPlaybackNonce) throws IOException, ExtractionException {
+            @Nonnull final String contentPlaybackNonce) {
         // @formatter:off
-        return JsonWriter.string((isTvHtml5DesktopJsonBuilder
-                        ? prepareTvHtml5EmbedJsonBuilder(localization, contentCountry, videoId)
-                        : prepareDesktopJsonBuilder(localization, contentCountry))
-                .object("playbackContext")
-                    .object("contentPlaybackContext")
-                        // Signature timestamp from the JavaScript base player is needed to get
-                        // working obfuscated URLs
-                        .value("signatureTimestamp", sts)
-                        .value("referer", "https://www.youtube.com/watch?v=" + videoId)
+        return JsonWriter.string(
+                prepareTvHtml5EmbedJsonBuilder(localization, contentCountry, videoId)
+                    .object("playbackContext")
+                        .object("contentPlaybackContext")
+                            // Signature timestamp from the JavaScript base player is needed to get
+                            // working obfuscated URLs
+                            .value("signatureTimestamp", sts)
+                            .value("referer", "https://www.youtube.com/watch?v=" + videoId)
+                        .end()
                     .end()
-                .end()
-                .value(CPN, contentPlaybackNonce)
-                .value(VIDEO_ID, videoId)
-                .value(CONTENT_CHECK_OK, true)
-                .value(RACY_CHECK_OK, true)
-                .done())
-                .getBytes(StandardCharsets.UTF_8);
+                    .value(CPN, contentPlaybackNonce)
+                    .value(VIDEO_ID, videoId)
+                    .value(CONTENT_CHECK_OK, true)
+                    .value(RACY_CHECK_OK, true)
+                    .done())
+                    .getBytes(StandardCharsets.UTF_8);
         // @formatter:on
     }
 
@@ -1495,9 +1392,10 @@ public final class YoutubeParsingHelper {
      */
     @Nonnull
     public static String getIosUserAgent(@Nullable final Localization localization) {
-        // Spoofing an iPhone 15 running iOS 17.1.2 with the hardcoded version of the iOS app
+        // Spoofing an iPhone 15 running iOS 17.5.1 with the hardcoded version of the iOS app
         return "com.google.ios.youtube/" + IOS_YOUTUBE_CLIENT_VERSION
-                + "(" + IOS_DEVICE_MODEL + "; U; CPU iOS 17_1_2 like Mac OS X; "
+                + "(" + IOS_DEVICE_MODEL + "; U; CPU iOS "
+                + IOS_USER_AGENT_VERSION + " like Mac OS X; "
                 + (localization != null ? localization : Localization.DEFAULT).getCountryCode()
                 + ")";
     }
@@ -1508,7 +1406,8 @@ public final class YoutubeParsingHelper {
     @Nonnull
     public static Map<String, List<String>> getYoutubeMusicHeaders() {
         final var headers = new HashMap<>(getOriginReferrerHeaders(YOUTUBE_MUSIC_URL));
-        headers.putAll(getClientHeaders(youtubeMusicKey[1], youtubeMusicKey[2]));
+        headers.putAll(getClientHeaders(YOUTUBE_MUSIC_CLIENT_ID,
+                youtubeMusicClientVersion));
         return headers;
     }
 
@@ -1530,7 +1429,7 @@ public final class YoutubeParsingHelper {
     public static Map<String, List<String>> getClientInfoHeaders()
             throws ExtractionException, IOException {
         final var headers = new HashMap<>(getOriginReferrerHeaders("https://www.youtube.com"));
-        headers.putAll(getClientHeaders("1", getClientVersion()));
+        headers.putAll(getClientHeaders(WEB_CLIENT_ID, getClientVersion()));
         return headers;
     }
 
@@ -1614,8 +1513,10 @@ public final class YoutubeParsingHelper {
             final String alertText = getTextFromObject(alertRenderer.getObject("text"));
             final String alertType = alertRenderer.getString("type", "");
             if (alertType.equalsIgnoreCase("ERROR")) {
-                if (alertText != null && alertText.contains("This account has been terminated")) {
-                    if (alertText.contains("violation") || alertText.contains("violating")
+                if (alertText != null
+                        && (alertText.contains("This account has been terminated")
+                        || alertText.contains("This channel was removed"))) {
+                    if (alertText.matches(".*violat(ed|ion|ing).*")
                             || alertText.contains("infringement")) {
                         // Possible error messages:
                         // "This account has been terminated for a violation of YouTube's Terms of
@@ -1637,6 +1538,7 @@ public final class YoutubeParsingHelper {
                         //     the user posted."
                         // "This account has been terminated because it is linked to an account that
                         //     received multiple third-party claims of copyright infringement."
+                        // "This channel was removed because it violated our Community Guidelines."
                         throw new AccountTerminatedException(alertText,
                                 AccountTerminatedException.Reason.VIOLATION);
                     } else {
@@ -1648,120 +1550,6 @@ public final class YoutubeParsingHelper {
         }
     }
 
-    @Nonnull
-    public static List<MetaInfo> getMetaInfo(@Nonnull final JsonArray contents)
-            throws ParsingException {
-        final List<MetaInfo> metaInfo = new ArrayList<>();
-        for (final Object content : contents) {
-            final JsonObject resultObject = (JsonObject) content;
-            if (resultObject.has("itemSectionRenderer")) {
-                for (final Object sectionContentObject
-                        : resultObject.getObject("itemSectionRenderer").getArray("contents")) {
-
-                    final JsonObject sectionContent = (JsonObject) sectionContentObject;
-                    if (sectionContent.has("infoPanelContentRenderer")) {
-                        metaInfo.add(getInfoPanelContent(sectionContent
-                                .getObject("infoPanelContentRenderer")));
-                    }
-                    if (sectionContent.has("clarificationRenderer")) {
-                        metaInfo.add(getClarificationRendererContent(sectionContent
-                                .getObject("clarificationRenderer")
-                        ));
-                    }
-
-                }
-            }
-        }
-        return metaInfo;
-    }
-
-    @Nonnull
-    private static MetaInfo getInfoPanelContent(@Nonnull final JsonObject infoPanelContentRenderer)
-            throws ParsingException {
-        final MetaInfo metaInfo = new MetaInfo();
-        final StringBuilder sb = new StringBuilder();
-        for (final Object paragraph : infoPanelContentRenderer.getArray("paragraphs")) {
-            if (sb.length() != 0) {
-                sb.append("<br>");
-            }
-            sb.append(YoutubeParsingHelper.getTextFromObject((JsonObject) paragraph));
-        }
-        metaInfo.setContent(new Description(sb.toString(), Description.HTML));
-        if (infoPanelContentRenderer.has("sourceEndpoint")) {
-            final String metaInfoLinkUrl = YoutubeParsingHelper.getUrlFromNavigationEndpoint(
-                    infoPanelContentRenderer.getObject("sourceEndpoint"));
-            try {
-                metaInfo.addUrl(new URL(Objects.requireNonNull(extractCachedUrlIfNeeded(
-                        metaInfoLinkUrl))));
-            } catch (final NullPointerException | MalformedURLException e) {
-                throw new ParsingException("Could not get metadata info URL", e);
-            }
-
-            final String metaInfoLinkText = YoutubeParsingHelper.getTextFromObject(
-                    infoPanelContentRenderer.getObject("inlineSource"));
-            if (isNullOrEmpty(metaInfoLinkText)) {
-                throw new ParsingException("Could not get metadata info link text.");
-            }
-            metaInfo.addUrlText(metaInfoLinkText);
-        }
-
-        return metaInfo;
-    }
-
-    @Nonnull
-    private static MetaInfo getClarificationRendererContent(
-            @Nonnull final JsonObject clarificationRenderer) throws ParsingException {
-        final MetaInfo metaInfo = new MetaInfo();
-
-        final String title = YoutubeParsingHelper.getTextFromObject(clarificationRenderer
-                .getObject("contentTitle"));
-        final String text = YoutubeParsingHelper.getTextFromObject(clarificationRenderer
-                .getObject("text"));
-        if (title == null || text == null) {
-            throw new ParsingException("Could not extract clarification renderer content");
-        }
-        metaInfo.setTitle(title);
-        metaInfo.setContent(new Description(text, Description.PLAIN_TEXT));
-
-        if (clarificationRenderer.has("actionButton")) {
-            final JsonObject actionButton = clarificationRenderer.getObject("actionButton")
-                    .getObject("buttonRenderer");
-            try {
-                final String url = YoutubeParsingHelper.getUrlFromNavigationEndpoint(actionButton
-                        .getObject("command"));
-                metaInfo.addUrl(new URL(Objects.requireNonNull(extractCachedUrlIfNeeded(url))));
-            } catch (final NullPointerException | MalformedURLException e) {
-                throw new ParsingException("Could not get metadata info URL", e);
-            }
-
-            final String metaInfoLinkText = YoutubeParsingHelper.getTextFromObject(
-                    actionButton.getObject("text"));
-            if (isNullOrEmpty(metaInfoLinkText)) {
-                throw new ParsingException("Could not get metadata info link text.");
-            }
-            metaInfo.addUrlText(metaInfoLinkText);
-        }
-
-        if (clarificationRenderer.has("secondaryEndpoint") && clarificationRenderer
-                .has("secondarySource")) {
-            final String url = getUrlFromNavigationEndpoint(clarificationRenderer
-                    .getObject("secondaryEndpoint"));
-            // Ignore Google URLs, because those point to a Google search about "Covid-19"
-            if (url != null && !isGoogleURL(url)) {
-                try {
-                    metaInfo.addUrl(new URL(url));
-                    final String description = getTextFromObject(clarificationRenderer
-                            .getObject("secondarySource"));
-                    metaInfo.addUrlText(description == null ? url : description);
-                } catch (final MalformedURLException e) {
-                    throw new ParsingException("Could not get metadata info secondary URL", e);
-                }
-            }
-        }
-
-        return metaInfo;
-    }
-
     /**
      * Sometimes, YouTube provides URLs which use Google's cache. They look like
      * {@code https://webcache.googleusercontent.com/search?q=cache:CACHED_URL}
@@ -1796,6 +1584,29 @@ public final class YoutubeParsingHelper {
         return false;
     }
 
+    public static boolean hasArtistOrVerifiedIconBadgeAttachment(
+            @Nonnull final JsonArray attachmentRuns) {
+        return attachmentRuns.stream()
+                .filter(JsonObject.class::isInstance)
+                .map(JsonObject.class::cast)
+                .anyMatch(attachmentRun -> attachmentRun.getObject("element")
+                        .getObject("type")
+                        .getObject("imageType")
+                        .getObject("image")
+                        .getArray("sources")
+                        .stream()
+                        .filter(JsonObject.class::isInstance)
+                        .map(JsonObject.class::cast)
+                        .anyMatch(source -> {
+                            final String imageName = source.getObject("clientResource")
+                                    .getString("imageName");
+                            return "CHECK_CIRCLE_FILLED".equals(imageName)
+                                    || "AUDIO_BADGE".equals(imageName)
+                                    || "MUSIC_FILLED".equals(imageName);
+                        }));
+
+    }
+
     /**
      * Generate a content playback nonce (also called {@code cpn}), sent by YouTube clients in
      * playback requests (and also for some clients, in the player request body).
@@ -1932,9 +1743,12 @@ public final class YoutubeParsingHelper {
             case "original":
                 return AudioTrackType.ORIGINAL;
             case "dubbed":
+            case "dubbed-auto":
                 return AudioTrackType.DUBBED;
             case "descriptive":
                 return AudioTrackType.DESCRIPTIVE;
+            case "secondary":
+                return AudioTrackType.SECONDARY;
             default:
                 return null;
         }

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSignatureUtils.java

@@ -1,5 +1,7 @@
 package org.schabi.newpipe.extractor.services.youtube;
 
+import static org.schabi.newpipe.extractor.utils.Parser.matchGroup1MultiplePatterns;
+
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
 import org.schabi.newpipe.extractor.utils.JavaScript;
 import org.schabi.newpipe.extractor.utils.Parser;
@@ -20,13 +22,13 @@ final class YoutubeSignatureUtils {
      */
     static final String DEOBFUSCATION_FUNCTION_NAME = "deobfuscate";
 
-    private static final String[] FUNCTION_REGEXES = {
-            "\\bm=([a-zA-Z0-9$]{2,})\\(decodeURIComponent\\(h\\.s\\)\\)",
-            "\\bc&&\\(c=([a-zA-Z0-9$]{2,})\\(decodeURIComponent\\(c\\)\\)",
+    private static final Pattern[] FUNCTION_REGEXES = {
             // CHECKSTYLE:OFF
-            "(?:\\b|[^a-zA-Z0-9$])([a-zA-Z0-9$]{2,})\\s*=\\s*function\\(\\s*a\\s*\\)\\s*\\{\\s*a\\s*=\\s*a\\.split\\(\\s*\"\"\\s*\\)",
+            Pattern.compile("\\bm=([a-zA-Z0-9$]{2,})\\(decodeURIComponent\\(h\\.s\\)\\)"),
+            Pattern.compile("\\bc&&\\(c=([a-zA-Z0-9$]{2,})\\(decodeURIComponent\\(c\\)\\)"),
+            Pattern.compile("(?:\\b|[^a-zA-Z0-9$])([a-zA-Z0-9$]{2,})\\s*=\\s*function\\(\\s*a\\s*\\)\\s*\\{\\s*a\\s*=\\s*a\\.split\\(\\s*\"\"\\s*\\)"),
+            Pattern.compile("([\\w$]+)\\s*=\\s*function\\((\\w+)\\)\\{\\s*\\2=\\s*\\2\\.split\\(\"\"\\)\\s*;")
             // CHECKSTYLE:ON
-            "([\\w$]+)\\s*=\\s*function\\((\\w+)\\)\\{\\s*\\2=\\s*\\2\\.split\\(\"\"\\)\\s*;"
     };
 
     private static final String STS_REGEX = "signatureTimestamp[=:](\\d+)";
@@ -104,19 +106,12 @@ final class YoutubeSignatureUtils {
     @Nonnull
     private static String getDeobfuscationFunctionName(@Nonnull final String javaScriptPlayerCode)
             throws ParsingException {
-        Parser.RegexException exception = null;
-        for (final String regex : FUNCTION_REGEXES) {
-            try {
-                return Parser.matchGroup1(regex, javaScriptPlayerCode);
-            } catch (final Parser.RegexException e) {
-                if (exception == null) {
-                    exception = e;
-                }
-            }
+        try {
+            return matchGroup1MultiplePatterns(FUNCTION_REGEXES, javaScriptPlayerCode);
+        } catch (final Parser.RegexException e) {
+            throw new ParsingException(
+                    "Could not find deobfuscation function with any of the known patterns", e);
         }
-
-        throw new ParsingException(
-                "Could not find deobfuscation function with any of the known patterns", exception);
     }
 
     @Nonnull

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeThrottlingParameterUtils.java

@@ -1,5 +1,7 @@
 package org.schabi.newpipe.extractor.services.youtube;
 
+import static org.schabi.newpipe.extractor.utils.Parser.matchMultiplePatterns;
+
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
 import org.schabi.newpipe.extractor.utils.JavaScript;
 import org.schabi.newpipe.extractor.utils.Parser;
@@ -18,10 +20,86 @@ final class YoutubeThrottlingParameterUtils {
 
     private static final Pattern THROTTLING_PARAM_PATTERN = Pattern.compile("[&?]n=([^&]+)");
 
-    private static final Pattern DEOBFUSCATION_FUNCTION_NAME_PATTERN = Pattern.compile(
-            // CHECKSTYLE:OFF
-            "\\.get\\(\"n\"\\)\\)&&\\([a-zA-Z0-9$_]=([a-zA-Z0-9$_]+)(?:\\[(\\d+)])?\\([a-zA-Z0-9$_]\\)");
-            // CHECKSTYLE:ON
+    private static final String SINGLE_CHAR_VARIABLE_REGEX = "[a-zA-Z0-9$_]";
+
+    private static final String MULTIPLE_CHARS_REGEX = SINGLE_CHAR_VARIABLE_REGEX + "+";
+
+    private static final String ARRAY_ACCESS_REGEX = "\\[(\\d+)]";
+
+    // CHECKSTYLE:OFF
+    private static final Pattern[] DEOBFUSCATION_FUNCTION_NAME_REGEXES = {
+
+            /*
+             * The first regex matches the following text, where we want Wma and the array index
+             * accessed:
+             *
+             * a.D&&(b="nn"[+a.D],WL(a),c=a.j[b]||null)&&(c=SDa[0](c),a.set(b,c),SDa.length||Wma("")
+             */
+            Pattern.compile(SINGLE_CHAR_VARIABLE_REGEX + "=\"nn\"\\[\\+" + MULTIPLE_CHARS_REGEX
+                    + "\\." + MULTIPLE_CHARS_REGEX + "]," + MULTIPLE_CHARS_REGEX + "\\("
+                    + MULTIPLE_CHARS_REGEX + "\\)," + MULTIPLE_CHARS_REGEX + "="
+                    + MULTIPLE_CHARS_REGEX + "\\." + MULTIPLE_CHARS_REGEX + "\\["
+                    + MULTIPLE_CHARS_REGEX + "]\\|\\|null\\).+\\|\\|(" + MULTIPLE_CHARS_REGEX
+                    + ")\\(\"\"\\)"),
+
+            /*
+             * The second regex matches the following text, where we want SDa and the array index
+             * accessed:
+             *
+             * a.D&&(b="nn"[+a.D],WL(a),c=a.j[b]||null)&&(c=SDa[0](c),a.set(b,c),SDa.length||Wma("")
+             */
+            Pattern.compile(SINGLE_CHAR_VARIABLE_REGEX + "=\"nn\"\\[\\+" + MULTIPLE_CHARS_REGEX
+                    + "\\." + MULTIPLE_CHARS_REGEX + "]," + MULTIPLE_CHARS_REGEX + "\\("
+                    + MULTIPLE_CHARS_REGEX + "\\)," + MULTIPLE_CHARS_REGEX + "="
+                    + MULTIPLE_CHARS_REGEX + "\\." + MULTIPLE_CHARS_REGEX + "\\["
+                    + MULTIPLE_CHARS_REGEX + "]\\|\\|null\\)&&\\(" + MULTIPLE_CHARS_REGEX + "=("
+                    + MULTIPLE_CHARS_REGEX + ")" + ARRAY_ACCESS_REGEX),
+
+            /*
+             * The third regex matches the following text, where we want rma:
+             *
+             * a.D&&(b="nn"[+a.D],c=a.get(b))&&(c=rDa[0](c),a.set(b,c),rDa.length||rma("")
+             */
+            Pattern.compile(SINGLE_CHAR_VARIABLE_REGEX + "=\"nn\"\\[\\+" + MULTIPLE_CHARS_REGEX
+                    + "\\." + MULTIPLE_CHARS_REGEX + "]," + MULTIPLE_CHARS_REGEX + "="
+                    + MULTIPLE_CHARS_REGEX + "\\.get\\(" + MULTIPLE_CHARS_REGEX + "\\)\\).+\\|\\|("
+                    + MULTIPLE_CHARS_REGEX + ")\\(\"\"\\)"),
+
+            /*
+             * The fourth regex matches the following text, where we want rDa and the array index
+             * accessed:
+             *
+             * a.D&&(b="nn"[+a.D],c=a.get(b))&&(c=rDa[0](c),a.set(b,c),rDa.length||rma("")
+             */
+            Pattern.compile(SINGLE_CHAR_VARIABLE_REGEX + "=\"nn\"\\[\\+" + MULTIPLE_CHARS_REGEX
+                    + "\\." + MULTIPLE_CHARS_REGEX + "]," + MULTIPLE_CHARS_REGEX + "="
+                    + MULTIPLE_CHARS_REGEX + "\\.get\\(" + MULTIPLE_CHARS_REGEX + "\\)\\)&&\\("
+                    + MULTIPLE_CHARS_REGEX + "=(" + MULTIPLE_CHARS_REGEX + ")\\[(\\d+)]"),
+
+            /*
+             * The fifth regex matches the following text, where we want BDa and the array index
+             * accessed:
+             *
+             * (b=String.fromCharCode(110),c=a.get(b))&&(c=BDa[0](c)
+             */
+            Pattern.compile("\\(" + SINGLE_CHAR_VARIABLE_REGEX + "=String\\.fromCharCode\\(110\\),"
+                    + SINGLE_CHAR_VARIABLE_REGEX + "=" + SINGLE_CHAR_VARIABLE_REGEX + "\\.get\\("
+                    + SINGLE_CHAR_VARIABLE_REGEX + "\\)\\)" + "&&\\(" + SINGLE_CHAR_VARIABLE_REGEX
+                    + "=(" + MULTIPLE_CHARS_REGEX + ")" + "(?:" + ARRAY_ACCESS_REGEX + ")?\\("
+                    + SINGLE_CHAR_VARIABLE_REGEX + "\\)"),
+
+            /*
+             * The sixth regex matches the following text, where we want Yva and the array index
+             * accessed:
+             *
+             * .get("n"))&&(b=Yva[0](b)
+             */
+            Pattern.compile("\\.get\\(\"n\"\\)\\)&&\\(" + SINGLE_CHAR_VARIABLE_REGEX
+                    + "=(" + MULTIPLE_CHARS_REGEX + ")(?:" + ARRAY_ACCESS_REGEX + ")?\\("
+                    + SINGLE_CHAR_VARIABLE_REGEX + "\\)")
+    };
+    // CHECKSTYLE:ON
+
 
     // Escape the curly end brace to allow compatibility with Android's regex engine
     // See https://stackoverflow.com/q/45074813
@@ -48,11 +126,13 @@ final class YoutubeThrottlingParameterUtils {
     @Nonnull
     static String getDeobfuscationFunctionName(@Nonnull final String javaScriptPlayerCode)
             throws ParsingException {
-        final Matcher matcher = DEOBFUSCATION_FUNCTION_NAME_PATTERN.matcher(javaScriptPlayerCode);
-        if (!matcher.find()) {
-            throw new ParsingException("Failed to find deobfuscation function name pattern \""
-                    + DEOBFUSCATION_FUNCTION_NAME_PATTERN
-                    + "\" in the base JavaScript player code");
+        final Matcher matcher;
+        try {
+            matcher = matchMultiplePatterns(DEOBFUSCATION_FUNCTION_NAME_REGEXES,
+                    javaScriptPlayerCode);
+        } catch (final Parser.RegexException e) {
+            throw new ParsingException("Could not find deobfuscation function with any of the "
+                    + "known patterns in the base JavaScript player code", e);
         }
 
         final String functionName = matcher.group(1);

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/dashmanifestcreators/YoutubeDashManifestCreatorsUtils.java

@@ -119,7 +119,7 @@ public final class YoutubeDashManifestCreatorsUtils {
     /**
      * Generate a {@link Document} with common manifest creator elements added to it.
      *
-     * <p>
+     * <br>
      * Those are:
      * <ul>
      *     <li>{@code MPD} (using {@link #generateDocumentAndMpdElement(long)});</li>
@@ -132,7 +132,7 @@ public final class YoutubeDashManifestCreatorsUtils {
      *     <li>and, for audio streams, {@code AudioChannelConfiguration} (using
      *     {@link #generateAudioChannelConfigurationElement(Document, ItagItem)}).</li>
      * </ul>
-     * </p>
+     *
      *
      * @param itagItem the {@link ItagItem} associated to the stream, which must not be null
      * @param streamDuration the duration of the stream, in milliseconds
@@ -325,6 +325,8 @@ public final class YoutubeDashManifestCreatorsUtils {
                 case DESCRIPTIVE:
                     return "description";
                 default:
+                    // Secondary track types do not seem to have a dedicated role in the DASH
+                    // specification, so use alternate for them
                     return "alternate";
             }
         }
@@ -494,7 +496,6 @@ public final class YoutubeDashManifestCreatorsUtils {
      * This method is only used when generating DASH manifests from OTF and post-live-DVR streams.
      * </p>
      *
-     * <p>
      * It will produce a {@code <SegmentTemplate>} element with the following attributes:
      * <ul>
      *     <li>{@code startNumber}, which takes the value {@code 0} for post-live-DVR streams and
@@ -505,7 +506,6 @@ public final class YoutubeDashManifestCreatorsUtils {
      *     <li>{@code initialization} (only for OTF streams), which is the base URL of the stream
      *     on which is appended {@link #SQ_0}.</li>
      * </ul>
-     * </p>
      *
      * <p>
      * The {@code <Representation>} element needs to be generated before this element with
@@ -578,8 +578,8 @@ public final class YoutubeDashManifestCreatorsUtils {
 
     /**
      * Get the "initialization" {@link Response response} of a stream.
-     *
-     * <p>This method fetches, for OTF streams and for post-live-DVR streams:
+     * <br>
+     * This method fetches, for OTF streams and for post-live-DVR streams:
      *     <ul>
      *         <li>the base URL of the stream, to which are appended {@link #SQ_0} and
      *         {@link #RN_0} parameters, with a {@code GET} request for streaming URLs from HTML5
@@ -588,7 +588,6 @@ public final class YoutubeDashManifestCreatorsUtils {
      *         <li>for streaming URLs from HTML5 clients, the {@link #ALR_YES} param is also added.
      *         </li>
      *     </ul>
-     * </p>
      *
      * @param baseStreamingUrl the base URL of the stream, which must not be null
      * @param itagItem         the {@link ItagItem} of stream, which must not be null

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/dashmanifestcreators/YoutubeOtfDashManifestCreator.java

@@ -53,38 +53,37 @@ public final class YoutubeOtfDashManifestCreator {
      * livestreams which have just been re-encoded as normal videos.
      * </p>
      *
-     * <p>This method needs:
-     *     <ul>
-     *         <li>the base URL of the stream (which, if you try to access to it, returns HTTP
-     *         status code 404 after redirects, and if the URL is valid);</li>
-     *         <li>an {@link ItagItem}, which needs to contain the following information:
-     *              <ul>
-     *                  <li>its type (see {@link ItagItem.ItagType}), to identify if the content is
-     *                  an audio or a video stream;</li>
-     *                  <li>its bitrate;</li>
-     *                  <li>its mime type;</li>
-     *                  <li>its codec(s);</li>
-     *                  <li>for an audio stream: its audio channels;</li>
-     *                  <li>for a video stream: its width and height.</li>
-     *              </ul>
-     *         </li>
-     *         <li>the duration of the video, which will be used if the duration could not be
-     *         parsed from the first sequence of the stream.</li>
-     *     </ul>
-     * </p>
+     * This method needs:
+     * <ul>
+     *     <li>the base URL of the stream (which, if you try to access to it, returns HTTP
+     *     status code 404 after redirects, and if the URL is valid);</li>
+     *     <li>an {@link ItagItem}, which needs to contain the following information:
+     *          <ul>
+     *              <li>its type (see {@link ItagItem.ItagType}), to identify if the content is
+     *              an audio or a video stream;</li>
+     *              <li>its bitrate;</li>
+     *              <li>its mime type;</li>
+     *              <li>its codec(s);</li>
+     *              <li>for an audio stream: its audio channels;</li>
+     *              <li>for a video stream: its width and height.</li>
+     *          </ul>
+     *     </li>
+     *     <li>the duration of the video, which will be used if the duration could not be
+     *     parsed from the first sequence of the stream.</li>
+     * </ul>
+     *
+     * In order to generate the DASH manifest, this method will:
+     * <ul>
+     *     <li>request the first sequence of the stream (the base URL on which the first
+     *     sequence parameter is appended (see {@link YoutubeDashManifestCreatorsUtils#SQ_0}))
+     *     with a {@code POST} or {@code GET} request (depending of the client on which the
+     *     streaming URL comes from is a mobile one ({@code POST}) or not ({@code GET}));</li>
+     *     <li>follow its redirection(s), if any;</li>
+     *     <li>save the last URL, remove the first sequence parameter;</li>
+     *     <li>use the information provided in the {@link ItagItem} to generate all
+     *     elements of the DASH manifest.</li>
+     * </ul>
      *
-     * <p>In order to generate the DASH manifest, this method will:
-     *      <ul>
-     *          <li>request the first sequence of the stream (the base URL on which the first
-     *          sequence parameter is appended (see {@link YoutubeDashManifestCreatorsUtils#SQ_0}))
-     *          with a {@code POST} or {@code GET} request (depending of the client on which the
-     *          streaming URL comes from is a mobile one ({@code POST}) or not ({@code GET}));</li>
-     *          <li>follow its redirection(s), if any;</li>
-     *          <li>save the last URL, remove the first sequence parameter;</li>
-     *          <li>use the information provided in the {@link ItagItem} to generate all
-     *          elements of the DASH manifest.</li>
-     *      </ul>
-     * </p>
      *
      * <p>
      * If the duration cannot be extracted, the {@code durationSecondsFallback} value will be used

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/dashmanifestcreators/YoutubePostLiveStreamDvrDashManifestCreator.java

@@ -56,38 +56,36 @@ public final class YoutubePostLiveStreamDvrDashManifestCreator {
      * the time)
      * </p>
      *
-     * <p>This method needs:
-     *     <ul>
-     *         <li>the base URL of the stream (which, if you try to access to it, returns HTTP
-     *         status code 404 after redirects, and if the URL is valid);</li>
-     *         <li>an {@link ItagItem}, which needs to contain the following information:
-     *              <ul>
-     *                  <li>its type (see {@link ItagItem.ItagType}), to identify if the content is
-     *                  an audio or a video stream;</li>
-     *                  <li>its bitrate;</li>
-     *                  <li>its mime type;</li>
-     *                  <li>its codec(s);</li>
-     *                  <li>for an audio stream: its audio channels;</li>
-     *                  <li>for a video stream: its width and height.</li>
-     *              </ul>
-     *         </li>
-     *         <li>the duration of the video, which will be used if the duration could not be
-     *         parsed from the first sequence of the stream.</li>
-     *     </ul>
-     * </p>
+     * This method needs:
+     * <ul>
+     *     <li>the base URL of the stream (which, if you try to access to it, returns HTTP
+     *     status code 404 after redirects, and if the URL is valid);</li>
+     *     <li>an {@link ItagItem}, which needs to contain the following information:
+     *         <ul>
+     *             <li>its type (see {@link ItagItem.ItagType}), to identify if the content is
+     *             an audio or a video stream;</li>
+     *             <li>its bitrate;</li>
+     *             <li>its mime type;</li>
+     *             <li>its codec(s);</li>
+     *             <li>for an audio stream: its audio channels;</li>
+     *             <li>for a video stream: its width and height.</li>
+     *         </ul>
+     *     </li>
+     *     <li>the duration of the video, which will be used if the duration could not be
+     *     parsed from the first sequence of the stream.</li>
+     * </ul>
      *
-     * <p>In order to generate the DASH manifest, this method will:
-     *      <ul>
-     *          <li>request the first sequence of the stream (the base URL on which the first
-     *          sequence parameter is appended (see {@link YoutubeDashManifestCreatorsUtils#SQ_0}))
-     *          with a {@code POST} or {@code GET} request (depending of the client on which the
-     *          streaming URL comes from is a mobile one ({@code POST}) or not ({@code GET}));</li>
-     *          <li>follow its redirection(s), if any;</li>
-     *          <li>save the last URL, remove the first sequence parameters;</li>
-     *          <li>use the information provided in the {@link ItagItem} to generate all elements
-     *          of the DASH manifest.</li>
-     *      </ul>
-     * </p>
+     * In order to generate the DASH manifest, this method will:
+     * <ul>
+     *     <li>request the first sequence of the stream (the base URL on which the first
+     *     sequence parameter is appended (see {@link YoutubeDashManifestCreatorsUtils#SQ_0}))
+     *     with a {@code POST} or {@code GET} request (depending of the client on which the
+     *     streaming URL comes from is a mobile one ({@code POST}) or not ({@code GET}));</li>
+     *     <li>follow its redirection(s), if any;</li>
+     *     <li>save the last URL, remove the first sequence parameters;</li>
+     *     <li>use the information provided in the {@link ItagItem} to generate all elements
+     *     of the DASH manifest.</li>
+     * </ul>
      *
      * <p>
      * If the duration cannot be extracted, the {@code durationSecondsFallback} value will be used

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/dashmanifestcreators/YoutubeProgressiveDashManifestCreator.java

@@ -47,26 +47,25 @@ public final class YoutubeProgressiveDashManifestCreator {
      * YouTube partner, and on videos with a large number of views.
      * </p>
      *
-     * <p>This method needs:
-     *     <ul>
-     *         <li>the base URL of the stream (which, if you try to access to it, returns the whole
-     *         stream, after redirects, and if the URL is valid);</li>
-     *         <li>an {@link ItagItem}, which needs to contain the following information:
-     *              <ul>
-     *                  <li>its type (see {@link ItagItem.ItagType}), to identify if the content is
-     *                  an audio or a video stream;</li>
-     *                  <li>its bitrate;</li>
-     *                  <li>its mime type;</li>
-     *                  <li>its codec(s);</li>
-     *                  <li>for an audio stream: its audio channels;</li>
-     *                  <li>for a video stream: its width and height.</li>
-     *              </ul>
-     *         </li>
-     *         <li>the duration of the video (parameter {@code durationSecondsFallback}), which
-     *         will be used as the stream duration if the duration could not be parsed from the
-     *         {@link ItagItem}.</li>
-     *     </ul>
-     * </p>
+     * This method needs:
+     * <ul>
+     *     <li>the base URL of the stream (which, if you try to access to it, returns the whole
+     *     stream, after redirects, and if the URL is valid);</li>
+     *     <li>an {@link ItagItem}, which needs to contain the following information:
+     *          <ul>
+     *              <li>its type (see {@link ItagItem.ItagType}), to identify if the content is
+     *              an audio or a video stream;</li>
+     *              <li>its bitrate;</li>
+     *              <li>its mime type;</li>
+     *              <li>its codec(s);</li>
+     *              <li>for an audio stream: its audio channels;</li>
+     *              <li>for a video stream: its width and height.</li>
+     *          </ul>
+     *     </li>
+     *     <li>the duration of the video (parameter {@code durationSecondsFallback}), which
+     *     will be used as the stream duration if the duration could not be parsed from the
+     *     {@link ItagItem}.</li>
+     * </ul>
      *
      * @param progressiveStreamingBaseUrl the base URL of the progressive stream, which must not be
      *                                    null

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/ItagInfo.java

@@ -8,15 +8,13 @@ import java.io.Serializable;
 /**
  * Class to build easier {@link org.schabi.newpipe.extractor.stream.Stream}s for
  * {@link YoutubeStreamExtractor}.
- *
- * <p>
+ * <br>
  * It stores, per stream:
  * <ul>
  *     <li>its content (the URL/the base URL of streams);</li>
  *     <li>whether its content is the URL the content itself or the base URL;</li>
  *     <li>its associated {@link ItagItem}.</li>
  * </ul>
- * </p>
  */
 final class ItagInfo implements Serializable {
     @Nonnull

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeBaseShowInfoItemExtractor.java

@@ -0,0 +1,65 @@
+package org.schabi.newpipe.extractor.services.youtube.extractors;
+
+import com.grack.nanojson.JsonObject;
+import org.schabi.newpipe.extractor.Image;
+import org.schabi.newpipe.extractor.exceptions.ParsingException;
+import org.schabi.newpipe.extractor.playlist.PlaylistInfoItemExtractor;
+import org.schabi.newpipe.extractor.utils.Utils;
+
+import javax.annotation.Nonnull;
+import java.util.List;
+
+import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject;
+import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getThumbnailsFromInfoItem;
+import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getUrlFromNavigationEndpoint;
+
+/**
+ * The base {@link PlaylistInfoItemExtractor} for shows playlists UI elements.
+ */
+abstract class YoutubeBaseShowInfoItemExtractor implements PlaylistInfoItemExtractor {
+
+    @Nonnull
+    protected final JsonObject showRenderer;
+
+    YoutubeBaseShowInfoItemExtractor(@Nonnull final JsonObject showRenderer) {
+        this.showRenderer = showRenderer;
+    }
+
+    @Override
+    public String getName() throws ParsingException {
+        return showRenderer.getString("title");
+    }
+
+    @Override
+    public String getUrl() throws ParsingException {
+        return getUrlFromNavigationEndpoint(showRenderer.getObject("navigationEndpoint"));
+    }
+
+    @Nonnull
+    @Override
+    public List<Image> getThumbnails() throws ParsingException {
+        return getThumbnailsFromInfoItem(showRenderer.getObject("thumbnailRenderer")
+                .getObject("showCustomThumbnailRenderer"));
+    }
+
+    @Override
+    public long getStreamCount() throws ParsingException {
+        // The stream count should be always returned in the first text object for English
+        // localizations, but the complete text is parsed for reliability purposes
+        final String streamCountText = getTextFromObject(
+                showRenderer.getObject("thumbnailOverlays")
+                        .getObject("thumbnailOverlayBottomPanelRenderer")
+                        .getObject("text"));
+        if (streamCountText == null) {
+            throw new ParsingException("Could not get stream count");
+        }
+
+        try {
+            // The data returned could be a human/shortened number, but no show with more than 1000
+            // videos has been found at the time this code was written
+            return Long.parseLong(Utils.removeNonDigitCharacters(streamCountText));
+        } catch (final NumberFormatException e) {
+            throw new ParsingException("Could not convert stream count to a long", e);
+        }
+    }
+}

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java

@@ -23,7 +23,6 @@ package org.schabi.newpipe.extractor.services.youtube.extractors;
 import static org.schabi.newpipe.extractor.services.youtube.YoutubeChannelHelper.getChannelResponse;
 import static org.schabi.newpipe.extractor.services.youtube.YoutubeChannelHelper.resolveChannelId;
 import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject;
-import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
 
 import com.grack.nanojson.JsonArray;
 import com.grack.nanojson.JsonObject;
@@ -59,10 +58,23 @@ import javax.annotation.Nullable;
 
 public class YoutubeChannelExtractor extends ChannelExtractor {
 
+    // Constants of objects used multiples from channel responses
+    private static final String IMAGE = "image";
+    private static final String CONTENTS = "contents";
+    private static final String CONTENT_PREVIEW_IMAGE_VIEW_MODEL = "contentPreviewImageViewModel";
+    private static final String PAGE_HEADER_VIEW_MODEL = "pageHeaderViewModel";
+    private static final String TAB_RENDERER = "tabRenderer";
+    private static final String CONTENT = "content";
+    private static final String METADATA = "metadata";
+    private static final String AVATAR = "avatar";
+    private static final String THUMBNAILS = "thumbnails";
+    private static final String SOURCES = "sources";
+    private static final String BANNER = "banner";
+
     private JsonObject jsonResponse;
 
-    @SuppressWarnings("OptionalUsedAsFieldOrParameterType")
-    private Optional<ChannelHeader> channelHeader;
+    @Nullable
+    private ChannelHeader channelHeader;
 
     private String channelId;
 
@@ -95,28 +107,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
         jsonResponse = data.jsonResponse;
         channelHeader = YoutubeChannelHelper.getChannelHeader(jsonResponse);
         channelId = data.channelId;
-        channelAgeGateRenderer = getChannelAgeGateRenderer();
-    }
-
-    @Nullable
-    private JsonObject getChannelAgeGateRenderer() {
-        return jsonResponse.getObject("contents")
-                .getObject("twoColumnBrowseResultsRenderer")
-                .getArray("tabs")
-                .stream()
-                .filter(JsonObject.class::isInstance)
-                .map(JsonObject.class::cast)
-                .flatMap(tab -> tab.getObject("tabRenderer")
-                        .getObject("content")
-                        .getObject("sectionListRenderer")
-                        .getArray("contents")
-                        .stream()
-                        .filter(JsonObject.class::isInstance)
-                        .map(JsonObject.class::cast))
-                .filter(content -> content.has("channelAgeGateRenderer"))
-                .map(content -> content.getObject("channelAgeGateRenderer"))
-                .findFirst()
-                .orElse(null);
+        channelAgeGateRenderer = YoutubeChannelHelper.getChannelAgeGateRenderer(jsonResponse);
     }
 
     @Nonnull
@@ -133,62 +124,15 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
     @Override
     public String getId() throws ParsingException {
         assertPageFetched();
-        return channelHeader.map(header -> header.json)
-                .flatMap(header -> Optional.ofNullable(header.getString("channelId"))
-                        .or(() -> Optional.ofNullable(header.getObject("navigationEndpoint")
-                                .getObject("browseEndpoint")
-                                .getString("browseId"))
-                ))
-                .or(() -> Optional.ofNullable(channelId))
-                .orElseThrow(() -> new ParsingException("Could not get channel ID"));
+        return YoutubeChannelHelper.getChannelId(channelHeader, jsonResponse, channelId);
     }
 
     @Nonnull
     @Override
     public String getName() throws ParsingException {
         assertPageFetched();
-        if (channelAgeGateRenderer != null) {
-            final String title = channelAgeGateRenderer.getString("channelTitle");
-            if (isNullOrEmpty(title)) {
-                throw new ParsingException("Could not get channel name");
-            }
-            return title;
-        }
-
-        final String metadataRendererTitle = jsonResponse.getObject("metadata")
-                .getObject("channelMetadataRenderer")
-                .getString("title");
-        if (!isNullOrEmpty(metadataRendererTitle)) {
-            return metadataRendererTitle;
-        }
-
-        return channelHeader.map(header -> {
-            final JsonObject channelJson = header.json;
-            switch (header.headerType) {
-                case PAGE:
-                    return channelJson.getObject("content")
-                            .getObject("pageHeaderViewModel")
-                            .getObject("title")
-                            .getObject("dynamicTextViewModel")
-                            .getObject("text")
-                            .getString("content", channelJson.getString("pageTitle"));
-
-                case CAROUSEL:
-                case INTERACTIVE_TABBED:
-                    return getTextFromObject(channelJson.getObject("title"));
-
-                case C4_TABBED:
-                default:
-                    return channelJson.getString("title");
-            }
-        })
-        // The channel name from a microformatDataRenderer may be different from the one displayed,
-        // especially for auto-generated channels, depending on the language requested for the
-        // interface (hl parameter of InnerTube requests' payload)
-        .or(() -> Optional.ofNullable(jsonResponse.getObject("microformat")
-                .getObject("microformatDataRenderer")
-                .getString("title")))
-        .orElseThrow(() -> new ParsingException("Could not get channel name"));
+        return YoutubeChannelHelper.getChannelName(
+                channelHeader, channelAgeGateRenderer, jsonResponse);
     }
 
     @Nonnull
@@ -196,33 +140,46 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
     public List<Image> getAvatars() throws ParsingException {
         assertPageFetched();
         if (channelAgeGateRenderer != null) {
-            return Optional.ofNullable(channelAgeGateRenderer.getObject("avatar")
-                    .getArray("thumbnails"))
+            return Optional.ofNullable(channelAgeGateRenderer.getObject(AVATAR)
+                    .getArray(THUMBNAILS))
                     .map(YoutubeParsingHelper::getImagesFromThumbnailsArray)
                     .orElseThrow(() -> new ParsingException("Could not get avatars"));
         }
 
-        return channelHeader.map(header -> {
-            switch (header.headerType) {
-                case PAGE:
-                    return header.json.getObject("content")
-                            .getObject("pageHeaderViewModel")
-                            .getObject("image")
-                            .getObject("contentPreviewImageViewModel")
-                            .getObject("image")
-                            .getArray("sources");
+        return Optional.ofNullable(channelHeader)
+                .map(header -> {
+                    switch (header.headerType) {
+                        case PAGE:
+                            final JsonObject imageObj = header.json.getObject(CONTENT)
+                                    .getObject(PAGE_HEADER_VIEW_MODEL)
+                                    .getObject(IMAGE);
 
-                case INTERACTIVE_TABBED:
-                    return header.json.getObject("boxArt")
-                            .getArray("thumbnails");
+                            if (imageObj.has(CONTENT_PREVIEW_IMAGE_VIEW_MODEL)) {
+                                return imageObj.getObject(CONTENT_PREVIEW_IMAGE_VIEW_MODEL)
+                                        .getObject(IMAGE)
+                                        .getArray(SOURCES);
+                            }
 
-                case C4_TABBED:
-                case CAROUSEL:
-                default:
-                    return header.json.getObject("avatar")
-                            .getArray("thumbnails");
-            }
-        })
+                            if (imageObj.has("decoratedAvatarViewModel")) {
+                                return imageObj.getObject("decoratedAvatarViewModel")
+                                        .getObject(AVATAR)
+                                        .getObject("avatarViewModel")
+                                        .getObject(IMAGE)
+                                        .getArray(SOURCES);
+                            }
+
+                            // Return an empty avatar array as a fallback
+                            return new JsonArray();
+                        case INTERACTIVE_TABBED:
+                            return header.json.getObject("boxArt")
+                                    .getArray(THUMBNAILS);
+                        case C4_TABBED:
+                        case CAROUSEL:
+                        default:
+                            return header.json.getObject(AVATAR)
+                                    .getArray(THUMBNAILS);
+                    }
+                })
                 .map(YoutubeParsingHelper::getImagesFromThumbnailsArray)
                 .orElseThrow(() -> new ParsingException("Could not get avatars"));
     }
@@ -235,10 +192,28 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
             return List.of();
         }
 
-        // No banner is available on pageHeaderRenderer headers
-        return channelHeader.filter(header -> header.headerType != HeaderType.PAGE)
-                .map(header -> header.json.getObject("banner")
-                        .getArray("thumbnails"))
+        return Optional.ofNullable(channelHeader)
+                .map(header -> {
+                    if (header.headerType == HeaderType.PAGE) {
+                        final JsonObject pageHeaderViewModel = header.json.getObject(CONTENT)
+                                .getObject(PAGE_HEADER_VIEW_MODEL);
+
+                        if (pageHeaderViewModel.has(BANNER)) {
+                            return pageHeaderViewModel.getObject(BANNER)
+                                    .getObject("imageBannerViewModel")
+                                    .getObject(IMAGE)
+                                    .getArray(SOURCES);
+                        }
+
+                        // No banner is available (this should happen on pageHeaderRenderers of
+                        // system channels), use an empty JsonArray instead
+                        return new JsonArray();
+                    }
+
+                    return header.json
+                            .getObject(BANNER)
+                            .getArray(THUMBNAILS);
+                })
                 .map(YoutubeParsingHelper::getImagesFromThumbnailsArray)
                 .orElse(List.of());
     }
@@ -261,17 +236,17 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
             return UNKNOWN_SUBSCRIBER_COUNT;
         }
 
-        if (channelHeader.isPresent()) {
-            final ChannelHeader header = channelHeader.get();
-
-            if (header.headerType == HeaderType.INTERACTIVE_TABBED
-                    || header.headerType == HeaderType.PAGE) {
-                // No subscriber count is available on interactiveTabbedHeaderRenderer and
-                // pageHeaderRenderer headers
+        if (channelHeader != null) {
+            if (channelHeader.headerType == HeaderType.INTERACTIVE_TABBED) {
+                // No subscriber count is available on interactiveTabbedHeaderRenderer header
                 return UNKNOWN_SUBSCRIBER_COUNT;
             }
 
-            final JsonObject headerJson = header.json;
+            final JsonObject headerJson = channelHeader.json;
+            if (channelHeader.headerType == HeaderType.PAGE) {
+                return getSubscriberCountFromPageChannelHeader(headerJson);
+            }
+
             JsonObject textObject = null;
 
             if (headerJson.has("subscriberCountText")) {
@@ -292,6 +267,51 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
         return UNKNOWN_SUBSCRIBER_COUNT;
     }
 
+    private long getSubscriberCountFromPageChannelHeader(@Nonnull final JsonObject headerJson)
+            throws ParsingException {
+        final JsonObject metadataObject = headerJson.getObject(CONTENT)
+                .getObject(PAGE_HEADER_VIEW_MODEL)
+                .getObject(METADATA);
+        if (metadataObject.has("contentMetadataViewModel")) {
+            final JsonArray metadataPart = metadataObject.getObject("contentMetadataViewModel")
+                    .getArray("metadataRows")
+                    .stream()
+                    .filter(JsonObject.class::isInstance)
+                    .map(JsonObject.class::cast)
+                    .map(metadataRow -> metadataRow.getArray("metadataParts"))
+                    /*
+                    Find metadata parts which have two elements: channel handle and subscriber
+                    count.
+
+                    On autogenerated music channels, the subscriber count is not shown with this
+                    header.
+
+                    Use the first metadata parts object found.
+                     */
+                    .filter(metadataParts -> metadataParts.size() == 2)
+                    .findFirst()
+                    .orElse(null);
+            if (metadataPart == null) {
+                // As the parsing of the metadata parts object needed to get the subscriber count
+                // is fragile, return UNKNOWN_SUBSCRIBER_COUNT when it cannot be got
+                return UNKNOWN_SUBSCRIBER_COUNT;
+            }
+
+            try {
+                // The subscriber count is at the same position for all languages as of 02/03/2024
+                return Utils.mixedNumberWordToLong(metadataPart.getObject(0)
+                        .getObject("text")
+                        .getString(CONTENT));
+            } catch (final NumberFormatException e) {
+                throw new ParsingException("Could not get subscriber count", e);
+            }
+        }
+
+        // If the channel header has no contentMetadataViewModel (which is the case for system
+        // channels using this header), return UNKNOWN_SUBSCRIBER_COUNT
+        return UNKNOWN_SUBSCRIBER_COUNT;
+    }
+
     @Override
     public String getDescription() throws ParsingException {
         assertPageFetched();
@@ -300,29 +320,20 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
         }
 
         try {
-            if (channelHeader.isPresent()) {
-                final ChannelHeader header = channelHeader.get();
-
-                if (header.headerType == HeaderType.PAGE) {
-                    // A pageHeaderRenderer doesn't contain a description
-                    return null;
-                }
-
-                if (header.headerType == HeaderType.INTERACTIVE_TABBED) {
-                    /*
-                    In an interactiveTabbedHeaderRenderer, the real description, is only available
-                    in its header
-                    The other one returned in non-About tabs accessible in the
-                    microformatDataRenderer object of the response may be completely different
-                    The description extracted is incomplete and the original one can be only
-                    accessed from the About tab
-                     */
-                    return getTextFromObject(header.json.getObject("description"));
-                }
+            if (channelHeader != null
+                    && channelHeader.headerType == HeaderType.INTERACTIVE_TABBED) {
+                /*
+                In an interactiveTabbedHeaderRenderer, the real description, is only available
+                in its header
+                The other one returned in non-About tabs accessible in the
+                microformatDataRenderer object of the response may be completely different
+                The description extracted is incomplete and the original one can be only
+                accessed from the About tab
+                 */
+                return getTextFromObject(channelHeader.json.getObject("description"));
             }
 
-            // The description is cut and the original one can be only accessed from the About tab
-            return jsonResponse.getObject("metadata")
+            return jsonResponse.getObject(METADATA)
                     .getObject("channelMetadataRenderer")
                     .getString("description");
         } catch (final Exception e) {
@@ -350,31 +361,16 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
     public boolean isVerified() throws ParsingException {
         assertPageFetched();
         if (channelAgeGateRenderer != null) {
+            // Verified status is unknown with channelAgeGateRenderers, return false in this case
             return false;
         }
 
-        if (channelHeader.isPresent()) {
-            final ChannelHeader header = channelHeader.get();
-
-            // carouselHeaderRenderer and pageHeaderRenderer does not contain any verification
-            // badges
-            // Since they are only shown on YouTube internal channels or on channels of large
-            // organizations broadcasting live events, we can assume the channel to be verified
-            if (header.headerType == HeaderType.CAROUSEL || header.headerType == HeaderType.PAGE) {
-                return true;
-            }
-
-            if (header.headerType == HeaderType.INTERACTIVE_TABBED) {
-                // If the header has an autoGenerated property, it should mean that the channel has
-                // been auto generated by YouTube: we can assume the channel to be verified in this
-                // case
-                return header.json.has("autoGenerated");
-            }
-
-            return YoutubeParsingHelper.isVerified(header.json.getArray("badges"));
+        if (channelHeader == null) {
+            throw new ParsingException(
+            "Could not get channel verified status, no channel header has been extracted");
         }
 
-        return false;
+        return YoutubeChannelHelper.isChannelVerified(channelHeader);
     }
 
     @Nonnull
@@ -390,7 +386,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
 
     @Nonnull
     private List<ListLinkHandler> getTabsForNonAgeRestrictedChannels() throws ParsingException {
-        final JsonArray responseTabs = jsonResponse.getObject("contents")
+        final JsonArray responseTabs = jsonResponse.getObject(CONTENTS)
                 .getObject("twoColumnBrowseResultsRenderer")
                 .getArray("tabs");
 
@@ -411,8 +407,8 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
         responseTabs.stream()
                 .filter(JsonObject.class::isInstance)
                 .map(JsonObject.class::cast)
-                .filter(tab -> tab.has("tabRenderer"))
-                .map(tab -> tab.getObject("tabRenderer"))
+                .filter(tab -> tab.has(TAB_RENDERER))
+                .map(tab -> tab.getObject(TAB_RENDERER))
                 .forEach(tabRenderer -> {
                     final String tabUrl = tabRenderer.getObject("endpoint")
                             .getObject("commandMetadata")
@@ -426,6 +422,19 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
 
                         final String urlSuffix = urlParts[urlParts.length - 1];
 
+                        /*
+                        Make a copy of the channelHeader member to avoid keeping a reference to
+                        this YoutubeChannelExtractor instance which would prevent serialization of
+                        the ReadyChannelTabListLinkHandler instance created above
+                         */
+                        final ChannelHeader channelHeaderCopy;
+                        if (channelHeader == null) {
+                            channelHeaderCopy = null;
+                        } else {
+                            channelHeaderCopy = new ChannelHeader(channelHeader.json,
+                                    channelHeader.headerType);
+                        }
+
                         switch (urlSuffix) {
                             case "videos":
                                 // Since the Videos tab has already its contents fetched, make
@@ -436,8 +445,8 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
                                         channelId,
                                         ChannelTabs.VIDEOS,
                                         (service, linkHandler) -> new VideosTabExtractor(
-                                                service, linkHandler, tabRenderer, name, id, url)));
-
+                                                service, linkHandler, tabRenderer,
+                                                channelHeaderCopy, name, id, url)));
                                 break;
                             case "shorts":
                                 addNonVideosTab.accept(ChannelTabs.SHORTS);
@@ -445,9 +454,15 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
                             case "streams":
                                 addNonVideosTab.accept(ChannelTabs.LIVESTREAMS);
                                 break;
+                            case "releases":
+                                addNonVideosTab.accept(ChannelTabs.ALBUMS);
+                                break;
                             case "playlists":
                                 addNonVideosTab.accept(ChannelTabs.PLAYLISTS);
                                 break;
+                            default:
+                                // Unsupported channel tab, ignore it
+                                break;
                         }
                     }
                 });

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelTabExtractor.java

@@ -29,8 +29,6 @@ import static org.schabi.newpipe.extractor.services.youtube.YoutubeChannelHelper
 import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.DISABLE_PRETTY_PRINT_PARAMETER;
 import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.YOUTUBEI_V1_URL;
 import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonPostResponse;
-import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getKey;
-import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject;
 import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareDesktopJsonBuilder;
 import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
 
@@ -38,34 +36,21 @@ import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
  * A {@link ChannelTabExtractor} implementation for the YouTube service.
  *
  * <p>
- * It currently supports {@code Videos}, {@code Shorts}, {@code Live}, {@code Playlists} and
- * {@code Channels} tabs.
+ * It currently supports {@code Videos}, {@code Shorts}, {@code Live}, {@code Playlists},
+ * {@code Albums} and {@code Channels} tabs.
  * </p>
  */
 public class YoutubeChannelTabExtractor extends ChannelTabExtractor {
 
-    /**
-     * Whether the visitor data extracted from the initial channel response is required to be used
-     * for continuations.
-     *
-     * <p>
-     * A valid {@code visitorData} is required to get continuations of shorts in channels.
-     * </p>
-     *
-     * <p>
-     * It should be not used when it is not needed, in order to reduce YouTube's tracking.
-     * </p>
-     */
-    private final boolean useVisitorData;
+    @Nullable
+    protected YoutubeChannelHelper.ChannelHeader channelHeader;
+
     private JsonObject jsonResponse;
     private String channelId;
-    @Nullable
-    private String visitorData;
 
     public YoutubeChannelTabExtractor(final StreamingService service,
                                       final ListLinkHandler linkHandler) {
         super(service, linkHandler);
-        useVisitorData = getName().equals(ChannelTabs.SHORTS);
     }
 
     @Nonnull
@@ -78,6 +63,8 @@ public class YoutubeChannelTabExtractor extends ChannelTabExtractor {
                 return "EgZzaG9ydHPyBgUKA5oBAA%3D%3D";
             case ChannelTabs.LIVESTREAMS:
                 return "EgdzdHJlYW1z8gYECgJ6AA%3D%3D";
+            case ChannelTabs.ALBUMS:
+                return "EghyZWxlYXNlc_IGBQoDsgEA";
             case ChannelTabs.PLAYLISTS:
                 return "EglwbGF5bGlzdHPyBgQKAkIA";
             default:
@@ -88,18 +75,16 @@ public class YoutubeChannelTabExtractor extends ChannelTabExtractor {
     @Override
     public void onFetchPage(@Nonnull final Downloader downloader) throws IOException,
             ExtractionException {
-        channelId = resolveChannelId(super.getId());
+        final String channelIdFromId = resolveChannelId(super.getId());
 
         final String params = getChannelTabsParameters();
 
-        final YoutubeChannelHelper.ChannelResponseData data = getChannelResponse(channelId,
+        final YoutubeChannelHelper.ChannelResponseData data = getChannelResponse(channelIdFromId,
                 params, getExtractorLocalization(), getExtractorContentCountry());
 
         jsonResponse = data.jsonResponse;
+        channelHeader = YoutubeChannelHelper.getChannelHeader(jsonResponse);
         channelId = data.channelId;
-        if (useVisitorData) {
-            visitorData = jsonResponse.getObject("responseContext").getString("visitorData");
-        }
     }
 
     @Nonnull
@@ -116,60 +101,13 @@ public class YoutubeChannelTabExtractor extends ChannelTabExtractor {
     @Nonnull
     @Override
     public String getId() throws ParsingException {
-        final String id = jsonResponse.getObject("header")
-                .getObject("c4TabbedHeaderRenderer")
-                .getString("channelId", "");
-
-        if (!id.isEmpty()) {
-            return id;
-        }
-
-        final Optional<String> carouselHeaderId = jsonResponse.getObject("header")
-                .getObject("carouselHeaderRenderer")
-                .getArray("contents")
-                .stream()
-                .filter(JsonObject.class::isInstance)
-                .map(JsonObject.class::cast)
-                .filter(item -> item.has("topicChannelDetailsRenderer"))
-                .findFirst()
-                .flatMap(item ->
-                        Optional.ofNullable(item.getObject("topicChannelDetailsRenderer")
-                                .getObject("navigationEndpoint")
-                                .getObject("browseEndpoint")
-                                .getString("browseId")));
-        if (carouselHeaderId.isPresent()) {
-            return carouselHeaderId.get();
-        }
-
-        if (!isNullOrEmpty(channelId)) {
-            return channelId;
-        } else {
-            throw new ParsingException("Could not get channel ID");
-        }
+       return YoutubeChannelHelper.getChannelId(channelHeader, jsonResponse, channelId);
     }
 
-    protected String getChannelName() {
-        final String metadataName = jsonResponse.getObject("metadata")
-                .getObject("channelMetadataRenderer")
-                .getString("title");
-        if (!isNullOrEmpty(metadataName)) {
-            return metadataName;
-        }
-
-        return YoutubeChannelHelper.getChannelHeader(jsonResponse)
-                .map(header -> {
-                    final Object title = header.json.get("title");
-                    if (title instanceof String) {
-                        return (String) title;
-                    } else if (title instanceof JsonObject) {
-                        final String headerName = getTextFromObject((JsonObject) title);
-                        if (!isNullOrEmpty(headerName)) {
-                            return headerName;
-                        }
-                    }
-                    return "";
-                })
-                .orElse("");
+    protected String getChannelName() throws ParsingException {
+        return YoutubeChannelHelper.getChannelName(channelHeader,
+                YoutubeChannelHelper.getChannelAgeGateRenderer(jsonResponse),
+                jsonResponse);
     }
 
     @Nonnull
@@ -203,18 +141,28 @@ public class YoutubeChannelTabExtractor extends ChannelTabExtractor {
             }
         }
 
+        final VerifiedStatus verifiedStatus;
+        if (channelHeader == null) {
+            verifiedStatus = VerifiedStatus.UNKNOWN;
+        } else {
+            verifiedStatus = YoutubeChannelHelper.isChannelVerified(channelHeader)
+                    ? VerifiedStatus.VERIFIED
+                    : VerifiedStatus.UNVERIFIED;
+        }
+
         // If a channel tab is fetched, the next page requires channel ID and name, as channel
         // streams don't have their channel specified.
         // We also need to set the visitor data here when it should be enabled, as it is required
         // to get continuations on some channel tabs, and we need a way to pass it between pages
-        final List<String> channelIds = useVisitorData && !isNullOrEmpty(visitorData)
-                ? List.of(getChannelName(), getUrl(), visitorData)
-                : List.of(getChannelName(), getUrl());
+        final String channelName = getChannelName();
+        final String channelUrl = getUrl();
 
-        final JsonObject continuation = collectItemsFrom(collector, items, channelIds)
+        final JsonObject continuation = collectItemsFrom(collector, items, verifiedStatus,
+                channelName, channelUrl)
                 .orElse(null);
 
-        final Page nextPage = getNextPageFrom(continuation, channelIds);
+        final Page nextPage = getNextPageFrom(
+                continuation, List.of(channelName, channelUrl, verifiedStatus.toString()));
 
         return new InfoItemsPage<>(collector, nextPage);
     }
@@ -280,16 +228,48 @@ public class YoutubeChannelTabExtractor extends ChannelTabExtractor {
     private Optional<JsonObject> collectItemsFrom(@Nonnull final MultiInfoItemsCollector collector,
                                                   @Nonnull final JsonArray items,
                                                   @Nonnull final List<String> channelIds) {
+        final String channelName;
+        final String channelUrl;
+        VerifiedStatus verifiedStatus;
+
+        if (channelIds.size() >= 3) {
+            channelName = channelIds.get(0);
+            channelUrl = channelIds.get(1);
+            try {
+                verifiedStatus = VerifiedStatus.valueOf(channelIds.get(2));
+            } catch (final IllegalArgumentException e) {
+                // An IllegalArgumentException can be thrown if someone passes a third channel ID
+                // which is not of the enum type in the getPage method, use the UNKNOWN
+                // VerifiedStatus enum value in this case
+                verifiedStatus = VerifiedStatus.UNKNOWN;
+            }
+        } else {
+            channelName = null;
+            channelUrl = null;
+            verifiedStatus = VerifiedStatus.UNKNOWN;
+        }
+
+        return collectItemsFrom(collector, items, verifiedStatus, channelName, channelUrl);
+    }
+
+    private Optional<JsonObject> collectItemsFrom(@Nonnull final MultiInfoItemsCollector collector,
+                                                  @Nonnull final JsonArray items,
+                                                  @Nonnull final VerifiedStatus verifiedStatus,
+                                                  @Nullable final String channelName,
+                                                  @Nullable final String channelUrl) {
         return items.stream()
                 .filter(JsonObject.class::isInstance)
                 .map(JsonObject.class::cast)
-                .map(item -> collectItem(collector, item, channelIds))
+                .map(item -> collectItem(
+                        collector, item, verifiedStatus, channelName, channelUrl))
                 .reduce(Optional.empty(), (c1, c2) -> c1.or(() -> c2));
     }
 
     private Optional<JsonObject> collectItem(@Nonnull final MultiInfoItemsCollector collector,
                                              @Nonnull final JsonObject item,
-                                             @Nonnull final List<String> channelIds) {
+                                             @Nonnull final VerifiedStatus channelVerifiedStatus,
+                                             @Nullable final String channelName,
+                                             @Nullable final String channelUrl) {
         final TimeAgoParser timeAgoParser = getTimeAgoParser();
 
         if (item.has("richItemRenderer")) {
@@ -297,33 +277,46 @@ public class YoutubeChannelTabExtractor extends ChannelTabExtractor {
                     .getObject("content");
 
             if (richItem.has("videoRenderer")) {
-                getCommitVideoConsumer(collector, timeAgoParser, channelIds,
-                        richItem.getObject("videoRenderer"));
+                commitVideo(collector, timeAgoParser, richItem.getObject("videoRenderer"),
+                        channelVerifiedStatus, channelName, channelUrl);
             } else if (richItem.has("reelItemRenderer")) {
-                getCommitReelItemConsumer(collector, timeAgoParser, channelIds,
-                        richItem.getObject("reelItemRenderer"));
+                commitReel(collector, richItem.getObject("reelItemRenderer"),
+                        channelVerifiedStatus, channelName, channelUrl);
+            } else if (richItem.has("shortsLockupViewModel")) {
+                commitShortsLockup(collector, richItem.getObject("shortsLockupViewModel"),
+                        channelVerifiedStatus, channelName, channelUrl);
             } else if (richItem.has("playlistRenderer")) {
-                getCommitPlaylistConsumer(collector, channelIds,
-                        item.getObject("playlistRenderer"));
+                commitPlaylist(collector, richItem.getObject("playlistRenderer"),
+                        channelVerifiedStatus, channelName, channelUrl);
             }
         } else if (item.has("gridVideoRenderer")) {
-            getCommitVideoConsumer(collector, timeAgoParser, channelIds,
-                    item.getObject("gridVideoRenderer"));
+            commitVideo(collector, timeAgoParser, item.getObject("gridVideoRenderer"),
+                    channelVerifiedStatus, channelName, channelUrl);
         } else if (item.has("gridPlaylistRenderer")) {
-            getCommitPlaylistConsumer(collector, channelIds,
-                    item.getObject("gridPlaylistRenderer"));
+            commitPlaylist(collector, item.getObject("gridPlaylistRenderer"),
+                    channelVerifiedStatus, channelName, channelUrl);
+        } else if (item.has("gridShowRenderer")) {
+            collector.commit(new YoutubeGridShowRendererChannelInfoItemExtractor(
+                    item.getObject("gridShowRenderer"), channelVerifiedStatus, channelName,
+                    channelUrl));
         } else if (item.has("shelfRenderer")) {
             return collectItem(collector, item.getObject("shelfRenderer")
-                    .getObject("content"), channelIds);
+                    .getObject("content"), channelVerifiedStatus, channelName, channelUrl);
         } else if (item.has("itemSectionRenderer")) {
             return collectItemsFrom(collector, item.getObject("itemSectionRenderer")
-                    .getArray("contents"), channelIds);
+                    .getArray("contents"), channelVerifiedStatus, channelName, channelUrl);
         } else if (item.has("horizontalListRenderer")) {
             return collectItemsFrom(collector, item.getObject("horizontalListRenderer")
-                    .getArray("items"), channelIds);
+                    .getArray("items"), channelVerifiedStatus, channelName, channelUrl);
         } else if (item.has("expandedShelfContentsRenderer")) {
             return collectItemsFrom(collector, item.getObject("expandedShelfContentsRenderer")
-                    .getArray("items"), channelIds);
+                    .getArray("items"), channelVerifiedStatus, channelName, channelUrl);
+        } else if (item.has("lockupViewModel")) {
+            final JsonObject lockupViewModel = item.getObject("lockupViewModel");
+            if ("LOCKUP_CONTENT_TYPE_PLAYLIST".equals(lockupViewModel.getString("contentType"))) {
+                commitPlaylistLockup(collector, lockupViewModel, channelVerifiedStatus,
+                        channelName, channelUrl);
+            }
         } else if (item.has("continuationItemRenderer")) {
             return Optional.ofNullable(item.getObject("continuationItemRenderer"));
         }
@@ -331,73 +324,146 @@ public class YoutubeChannelTabExtractor extends ChannelTabExtractor {
         return Optional.empty();
     }
 
-    private void getCommitVideoConsumer(@Nonnull final MultiInfoItemsCollector collector,
-                                        @Nonnull final TimeAgoParser timeAgoParser,
-                                        @Nonnull final List<String> channelIds,
-                                        @Nonnull final JsonObject jsonObject) {
+    private static void commitReel(@Nonnull final MultiInfoItemsCollector collector,
+                                   @Nonnull final JsonObject reelItemRenderer,
+                                   @Nonnull final VerifiedStatus channelVerifiedStatus,
+                                   @Nullable final String channelName,
+                                   @Nullable final String channelUrl) {
+        collector.commit(
+                new YoutubeReelInfoItemExtractor(reelItemRenderer) {
+                    @Override
+                    public String getUploaderName() throws ParsingException {
+                        return isNullOrEmpty(channelName) ? super.getUploaderName() : channelName;
+                    }
+
+                    @Override
+                    public String getUploaderUrl() throws ParsingException {
+                        return isNullOrEmpty(channelUrl) ? super.getUploaderName() : channelUrl;
+                    }
+
+                    @Override
+                    public boolean isUploaderVerified() {
+                        return channelVerifiedStatus == VerifiedStatus.VERIFIED;
+                    }
+                });
+    }
+
+    private static void commitShortsLockup(@Nonnull final MultiInfoItemsCollector collector,
+                                           @Nonnull final JsonObject shortsLockupViewModel,
+                                           @Nonnull final VerifiedStatus channelVerifiedStatus,
+                                           @Nullable final String channelName,
+                                           @Nullable final String channelUrl) {
+        collector.commit(
+                new YoutubeShortsLockupInfoItemExtractor(shortsLockupViewModel) {
+                    @Override
+                    public String getUploaderName() throws ParsingException {
+                        return isNullOrEmpty(channelName) ? super.getUploaderName() : channelName;
+                    }
+
+                    @Override
+                    public String getUploaderUrl() throws ParsingException {
+                        return isNullOrEmpty(channelUrl) ? super.getUploaderName() : channelUrl;
+                    }
+
+                    @Override
+                    public boolean isUploaderVerified() {
+                        return channelVerifiedStatus == VerifiedStatus.VERIFIED;
+                    }
+                });
+    }
+
+    private void commitPlaylistLockup(@Nonnull final MultiInfoItemsCollector collector,
+                                      @Nonnull final JsonObject playlistLockupViewModel,
+                                      @Nonnull final VerifiedStatus channelVerifiedStatus,
+                                      @Nullable final String channelName,
+                                      @Nullable final String channelUrl) {
+        collector.commit(
+                new YoutubeMixOrPlaylistLockupInfoItemExtractor(playlistLockupViewModel) {
+                    @Override
+                    public String getUploaderName() throws ParsingException {
+                        return isNullOrEmpty(channelName) ? super.getUploaderName() : channelName;
+                    }
+
+                    @Override
+                    public String getUploaderUrl() throws ParsingException {
+                        return isNullOrEmpty(channelUrl) ? super.getUploaderName() : channelUrl;
+                    }
+
+                    @Override
+                    public boolean isUploaderVerified() throws ParsingException {
+                        switch (channelVerifiedStatus) {
+                            case VERIFIED:
+                                return true;
+                            case UNVERIFIED:
+                                return false;
+                            default:
+                                return super.isUploaderVerified();
+                        }
+                    }
+                });
+    }
+
+    private void commitVideo(@Nonnull final MultiInfoItemsCollector collector,
+                             @Nonnull final TimeAgoParser timeAgoParser,
+                             @Nonnull final JsonObject jsonObject,
+                             @Nonnull final VerifiedStatus channelVerifiedStatus,
+                             @Nullable final String channelName,
+                             @Nullable final String channelUrl) {
         collector.commit(
                 new YoutubeStreamInfoItemExtractor(jsonObject, timeAgoParser) {
                     @Override
                     public String getUploaderName() throws ParsingException {
-                        if (channelIds.size() >= 2) {
-                            return channelIds.get(0);
-                        }
-                        return super.getUploaderName();
+                        return isNullOrEmpty(channelName) ? super.getUploaderName() : channelName;
                     }
 
                     @Override
                     public String getUploaderUrl() throws ParsingException {
-                        if (channelIds.size() >= 2) {
-                            return channelIds.get(1);
+                        return isNullOrEmpty(channelUrl) ? super.getUploaderName() : channelUrl;
+                    }
+
+                    @SuppressWarnings("DuplicatedCode")
+                    @Override
+                    public boolean isUploaderVerified() throws ParsingException {
+                        switch (channelVerifiedStatus) {
+                            case VERIFIED:
+                                return true;
+                            case UNVERIFIED:
+                                return false;
+                            default:
+                                return super.isUploaderVerified();
                         }
-                        return super.getUploaderUrl();
                     }
                 });
     }
 
-    private void getCommitReelItemConsumer(@Nonnull final MultiInfoItemsCollector collector,
-                                           @Nonnull final TimeAgoParser timeAgoParser,
-                                           @Nonnull final List<String> channelIds,
-                                           @Nonnull final JsonObject jsonObject) {
-        collector.commit(
-                new YoutubeReelInfoItemExtractor(jsonObject, timeAgoParser) {
-                    @Override
-                    public String getUploaderName() throws ParsingException {
-                        if (channelIds.size() >= 2) {
-                            return channelIds.get(0);
-                        }
-                        return super.getUploaderName();
-                    }
-
-                    @Override
-                    public String getUploaderUrl() throws ParsingException {
-                        if (channelIds.size() >= 2) {
-                            return channelIds.get(1);
-                        }
-                        return super.getUploaderUrl();
-                    }
-                });
-    }
-
-    private void getCommitPlaylistConsumer(@Nonnull final MultiInfoItemsCollector collector,
-                                           @Nonnull final List<String> channelIds,
-                                           @Nonnull final JsonObject jsonObject) {
+    private void commitPlaylist(@Nonnull final MultiInfoItemsCollector collector,
+                                @Nonnull final JsonObject jsonObject,
+                                @Nonnull final VerifiedStatus channelVerifiedStatus,
+                                @Nullable final String channelName,
+                                @Nullable final String channelUrl) {
         collector.commit(
                 new YoutubePlaylistInfoItemExtractor(jsonObject) {
                     @Override
                     public String getUploaderName() throws ParsingException {
-                        if (channelIds.size() >= 2) {
-                            return channelIds.get(0);
-                        }
-                        return super.getUploaderName();
+                        return isNullOrEmpty(channelName) ? super.getUploaderName() : channelName;
                     }
 
                     @Override
                     public String getUploaderUrl() throws ParsingException {
-                        if (channelIds.size() >= 2) {
-                            return channelIds.get(1);
+                        return isNullOrEmpty(channelUrl) ? super.getUploaderName() : channelUrl;
+                    }
+
+                    @SuppressWarnings("DuplicatedCode")
+                    @Override
+                    public boolean isUploaderVerified() throws ParsingException {
+                        switch (channelVerifiedStatus) {
+                            case VERIFIED:
+                                return true;
+                            case UNVERIFIED:
+                                return false;
+                            default:
+                                return super.isUploaderVerified();
                         }
-                        return super.getUploaderUrl();
                     }
                 });
     }
@@ -415,14 +481,13 @@ public class YoutubeChannelTabExtractor extends ChannelTabExtractor {
                 .getString("token");
 
         final byte[] body = JsonWriter.string(prepareDesktopJsonBuilder(getExtractorLocalization(),
-                        getExtractorContentCountry(),
-                        useVisitorData && channelIds.size() >= 3 ? channelIds.get(2) : null)
+                        getExtractorContentCountry())
                         .value("continuation", continuation)
                         .done())
                 .getBytes(StandardCharsets.UTF_8);
 
-        return new Page(YOUTUBEI_V1_URL + "browse?key=" + getKey()
-                + DISABLE_PRETTY_PRINT_PARAMETER, null, channelIds, null, body);
+        return new Page(YOUTUBEI_V1_URL + "browse?" + DISABLE_PRETTY_PRINT_PARAMETER, null,
+                channelIds, null, body);
     }
 
     /**
@@ -431,20 +496,23 @@ public class YoutubeChannelTabExtractor extends ChannelTabExtractor {
      */
     public static final class VideosTabExtractor extends YoutubeChannelTabExtractor {
         private final JsonObject tabRenderer;
-        private final String channelName;
         private final String channelId;
+        private final String channelName;
         private final String channelUrl;
 
         VideosTabExtractor(final StreamingService service,
                            final ListLinkHandler linkHandler,
                            final JsonObject tabRenderer,
+                           @Nullable final YoutubeChannelHelper.ChannelHeader channelHeader,
                            final String channelName,
                            final String channelId,
                            final String channelUrl) {
             super(service, linkHandler);
+            this.channelHeader = channelHeader;
+
             this.tabRenderer = tabRenderer;
-            this.channelName = channelName;
             this.channelId = channelId;
+            this.channelName = channelName;
             this.channelUrl = channelUrl;
         }
 
@@ -475,4 +543,59 @@ public class YoutubeChannelTabExtractor extends ChannelTabExtractor {
             return Optional.of(tabRenderer);
         }
     }
+
+    /**
+     * Enum representing the verified state of a channel
+     */
+    private enum VerifiedStatus {
+        VERIFIED,
+        UNVERIFIED,
+        UNKNOWN
+    }
+
+    private static final class YoutubeGridShowRendererChannelInfoItemExtractor
+            extends YoutubeBaseShowInfoItemExtractor {
+
+        @Nonnull
+        private final VerifiedStatus verifiedStatus;
+
+        @Nullable
+        private final String channelName;
+
+        @Nullable
+        private final String channelUrl;
+
+        private YoutubeGridShowRendererChannelInfoItemExtractor(
+                @Nonnull final JsonObject gridShowRenderer,
+                @Nonnull final VerifiedStatus verifiedStatus,
+                @Nullable final String channelName,
+                @Nullable final String channelUrl) {
+            super(gridShowRenderer);
+            this.verifiedStatus = verifiedStatus;
+            this.channelName = channelName;
+            this.channelUrl = channelUrl;
+        }
+
+        @Override
+        public String getUploaderName() {
+            return channelName;
+        }
+
+        @Override
+        public String getUploaderUrl() {
+            return channelUrl;
+        }
+
+        @Override
+        public boolean isUploaderVerified() throws ParsingException {
+            switch (verifiedStatus) {
+                case VERIFIED:
+                    return true;
+                case UNVERIFIED:
+                    return false;
+                default:
+                    throw new ParsingException("Could not get uploader verification status");
+            }
+        }
+    }
 }

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsEUVMInfoItemExtractor.java

@@ -0,0 +1,235 @@
+package org.schabi.newpipe.extractor.services.youtube.extractors;
+
+import com.grack.nanojson.JsonObject;
+import org.schabi.newpipe.extractor.Image;
+import org.schabi.newpipe.extractor.Page;
+import org.schabi.newpipe.extractor.comments.CommentsInfoItemExtractor;
+import org.schabi.newpipe.extractor.exceptions.ParsingException;
+import org.schabi.newpipe.extractor.localization.DateWrapper;
+import org.schabi.newpipe.extractor.localization.TimeAgoParser;
+import org.schabi.newpipe.extractor.stream.Description;
+import org.schabi.newpipe.extractor.utils.Utils;
+
+import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
+import java.util.List;
+import java.util.Objects;
+
+import static org.schabi.newpipe.extractor.services.youtube.YoutubeDescriptionHelper.attributedDescriptionToHtml;
+import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getImagesFromThumbnailsArray;
+import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
+
+/**
+ * A {@link CommentsInfoItemExtractor} for YouTube comment data returned in a view model and entity
+ * updates.
+ */
+class YoutubeCommentsEUVMInfoItemExtractor implements CommentsInfoItemExtractor {
+
+    private static final String AUTHOR = "author";
+    private static final String PROPERTIES = "properties";
+
+    @Nonnull
+    private final JsonObject commentViewModel;
+    @Nullable
+    private final JsonObject commentRepliesRenderer;
+    @Nonnull
+    private final JsonObject commentEntityPayload;
+    @Nonnull
+    private final JsonObject engagementToolbarStateEntityPayload;
+    @Nonnull
+    private final String videoUrl;
+    @Nonnull
+    private final TimeAgoParser timeAgoParser;
+
+    YoutubeCommentsEUVMInfoItemExtractor(
+            @Nonnull final JsonObject commentViewModel,
+            @Nullable final JsonObject commentRepliesRenderer,
+            @Nonnull final JsonObject commentEntityPayload,
+            @Nonnull final JsonObject engagementToolbarStateEntityPayload,
+            @Nonnull final String videoUrl,
+            @Nonnull final TimeAgoParser timeAgoParser) {
+        this.commentViewModel = commentViewModel;
+        this.commentRepliesRenderer = commentRepliesRenderer;
+        this.commentEntityPayload = commentEntityPayload;
+        this.engagementToolbarStateEntityPayload = engagementToolbarStateEntityPayload;
+        this.videoUrl = videoUrl;
+        this.timeAgoParser = timeAgoParser;
+    }
+
+    @Override
+    public String getName() throws ParsingException {
+        return getUploaderName();
+    }
+
+    @Override
+    public String getUrl() throws ParsingException {
+        return videoUrl;
+    }
+
+    @Nonnull
+    @Override
+    public List<Image> getThumbnails() throws ParsingException {
+        return getUploaderAvatars();
+    }
+
+    @Override
+    public int getLikeCount() throws ParsingException {
+        final String textualLikeCount = getTextualLikeCount();
+        try {
+            if (Utils.isBlank(textualLikeCount)) {
+                return 0;
+            }
+
+            return (int) Utils.mixedNumberWordToLong(textualLikeCount);
+        } catch (final Exception e) {
+            throw new ParsingException(
+                    "Unexpected error while converting textual like count to like count", e);
+        }
+    }
+
+    @Override
+    public String getTextualLikeCount() {
+        return commentEntityPayload.getObject("toolbar")
+                .getString("likeCountNotliked");
+    }
+
+    @Override
+    public Description getCommentText() throws ParsingException {
+        // Comments' text work in the same way as an attributed video description
+        return new Description(
+                attributedDescriptionToHtml(commentEntityPayload.getObject(PROPERTIES)
+                        .getObject("content")), Description.HTML);
+    }
+
+    @Override
+    public String getTextualUploadDate() throws ParsingException {
+        return commentEntityPayload.getObject(PROPERTIES)
+                .getString("publishedTime");
+    }
+
+    @Nullable
+    @Override
+    public DateWrapper getUploadDate() throws ParsingException {
+        final String textualPublishedTime = getTextualUploadDate();
+        if (isNullOrEmpty(textualPublishedTime)) {
+            return null;
+        }
+
+        return timeAgoParser.parse(textualPublishedTime);
+    }
+
+    @Override
+    public String getCommentId() throws ParsingException {
+        String commentId = commentEntityPayload.getObject(PROPERTIES)
+                .getString("commentId");
+        if (isNullOrEmpty(commentId)) {
+            commentId = commentViewModel.getString("commentId");
+            if (isNullOrEmpty(commentId)) {
+                throw new ParsingException("Could not get comment ID");
+            }
+        }
+        return commentId;
+    }
+
+    @Override
+    public String getUploaderUrl() throws ParsingException {
+        final JsonObject author = commentEntityPayload.getObject(AUTHOR);
+        String channelId = author.getString("channelId");
+        if (isNullOrEmpty(channelId)) {
+            channelId = author.getObject("channelCommand")
+                    .getObject("innertubeCommand")
+                    .getObject("browseEndpoint")
+                    .getString("browseId");
+            if (isNullOrEmpty(channelId)) {
+                channelId = author.getObject("avatar")
+                        .getObject("endpoint")
+                        .getObject("innertubeCommand")
+                        .getObject("browseEndpoint")
+                        .getString("browseId");
+                if (isNullOrEmpty(channelId)) {
+                    throw new ParsingException("Could not get channel ID");
+                }
+            }
+        }
+        return "https://www.youtube.com/channel/" + channelId;
+    }
+
+    @Override
+    public String getUploaderName() throws ParsingException {
+        return commentEntityPayload.getObject(AUTHOR)
+                .getString("displayName");
+    }
+
+    @Nonnull
+    @Override
+    public List<Image> getUploaderAvatars() throws ParsingException {
+        return getImagesFromThumbnailsArray(commentEntityPayload.getObject("avatar")
+                .getObject("image")
+                .getArray("sources"));
+    }
+
+    @Override
+    public boolean isHeartedByUploader() {
+        return "TOOLBAR_HEART_STATE_HEARTED".equals(
+                engagementToolbarStateEntityPayload.getString("heartState"));
+    }
+
+    @Override
+    public boolean isPinned() {
+        return commentViewModel.has("pinnedText");
+    }
+
+    @Override
+    public boolean isUploaderVerified() throws ParsingException {
+        final JsonObject author = commentEntityPayload.getObject(AUTHOR);
+        return author.getBoolean("isVerified") || author.getBoolean("isArtist");
+    }
+
+    @Override
+    public int getReplyCount() throws ParsingException {
+        // As YouTube allows replies up to 750 comments, we cannot check if the count returned is a
+        // mixed number or a real number
+        // Assume it is a mixed one, as it matches how numbers of most properties are returned
+        final String replyCountString = commentEntityPayload.getObject("toolbar")
+                .getString("replyCount");
+        if (isNullOrEmpty(replyCountString)) {
+            return 0;
+        }
+        return (int) Utils.mixedNumberWordToLong(replyCountString);
+    }
+
+    @Nullable
+    @Override
+    public Page getReplies() throws ParsingException {
+        if (isNullOrEmpty(commentRepliesRenderer)) {
+            return null;
+        }
+
+        final String continuation = commentRepliesRenderer.getArray("contents")
+                .stream()
+                .filter(JsonObject.class::isInstance)
+                .map(JsonObject.class::cast)
+                .map(content -> content.getObject("continuationItemRenderer", null))
+                .filter(Objects::nonNull)
+                .findFirst()
+                .map(continuationItemRenderer ->
+                                continuationItemRenderer.getObject("continuationEndpoint")
+                                        .getObject("continuationCommand")
+                                        .getString("token"))
+                .orElseThrow(() ->
+                        new ParsingException("Could not get comment replies continuation"));
+        return new Page(videoUrl, continuation);
+    }
+
+    @Override
+    public boolean isChannelOwner() {
+        return commentEntityPayload.getObject(AUTHOR)
+                .getBoolean("isCreator");
+    }
+
+    @Override
+    public boolean hasCreatorReply() {
+        return commentRepliesRenderer != null
+                && commentRepliesRenderer.has("viewRepliesCreatorThumbnail");
+    }
+}

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsExtractor.java

@@ -13,6 +13,7 @@ import org.schabi.newpipe.extractor.exceptions.ExtractionException;
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
 import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
 import org.schabi.newpipe.extractor.localization.Localization;
+import org.schabi.newpipe.extractor.localization.TimeAgoParser;
 import org.schabi.newpipe.extractor.utils.JsonUtils;
 import org.schabi.newpipe.extractor.utils.Utils;
 
@@ -21,7 +22,6 @@ import javax.annotation.Nullable;
 import java.io.IOException;
 import java.nio.charset.StandardCharsets;
 import java.util.Collections;
-import java.util.List;
 
 import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonPostResponse;
 import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject;
@@ -30,6 +30,9 @@ import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
 
 public class YoutubeCommentsExtractor extends CommentsExtractor {
 
+    private static final String COMMENT_VIEW_MODEL_KEY = "commentViewModel";
+    private static final String COMMENT_RENDERER_KEY = "commentRenderer";
+
     /**
      * Whether comments are disabled on video.
      */
@@ -74,8 +77,7 @@ public class YoutubeCommentsExtractor extends CommentsExtractor {
             return null;
         }
 
-        final String token = contents
-                .stream()
+        final String token = contents.stream()
                 // Only use JsonObjects
                 .filter(JsonObject.class::isInstance)
                 .map(JsonObject.class::cast)
@@ -120,6 +122,21 @@ public class YoutubeCommentsExtractor extends CommentsExtractor {
         }
     }
 
+    @Nonnull
+    private JsonObject getMutationPayloadFromEntityKey(@Nonnull final JsonArray mutations,
+                                                       @Nonnull final String commentKey)
+            throws ParsingException {
+        return mutations.stream()
+                .filter(JsonObject.class::isInstance)
+                .map(JsonObject.class::cast)
+                .filter(mutation -> commentKey.equals(
+                        mutation.getString("entityKey")))
+                .findFirst()
+                .orElseThrow(() -> new ParsingException(
+                        "Could not get comment entity payload mutation"))
+                .getObject("payload");
+    }
+
     @Nonnull
     private InfoItemsPage<CommentsInfoItem> getInfoItemsPageForDisabledComments() {
         return new InfoItemsPage<>(Collections.emptyList(), null, Collections.emptyList());
@@ -207,8 +224,8 @@ public class YoutubeCommentsExtractor extends CommentsExtractor {
         return new InfoItemsPage<>(collector, getNextPage(jsonObject));
     }
 
-    private void collectCommentsFrom(final CommentsInfoItemsCollector collector,
-                                     final JsonObject jsonObject)
+    private void collectCommentsFrom(@Nonnull final CommentsInfoItemsCollector collector,
+                                     @Nonnull final JsonObject jsonObject)
             throws ParsingException {
 
         final JsonArray onResponseReceivedEndpoints =
@@ -233,6 +250,8 @@ public class YoutubeCommentsExtractor extends CommentsExtractor {
 
         final JsonArray contents;
         try {
+            // A copy of the array is needed, otherwise the continuation item is removed from the
+            // original object which is used to get the continuation
             contents = new JsonArray(JsonUtils.getArray(commentsEndpoint, path));
         } catch (final Exception e) {
             // No comments
@@ -244,23 +263,80 @@ public class YoutubeCommentsExtractor extends CommentsExtractor {
             contents.remove(index);
         }
 
-        final String jsonKey = contents.getObject(0).has("commentThreadRenderer")
-                ? "commentThreadRenderer"
-                : "commentRenderer";
+        // The mutations object, which is returned in the comments' continuation
+        // It contains parts of comment data when comments are returned with a view model
+        final JsonArray mutations = jsonObject.getObject("frameworkUpdates")
+                .getObject("entityBatchUpdate")
+                .getArray("mutations");
+        final String videoUrl = getUrl();
+        final TimeAgoParser timeAgoParser = getTimeAgoParser();
 
-        final List<Object> comments;
-        try {
-            comments = JsonUtils.getValues(contents, jsonKey);
-        } catch (final Exception e) {
-            throw new ParsingException("Unable to get parse youtube comments", e);
+        for (final Object o : contents) {
+            if (!(o instanceof JsonObject)) {
+                continue;
+            }
+
+            collectCommentItem(mutations, (JsonObject) o, collector, videoUrl, timeAgoParser);
         }
+    }
 
-        final String url = getUrl();
-        comments.stream()
-                .filter(JsonObject.class::isInstance)
-                .map(JsonObject.class::cast)
-                .map(jObj -> new YoutubeCommentsInfoItemExtractor(jObj, url, getTimeAgoParser()))
-                .forEach(collector::commit);
+    private void collectCommentItem(@Nonnull final JsonArray mutations,
+                                    @Nonnull final JsonObject content,
+                                    @Nonnull final CommentsInfoItemsCollector collector,
+                                    @Nonnull final String videoUrl,
+                                    @Nonnull final TimeAgoParser timeAgoParser)
+            throws ParsingException {
+        if (content.has("commentThreadRenderer")) {
+            final JsonObject commentThreadRenderer =
+                    content.getObject("commentThreadRenderer");
+            if (commentThreadRenderer.has(COMMENT_VIEW_MODEL_KEY)) {
+                final JsonObject commentViewModel =
+                        commentThreadRenderer.getObject(COMMENT_VIEW_MODEL_KEY)
+                                .getObject(COMMENT_VIEW_MODEL_KEY);
+                collector.commit(new YoutubeCommentsEUVMInfoItemExtractor(
+                        commentViewModel,
+                        commentThreadRenderer.getObject("replies")
+                                .getObject("commentRepliesRenderer"),
+                        getMutationPayloadFromEntityKey(mutations,
+                                commentViewModel.getString("commentKey", ""))
+                                .getObject("commentEntityPayload"),
+                        getMutationPayloadFromEntityKey(mutations,
+                                commentViewModel.getString("toolbarStateKey", ""))
+                                .getObject("engagementToolbarStateEntityPayload"),
+                        videoUrl,
+                        timeAgoParser));
+            } else if (commentThreadRenderer.has("comment")) {
+                collector.commit(new YoutubeCommentsInfoItemExtractor(
+                        commentThreadRenderer.getObject("comment")
+                                .getObject(COMMENT_RENDERER_KEY),
+                        commentThreadRenderer.getObject("replies")
+                                .getObject("commentRepliesRenderer"),
+                        videoUrl,
+                        timeAgoParser));
+            }
+        } else if (content.has(COMMENT_VIEW_MODEL_KEY)) {
+            final JsonObject commentViewModel = content.getObject(COMMENT_VIEW_MODEL_KEY);
+            collector.commit(new YoutubeCommentsEUVMInfoItemExtractor(
+                    commentViewModel,
+                    null,
+                    getMutationPayloadFromEntityKey(mutations,
+                            commentViewModel.getString("commentKey", ""))
+                            .getObject("commentEntityPayload"),
+                    getMutationPayloadFromEntityKey(mutations,
+                            commentViewModel.getString("toolbarStateKey", ""))
+                            .getObject("engagementToolbarStateEntityPayload"),
+                    videoUrl,
+                    timeAgoParser));
+        } else if (content.has(COMMENT_RENDERER_KEY)) {
+            // commentRenderers are directly returned for comment replies, so there is no
+            // commentRepliesRenderer to provide
+            // Also, YouTube has only one comment reply level
+            collector.commit(new YoutubeCommentsInfoItemExtractor(
+                    content.getObject(COMMENT_RENDERER_KEY),
+                    null,
+                    videoUrl,
+                    timeAgoParser));
+        }
     }
 
     @Override
@@ -307,10 +383,11 @@ public class YoutubeCommentsExtractor extends CommentsExtractor {
             return -1;
         }
 
-        final JsonObject countText = ajaxJson
-                .getArray("onResponseReceivedEndpoints").getObject(0)
+        final JsonObject countText = ajaxJson.getArray("onResponseReceivedEndpoints")
+                .getObject(0)
                 .getObject("reloadContinuationItemsCommand")
-                .getArray("continuationItems").getObject(0)
+                .getArray("continuationItems")
+                .getObject(0)
                 .getObject("commentsHeaderRenderer")
                 .getObject("countText");
 

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsInfoItemExtractor.java

@@ -22,40 +22,36 @@ import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper
 
 public class YoutubeCommentsInfoItemExtractor implements CommentsInfoItemExtractor {
 
-    private final JsonObject json;
-    private JsonObject commentRenderer;
+    @Nonnull
+    private final JsonObject commentRenderer;
+    @Nullable
+    private final JsonObject commentRepliesRenderer;
+    @Nonnull
     private final String url;
+    @Nonnull
     private final TimeAgoParser timeAgoParser;
 
-    public YoutubeCommentsInfoItemExtractor(final JsonObject json,
-                                            final String url,
-                                            final TimeAgoParser timeAgoParser) {
-        this.json = json;
+    public YoutubeCommentsInfoItemExtractor(@Nonnull final JsonObject commentRenderer,
+                                            @Nullable final JsonObject commentRepliesRenderer,
+                                            @Nonnull final String url,
+                                            @Nonnull final TimeAgoParser timeAgoParser) {
+        this.commentRenderer = commentRenderer;
+        this.commentRepliesRenderer = commentRepliesRenderer;
         this.url = url;
         this.timeAgoParser = timeAgoParser;
     }
 
-    private JsonObject getCommentRenderer() throws ParsingException {
-        if (commentRenderer == null) {
-            if (json.has("comment")) {
-                commentRenderer = JsonUtils.getObject(json, "comment.commentRenderer");
-            } else {
-                commentRenderer = json;
-            }
-        }
-        return commentRenderer;
-    }
-
     @Nonnull
     private List<Image> getAuthorThumbnails() throws ParsingException {
         try {
-            return getImagesFromThumbnailsArray(JsonUtils.getArray(getCommentRenderer(),
+            return getImagesFromThumbnailsArray(JsonUtils.getArray(commentRenderer,
                     "authorThumbnail.thumbnails"));
         } catch (final Exception e) {
             throw new ParsingException("Could not get author thumbnails", e);
         }
     }
 
+    @Nonnull
     @Override
     public String getUrl() throws ParsingException {
         return url;
@@ -70,7 +66,7 @@ public class YoutubeCommentsInfoItemExtractor implements CommentsInfoItemExtract
     @Override
     public String getName() throws ParsingException {
         try {
-            return getTextFromObject(JsonUtils.getObject(getCommentRenderer(), "authorText"));
+            return getTextFromObject(JsonUtils.getObject(commentRenderer, "authorText"));
         } catch (final Exception e) {
             return "";
         }
@@ -79,7 +75,7 @@ public class YoutubeCommentsInfoItemExtractor implements CommentsInfoItemExtract
     @Override
     public String getTextualUploadDate() throws ParsingException {
         try {
-            return getTextFromObject(JsonUtils.getObject(getCommentRenderer(),
+            return getTextFromObject(JsonUtils.getObject(commentRenderer,
                     "publishedTimeText"));
         } catch (final Exception e) {
             throw new ParsingException("Could not get publishedTimeText", e);
@@ -90,8 +86,7 @@ public class YoutubeCommentsInfoItemExtractor implements CommentsInfoItemExtract
     @Override
     public DateWrapper getUploadDate() throws ParsingException {
         final String textualPublishedTime = getTextualUploadDate();
-        if (timeAgoParser != null && textualPublishedTime != null
-                && !textualPublishedTime.isEmpty()) {
+        if (textualPublishedTime != null && !textualPublishedTime.isEmpty()) {
             return timeAgoParser.parse(textualPublishedTime);
         } else {
             return null;
@@ -118,7 +113,7 @@ public class YoutubeCommentsInfoItemExtractor implements CommentsInfoItemExtract
         // Try first to get the exact like count by using the accessibility data
         final String likeCount;
         try {
-            likeCount = Utils.removeNonDigitCharacters(JsonUtils.getString(getCommentRenderer(),
+            likeCount = Utils.removeNonDigitCharacters(JsonUtils.getString(commentRenderer,
                     "actionButtons.commentActionButtonsRenderer.likeButton.toggleButtonRenderer"
                             + ".accessibilityData.accessibilityData.label"));
         } catch (final Exception e) {
@@ -170,11 +165,11 @@ public class YoutubeCommentsInfoItemExtractor implements CommentsInfoItemExtract
          */
         try {
             // If a comment has no likes voteCount is not set
-            if (!getCommentRenderer().has("voteCount")) {
+            if (!commentRenderer.has("voteCount")) {
                 return "";
             }
 
-            final JsonObject voteCountObj = JsonUtils.getObject(getCommentRenderer(), "voteCount");
+            final JsonObject voteCountObj = JsonUtils.getObject(commentRenderer, "voteCount");
             if (voteCountObj.isEmpty()) {
                 return "";
             }
@@ -184,10 +179,11 @@ public class YoutubeCommentsInfoItemExtractor implements CommentsInfoItemExtract
         }
     }
 
+    @Nonnull
     @Override
     public Description getCommentText() throws ParsingException {
         try {
-            final JsonObject contentText = JsonUtils.getObject(getCommentRenderer(), "contentText");
+            final JsonObject contentText = JsonUtils.getObject(commentRenderer, "contentText");
             if (contentText.isEmpty()) {
                 // completely empty comments as described in
                 // https://github.com/TeamNewPipe/NewPipeExtractor/issues/380#issuecomment-668808584
@@ -207,7 +203,7 @@ public class YoutubeCommentsInfoItemExtractor implements CommentsInfoItemExtract
     @Override
     public String getCommentId() throws ParsingException {
         try {
-            return JsonUtils.getString(getCommentRenderer(), "commentId");
+            return JsonUtils.getString(commentRenderer, "commentId");
         } catch (final Exception e) {
             throw new ParsingException("Could not get comment id", e);
         }
@@ -220,27 +216,26 @@ public class YoutubeCommentsInfoItemExtractor implements CommentsInfoItemExtract
     }
 
     @Override
-    public boolean isHeartedByUploader() throws ParsingException {
-        final JsonObject commentActionButtonsRenderer = getCommentRenderer()
-                .getObject("actionButtons")
+    public boolean isHeartedByUploader() {
+        final JsonObject commentActionButtonsRenderer = commentRenderer.getObject("actionButtons")
                 .getObject("commentActionButtonsRenderer");
         return commentActionButtonsRenderer.has("creatorHeart");
     }
 
     @Override
-    public boolean isPinned() throws ParsingException {
-        return getCommentRenderer().has("pinnedCommentBadge");
+    public boolean isPinned() {
+        return commentRenderer.has("pinnedCommentBadge");
     }
 
     @Override
     public boolean isUploaderVerified() throws ParsingException {
-        return getCommentRenderer().has("authorCommentBadge");
+        return commentRenderer.has("authorCommentBadge");
     }
 
     @Override
     public String getUploaderName() throws ParsingException {
         try {
-            return getTextFromObject(JsonUtils.getObject(getCommentRenderer(), "authorText"));
+            return getTextFromObject(JsonUtils.getObject(commentRenderer, "authorText"));
         } catch (final Exception e) {
             return "";
         }
@@ -249,7 +244,7 @@ public class YoutubeCommentsInfoItemExtractor implements CommentsInfoItemExtract
     @Override
     public String getUploaderUrl() throws ParsingException {
         try {
-            return "https://www.youtube.com/channel/" + JsonUtils.getString(getCommentRenderer(),
+            return "https://www.youtube.com/channel/" + JsonUtils.getString(commentRenderer,
                     "authorEndpoint.browseEndpoint.browseId");
         } catch (final Exception e) {
             return "";
@@ -257,19 +252,22 @@ public class YoutubeCommentsInfoItemExtractor implements CommentsInfoItemExtract
     }
 
     @Override
-    public int getReplyCount() throws ParsingException {
-        final JsonObject commentRendererJsonObject = getCommentRenderer();
-        if (commentRendererJsonObject.has("replyCount")) {
-            return commentRendererJsonObject.getInt("replyCount");
+    public int getReplyCount() {
+        if (commentRenderer.has("replyCount")) {
+            return commentRenderer.getInt("replyCount");
         }
         return UNKNOWN_REPLY_COUNT;
     }
 
     @Override
     public Page getReplies() {
+        if (commentRepliesRenderer == null) {
+            return null;
+        }
+
         try {
             final String id = JsonUtils.getString(
-                    JsonUtils.getArray(json, "replies.commentRepliesRenderer.contents")
+                    JsonUtils.getArray(commentRepliesRenderer, "contents")
                             .getObject(0),
                     "continuationItemRenderer.continuationEndpoint.continuationCommand.token");
             return new Page(url, id);
@@ -279,20 +277,17 @@ public class YoutubeCommentsInfoItemExtractor implements CommentsInfoItemExtract
     }
 
     @Override
-    public boolean isChannelOwner() throws ParsingException {
-        return getCommentRenderer().getBoolean("authorIsChannelOwner");
+    public boolean isChannelOwner() {
+        return commentRenderer.getBoolean("authorIsChannelOwner");
     }
 
-
     @Override
-    public boolean hasCreatorReply() throws ParsingException {
-        try {
-            final JsonObject commentRepliesRenderer = JsonUtils.getObject(json,
-                    "replies.commentRepliesRenderer");
-            return commentRepliesRenderer.has("viewRepliesCreatorThumbnail");
-        } catch (final Exception e) {
+    public boolean hasCreatorReply() {
+        if (commentRepliesRenderer == null) {
             return false;
         }
+
+        return commentRepliesRenderer.has("viewRepliesCreatorThumbnail");
     }
 
 }

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixOrPlaylistLockupInfoItemExtractor.java

@@ -0,0 +1,193 @@
+package org.schabi.newpipe.extractor.services.youtube.extractors;
+
+import com.grack.nanojson.JsonObject;
+import org.schabi.newpipe.extractor.Image;
+import org.schabi.newpipe.extractor.ListExtractor;
+import org.schabi.newpipe.extractor.exceptions.ParsingException;
+import org.schabi.newpipe.extractor.playlist.PlaylistInfo;
+import org.schabi.newpipe.extractor.playlist.PlaylistInfoItemExtractor;
+import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubePlaylistLinkHandlerFactory;
+import org.schabi.newpipe.extractor.utils.Utils;
+
+import javax.annotation.Nonnull;
+import java.util.List;
+
+import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.extractPlaylistTypeFromPlaylistId;
+import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getImagesFromThumbnailsArray;
+import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getUrlFromNavigationEndpoint;
+import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.hasArtistOrVerifiedIconBadgeAttachment;
+
+public class YoutubeMixOrPlaylistLockupInfoItemExtractor implements PlaylistInfoItemExtractor {
+
+    @Nonnull
+    private final JsonObject lockupViewModel;
+    @Nonnull
+    private final JsonObject thumbnailViewModel;
+    @Nonnull
+    private final JsonObject lockupMetadataViewModel;
+    @Nonnull
+    private final JsonObject firstMetadataRow;
+    @Nonnull
+    private PlaylistInfo.PlaylistType playlistType;
+
+    public YoutubeMixOrPlaylistLockupInfoItemExtractor(@Nonnull final JsonObject lockupViewModel) {
+        this.lockupViewModel = lockupViewModel;
+        this.thumbnailViewModel = lockupViewModel.getObject("contentImage")
+                .getObject("collectionThumbnailViewModel")
+                .getObject("primaryThumbnail")
+                .getObject("thumbnailViewModel");
+        this.lockupMetadataViewModel = lockupViewModel.getObject("metadata")
+                .getObject("lockupMetadataViewModel");
+        /*
+        The metadata rows are structured in the following way:
+        1st part: uploader info, playlist type, playlist updated date
+        2nd part: space row
+        3rd element: first video
+        4th (not always returned for playlists with less than 2 items?): second video
+        5th element (always returned, but at a different index for playlists with less than 2
+        items?): Show full playlist
+
+        The first metadata row has the following structure:
+        1st array element: uploader info
+        2nd element: playlist type (course, playlist, podcast)
+        3rd element (not always returned): playlist updated date
+         */
+        this.firstMetadataRow = lockupMetadataViewModel.getObject("metadata")
+                .getObject("contentMetadataViewModel")
+                .getArray("metadataRows")
+                .getObject(0);
+
+        try {
+            this.playlistType = extractPlaylistTypeFromPlaylistId(getPlaylistId());
+        } catch (final ParsingException e) {
+            // If we cannot extract the playlist type, fall back to the normal one
+            this.playlistType = PlaylistInfo.PlaylistType.NORMAL;
+        }
+    }
+
+    @Override
+    public String getUploaderName() throws ParsingException {
+        return firstMetadataRow.getArray("metadataParts")
+                .getObject(0)
+                .getObject("text")
+                .getString("content");
+    }
+
+    @Override
+    public String getUploaderUrl() throws ParsingException {
+        if (playlistType != PlaylistInfo.PlaylistType.NORMAL) {
+            // If the playlist is a mix, there is no uploader as they are auto-generated
+            return null;
+        }
+
+        return getUrlFromNavigationEndpoint(
+                firstMetadataRow.getArray("metadataParts")
+                        .getObject(0)
+                        .getObject("text")
+                        .getArray("commandRuns")
+                        .getObject(0)
+                        .getObject("onTap")
+                        .getObject("innertubeCommand"));
+    }
+
+    @Override
+    public boolean isUploaderVerified() throws ParsingException {
+        if (playlistType != PlaylistInfo.PlaylistType.NORMAL) {
+            // If the playlist is a mix, there is no uploader as they are auto-generated
+            return false;
+        }
+
+        return hasArtistOrVerifiedIconBadgeAttachment(
+                firstMetadataRow.getArray("metadataParts")
+                        .getObject(0)
+                        .getObject("text")
+                        .getArray("attachmentRuns"));
+    }
+
+    @Override
+    public long getStreamCount() throws ParsingException {
+        if (playlistType != PlaylistInfo.PlaylistType.NORMAL) {
+            // If the playlist is a mix, we are not able to get its stream count
+            return ListExtractor.ITEM_COUNT_INFINITE;
+        }
+
+        try {
+            return Long.parseLong(Utils.removeNonDigitCharacters(
+                    thumbnailViewModel.getArray("overlays")
+                            .stream()
+                            .filter(JsonObject.class::isInstance)
+                            .map(JsonObject.class::cast)
+                            .filter(overlay -> overlay.has("thumbnailOverlayBadgeViewModel"))
+                            .findFirst()
+                            .orElseThrow(() -> new ParsingException(
+                                    "Could not get thumbnailOverlayBadgeViewModel"))
+                            .getObject("thumbnailOverlayBadgeViewModel")
+                            .getArray("thumbnailBadges")
+                            .stream()
+                            .filter(JsonObject.class::isInstance)
+                            .map(JsonObject.class::cast)
+                            .filter(badge -> badge.has("thumbnailBadgeViewModel"))
+                            .findFirst()
+                            .orElseThrow(() ->
+                                    new ParsingException("Could not get thumbnailBadgeViewModel"))
+                            .getObject("thumbnailBadgeViewModel")
+                            .getString("text")));
+        } catch (final Exception e) {
+            throw new ParsingException("Could not get playlist stream count", e);
+        }
+    }
+
+    @Override
+    public String getName() throws ParsingException {
+        return lockupMetadataViewModel.getObject("title")
+                .getString("content");
+    }
+
+    @Override
+    public String getUrl() throws ParsingException {
+        // If the playlist item is a mix, we cannot return just its playlist ID as mix playlists
+        // are not viewable in playlist pages
+        // Use directly getUrlFromNavigationEndpoint in this case, which returns the watch URL with
+        // the mix playlist
+        if (playlistType == PlaylistInfo.PlaylistType.NORMAL) {
+            try {
+                return YoutubePlaylistLinkHandlerFactory.getInstance().getUrl(getPlaylistId());
+            } catch (final Exception ignored) {
+            }
+        }
+
+        return getUrlFromNavigationEndpoint(lockupViewModel.getObject("rendererContext")
+                .getObject("commandContext")
+                .getObject("onTap")
+                .getObject("innertubeCommand"));
+    }
+
+    @Nonnull
+    @Override
+    public List<Image> getThumbnails() throws ParsingException {
+        return getImagesFromThumbnailsArray(thumbnailViewModel.getObject("image")
+                .getArray("sources"));
+    }
+
+    @Nonnull
+    @Override
+    public PlaylistInfo.PlaylistType getPlaylistType() throws ParsingException {
+        return playlistType;
+    }
+
+    private String getPlaylistId() throws ParsingException {
+        String id = lockupViewModel.getString("contentId");
+        if (Utils.isNullOrEmpty(id)) {
+            id = lockupViewModel.getObject("rendererContext")
+                    .getObject("commandContext")
+                    .getObject("watchEndpoint")
+                    .getString("playlistId");
+        }
+
+        if (Utils.isNullOrEmpty(id)) {
+            throw new ParsingException("Could not get playlist ID");
+        }
+
+        return id;
+    }
+}

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java

@@ -4,7 +4,6 @@ import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper
 import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.YOUTUBEI_V1_URL;
 import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.extractCookieValue;
 import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.extractPlaylistTypeFromPlaylistId;
-import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getKey;
 import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getValidJsonResponseBody;
 import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getYouTubeHeaders;
 import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareDesktopJsonBuilder;
@@ -103,8 +102,8 @@ public class YoutubeMixPlaylistExtractor extends PlaylistExtractor {
         final var headers = getYouTubeHeaders();
 
         final Response response = getDownloader().postWithContentTypeJson(
-                YOUTUBEI_V1_URL + "next?key=" + getKey() + DISABLE_PRETTY_PRINT_PARAMETER,
-                headers, body, localization);
+                YOUTUBEI_V1_URL + "next?" + DISABLE_PRETTY_PRINT_PARAMETER, headers, body,
+                localization);
 
         initialData = JsonUtils.toJsonObject(getValidJsonResponseBody(response));
         playlistData = initialData
@@ -225,7 +224,8 @@ public class YoutubeMixPlaylistExtractor extends PlaylistExtractor {
                 .done())
                 .getBytes(StandardCharsets.UTF_8);
 
-        return new Page(YOUTUBEI_V1_URL + "next?key=" + getKey(), null, null, cookies, body);
+        return new Page(YOUTUBEI_V1_URL + "next?" + DISABLE_PRETTY_PRINT_PARAMETER, null, null,
+                cookies, body);
     }
 
     @Override

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSearchExtractor.java

@@ -3,6 +3,7 @@ package org.schabi.newpipe.extractor.services.youtube.extractors;
 import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.DISABLE_PRETTY_PRINT_PARAMETER;
 import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject;
 import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getValidJsonResponseBody;
+import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getYoutubeMusicClientVersion;
 import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getYoutubeMusicHeaders;
 import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.MUSIC_ALBUMS;
 import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.MUSIC_ARTISTS;
@@ -25,10 +26,8 @@ import org.schabi.newpipe.extractor.StreamingService;
 import org.schabi.newpipe.extractor.downloader.Downloader;
 import org.schabi.newpipe.extractor.exceptions.ExtractionException;
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
-import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
 import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandler;
 import org.schabi.newpipe.extractor.search.SearchExtractor;
-import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper;
 import org.schabi.newpipe.extractor.utils.JsonUtils;
 
 import java.io.IOException;
@@ -52,10 +51,8 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor {
     @Override
     public void onFetchPage(@Nonnull final Downloader downloader)
             throws IOException, ExtractionException {
-        final String[] youtubeMusicKeys = YoutubeParsingHelper.getYoutubeMusicKey();
-
-        final String url = "https://music.youtube.com/youtubei/v1/search?key="
-                + youtubeMusicKeys[0] + DISABLE_PRETTY_PRINT_PARAMETER;
+        final String url = "https://music.youtube.com/youtubei/v1/search?"
+                + DISABLE_PRETTY_PRINT_PARAMETER;
 
         final String params;
 
@@ -86,7 +83,7 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor {
                 .object("context")
                     .object("client")
                         .value("clientName", "WEB_REMIX")
-                        .value("clientVersion", youtubeMusicKeys[2])
+                        .value("clientVersion", getYoutubeMusicClientVersion())
                         .value("hl", "en-GB")
                         .value("gl", getExtractorContentCountry().getCountryCode())
                         .value("platform", "DESKTOP")
@@ -206,15 +203,13 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor {
 
         final MultiInfoItemsCollector collector = new MultiInfoItemsCollector(getServiceId());
 
-        final String[] youtubeMusicKeys = YoutubeParsingHelper.getYoutubeMusicKey();
-
         // @formatter:off
         final byte[] json = JsonWriter.string()
             .object()
                 .object("context")
                     .object("client")
                         .value("clientName", "WEB_REMIX")
-                        .value("clientVersion", youtubeMusicKeys[2])
+                        .value("clientVersion", getYoutubeMusicClientVersion())
                         .value("hl", "en-GB")
                         .value("gl", getExtractorContentCountry().getCountryCode())
                         .value("platform", "DESKTOP")
@@ -295,8 +290,7 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor {
     }
 
     @Nullable
-    private Page getNextPageFrom(final JsonArray continuations)
-            throws IOException, ParsingException, ReCaptchaException {
+    private Page getNextPageFrom(final JsonArray continuations) {
         if (isNullOrEmpty(continuations)) {
             return null;
         }
@@ -306,7 +300,6 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor {
         final String continuation = nextContinuationData.getString("continuation");
 
         return new Page("https://music.youtube.com/youtubei/v1/search?ctoken=" + continuation
-                + "&continuation=" + continuation + "&key="
-                + YoutubeParsingHelper.getYoutubeMusicKey()[0] + DISABLE_PRETTY_PRINT_PARAMETER);
+                + "&continuation=" + continuation + "&" + DISABLE_PRETTY_PRINT_PARAMETER);
     }
 }

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java

@@ -4,7 +4,6 @@ import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper
 import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.YOUTUBEI_V1_URL;
 import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.extractPlaylistTypeFromPlaylistUrl;
 import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonPostResponse;
-import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getKey;
 import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject;
 import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getImagesFromThumbnailsArray;
 import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getUrlFromNavigationEndpoint;
@@ -387,8 +386,7 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
                             .done())
                     .getBytes(StandardCharsets.UTF_8);
 
-            return new Page(YOUTUBEI_V1_URL + "browse?key=" + getKey()
-                    + DISABLE_PRETTY_PRINT_PARAMETER, body);
+            return new Page(YOUTUBEI_V1_URL + "browse?" + DISABLE_PRETTY_PRINT_PARAMETER, body);
         } else {
             return null;
         }
@@ -411,8 +409,7 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
                                     richItemRenderer.getObject("content");
                             if (richItemRendererContent.has(REEL_ITEM_RENDERER)) {
                                 collector.commit(new YoutubeReelInfoItemExtractor(
-                                        richItemRendererContent.getObject(REEL_ITEM_RENDERER),
-                                        timeAgoParser));
+                                        richItemRendererContent.getObject(REEL_ITEM_RENDERER)));
                             }
                         }
                     }

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeReelInfoItemExtractor.java

@@ -1,45 +1,46 @@
 package org.schabi.newpipe.extractor.services.youtube.extractors;
 
+import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject;
+import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getThumbnailsFromInfoItem;
+import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
+
 import com.grack.nanojson.JsonObject;
 
 import org.schabi.newpipe.extractor.Image;
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
 import org.schabi.newpipe.extractor.localization.DateWrapper;
-import org.schabi.newpipe.extractor.localization.TimeAgoParser;
 import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeStreamLinkHandlerFactory;
 import org.schabi.newpipe.extractor.stream.StreamInfoItemExtractor;
 import org.schabi.newpipe.extractor.stream.StreamType;
 import org.schabi.newpipe.extractor.utils.Utils;
 
+import java.util.List;
+
 import javax.annotation.Nonnull;
 import javax.annotation.Nullable;
 
-import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject;
-import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getThumbnailsFromInfoItem;
-import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
-
-import java.util.List;
-
 /**
- * A {@link StreamInfoItemExtractor} for YouTube's {@code reelItemRenderers}.
+ * A {@link StreamInfoItemExtractor} for YouTube's {@code reelItemRenderer}s.
  *
  * <p>
- * {@code reelItemRenderers} are returned on YouTube for their short-form contents on almost every
+ * {@code reelItemRenderer}s were returned on YouTube for their short-form contents on almost every
  * place and every major client. They provide a limited amount of information and do not provide
  * the exact view count, any uploader info (name, URL, avatar, verified status) and the upload date.
  * </p>
+ *
+ * <p>
+ * At the time this documentation has been updated, they are being replaced by
+ * {@code shortsLockupViewModel}s. See {@link YoutubeShortsLockupInfoItemExtractor} for an
+ * extractor for this new UI data type.
+ * </p>
  */
 public class YoutubeReelInfoItemExtractor implements StreamInfoItemExtractor {
 
     @Nonnull
     private final JsonObject reelInfo;
-    @Nullable
-    private final TimeAgoParser timeAgoParser;
 
-    public YoutubeReelInfoItemExtractor(@Nonnull final JsonObject reelInfo,
-                                        @Nullable final TimeAgoParser timeAgoParser) {
+    public YoutubeReelInfoItemExtractor(@Nonnull final JsonObject reelInfo) {
         this.reelInfo = reelInfo;
-        this.timeAgoParser = timeAgoParser;
     }
 
     @Override
@@ -68,28 +69,6 @@ public class YoutubeReelInfoItemExtractor implements StreamInfoItemExtractor {
         return StreamType.VIDEO_STREAM;
     }
 
-    @Override
-    public long getDuration() throws ParsingException {
-        // Duration of reelItems is only provided in the accessibility data
-        // example: "VIDEO TITLE - 49 seconds - play video"
-        // "VIDEO TITLE - 1 minute, 1 second - play video"
-        final String accessibilityLabel = reelInfo.getObject("accessibility")
-                .getObject("accessibilityData").getString("label");
-        if (accessibilityLabel == null || timeAgoParser == null) {
-            return 0;
-        }
-
-        // This approach may be language dependent
-        final String[] labelParts = accessibilityLabel.split(" [\u2013-] ");
-
-        if (labelParts.length > 2) {
-            final String textualDuration = labelParts[labelParts.length - 2];
-            return timeAgoParser.parseDuration(textualDuration);
-        }
-
-        return -1;
-    }
-
     @Override
     public long getViewCount() throws ParsingException {
         final String viewCountText = getTextFromObject(reelInfo.getObject("viewCountText"));
@@ -117,6 +96,11 @@ public class YoutubeReelInfoItemExtractor implements StreamInfoItemExtractor {
         return false;
     }
 
+    @Override
+    public long getDuration() throws ParsingException {
+        return -1;
+    }
+
     @Override
     public String getUploaderName() throws ParsingException {
         return null;

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java

@@ -3,7 +3,6 @@ package org.schabi.newpipe.extractor.services.youtube.extractors;
 import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.DISABLE_PRETTY_PRINT_PARAMETER;
 import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.YOUTUBEI_V1_URL;
 import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonPostResponse;
-import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getKey;
 import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject;
 import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareDesktopJsonBuilder;
 import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.ALL;
@@ -30,7 +29,7 @@ import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandler;
 import org.schabi.newpipe.extractor.localization.Localization;
 import org.schabi.newpipe.extractor.localization.TimeAgoParser;
 import org.schabi.newpipe.extractor.search.SearchExtractor;
-import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper;
+import org.schabi.newpipe.extractor.services.youtube.YoutubeMetaInfoHelper;
 import org.schabi.newpipe.extractor.utils.JsonUtils;
 
 import java.io.IOException;
@@ -151,7 +150,7 @@ public class YoutubeSearchExtractor extends SearchExtractor {
     @Nonnull
     @Override
     public List<MetaInfo> getMetaInfo() throws ParsingException {
-        return YoutubeParsingHelper.getMetaInfo(
+        return YoutubeMetaInfoHelper.getMetaInfo(
                 initialData.getObject("contents")
                         .getObject("twoColumnSearchResultsRenderer")
                         .getObject("primaryContents")
@@ -239,16 +238,27 @@ public class YoutubeSearchExtractor extends SearchExtractor {
             } else if (extractChannelResults && item.has("channelRenderer")) {
                 collector.commit(new YoutubeChannelInfoItemExtractor(
                         item.getObject("channelRenderer")));
-            } else if (extractPlaylistResults && item.has("playlistRenderer")) {
-                collector.commit(new YoutubePlaylistInfoItemExtractor(
-                        item.getObject("playlistRenderer")));
+            } else if (extractPlaylistResults) {
+                if (item.has("playlistRenderer")) {
+                    collector.commit(new YoutubePlaylistInfoItemExtractor(
+                            item.getObject("playlistRenderer")));
+                } else if (item.has("showRenderer")) {
+                    collector.commit(new YoutubeShowRendererInfoItemExtractor(
+                            item.getObject("showRenderer")));
+                } else if (item.has("lockupViewModel")) {
+                    final JsonObject lockupViewModel = item.getObject("lockupViewModel");
+                    if ("LOCKUP_CONTENT_TYPE_PLAYLIST".equals(
+                            lockupViewModel.getString("contentType"))) {
+                        collector.commit(
+                                new YoutubeMixOrPlaylistLockupInfoItemExtractor(lockupViewModel));
+                    }
+                }
             }
         }
     }
 
     @Nullable
-    private Page getNextPageFrom(final JsonObject continuationItemRenderer) throws IOException,
-            ExtractionException {
+    private Page getNextPageFrom(final JsonObject continuationItemRenderer) {
         if (isNullOrEmpty(continuationItemRenderer)) {
             return null;
         }
@@ -257,8 +267,7 @@ public class YoutubeSearchExtractor extends SearchExtractor {
                 .getObject("continuationCommand")
                 .getString("token");
 
-        final String url = YOUTUBEI_V1_URL + "search?key=" + getKey()
-                + DISABLE_PRETTY_PRINT_PARAMETER;
+        final String url = YOUTUBEI_V1_URL + "search?" + DISABLE_PRETTY_PRINT_PARAMETER;
 
         return new Page(url, token);
     }

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeShortsLockupInfoItemExtractor.java

@@ -0,0 +1,150 @@
+package org.schabi.newpipe.extractor.services.youtube.extractors;
+
+import com.grack.nanojson.JsonObject;
+import org.schabi.newpipe.extractor.Image;
+import org.schabi.newpipe.extractor.exceptions.ParsingException;
+import org.schabi.newpipe.extractor.localization.DateWrapper;
+import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeStreamLinkHandlerFactory;
+import org.schabi.newpipe.extractor.stream.StreamInfoItemExtractor;
+import org.schabi.newpipe.extractor.stream.StreamType;
+import org.schabi.newpipe.extractor.utils.Utils;
+
+import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
+
+import java.util.List;
+
+import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getImagesFromThumbnailsArray;
+import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
+
+/**
+ * A {@link StreamInfoItemExtractor} for YouTube's {@code shortsLockupViewModel}s.
+ *
+ * <p>
+ * {@code shortsLockupViewModel}s are returned on YouTube for their short-form contents on almost
+ * every place and every major client. They provide a limited amount of information and do not
+ * provide the exact view count, any uploader info (name, URL, avatar, verified status) and the
+ * upload date.
+ * </p>
+ *
+ * <p>
+ * At the time this documentation has been written, this data UI type is not fully used (rolled
+ * out), so {@code reelItemRenderer}s are also returned. See {@link YoutubeReelInfoItemExtractor}
+ * for an extractor for this UI data type.
+ * </p>
+ */
+public class YoutubeShortsLockupInfoItemExtractor implements StreamInfoItemExtractor {
+
+    @Nonnull
+    private final JsonObject shortsLockupViewModel;
+
+    public YoutubeShortsLockupInfoItemExtractor(@Nonnull final JsonObject shortsLockupViewModel) {
+        this.shortsLockupViewModel = shortsLockupViewModel;
+    }
+
+    @Override
+    public String getName() throws ParsingException {
+        return shortsLockupViewModel.getObject("overlayMetadata")
+                .getObject("primaryText")
+                .getString("content");
+    }
+
+    @Override
+    public String getUrl() throws ParsingException {
+        String videoId = shortsLockupViewModel.getObject("onTap")
+                .getObject("innertubeCommand")
+                .getObject("reelWatchEndpoint")
+                .getString("videoId");
+
+        if (isNullOrEmpty(videoId)) {
+            videoId = shortsLockupViewModel.getObject("inlinePlayerData")
+                    .getObject("onVisible")
+                    .getObject("innertubeCommand")
+                    .getObject("watchEndpoint")
+                    .getString("videoId");
+        }
+
+        if (isNullOrEmpty(videoId)) {
+            throw new ParsingException("Could not get video ID");
+        }
+
+        try {
+            return YoutubeStreamLinkHandlerFactory.getInstance().getUrl(videoId);
+        } catch (final Exception e) {
+            throw new ParsingException("Could not get URL", e);
+        }
+    }
+
+    @Nonnull
+    @Override
+    public List<Image> getThumbnails() throws ParsingException {
+        return getImagesFromThumbnailsArray(shortsLockupViewModel.getObject("thumbnail")
+                .getArray("sources"));
+    }
+
+    @Override
+    public StreamType getStreamType() throws ParsingException {
+        return StreamType.VIDEO_STREAM;
+    }
+
+    @Override
+    public long getViewCount() throws ParsingException {
+        final String viewCountText = shortsLockupViewModel.getObject("overlayMetadata")
+                        .getObject("secondaryText")
+                        .getString("content");
+        if (!isNullOrEmpty(viewCountText)) {
+            // This approach is language dependent
+            if (viewCountText.toLowerCase().contains("no views")) {
+                return 0;
+            }
+
+            return Utils.mixedNumberWordToLong(viewCountText);
+        }
+
+        throw new ParsingException("Could not get short view count");
+    }
+
+    @Override
+    public boolean isShortFormContent() {
+        return true;
+    }
+
+    // All the following properties cannot be obtained from shortsLockupViewModels
+
+    @Override
+    public boolean isAd() throws ParsingException {
+        return false;
+    }
+
+    @Override
+    public long getDuration() throws ParsingException {
+        return -1;
+    }
+
+    @Override
+    public String getUploaderName() throws ParsingException {
+        return null;
+    }
+
+    @Override
+    public String getUploaderUrl() throws ParsingException {
+        return null;
+    }
+
+    @Override
+    public boolean isUploaderVerified() throws ParsingException {
+        return false;
+    }
+
+    @Nullable
+    @Override
+    public String getTextualUploadDate() throws ParsingException {
+        return null;
+    }
+
+    @Nullable
+    @Override
+    public DateWrapper getUploadDate() throws ParsingException {
+        return null;
+    }
+}

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeShowRendererInfoItemExtractor.java

@@ -0,0 +1,57 @@
+package org.schabi.newpipe.extractor.services.youtube.extractors;
+
+import com.grack.nanojson.JsonObject;
+import org.schabi.newpipe.extractor.exceptions.ParsingException;
+
+import javax.annotation.Nonnull;
+
+import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject;
+import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getUrlFromObject;
+import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
+
+/**
+ * A {@link YoutubeBaseShowInfoItemExtractor} implementation for {@code showRenderer}s.
+ */
+class YoutubeShowRendererInfoItemExtractor extends YoutubeBaseShowInfoItemExtractor {
+
+    @Nonnull
+    private final JsonObject shortBylineText;
+    @Nonnull
+    private final JsonObject longBylineText;
+
+    YoutubeShowRendererInfoItemExtractor(@Nonnull final JsonObject showRenderer) {
+        super(showRenderer);
+        this.shortBylineText = showRenderer.getObject("shortBylineText");
+        this.longBylineText = showRenderer.getObject("longBylineText");
+    }
+
+    @Override
+    public String getUploaderName() throws ParsingException {
+        String name = getTextFromObject(longBylineText);
+        if (isNullOrEmpty(name)) {
+            name = getTextFromObject(shortBylineText);
+            if (isNullOrEmpty(name)) {
+                throw new ParsingException("Could not get uploader name");
+            }
+        }
+        return name;
+    }
+
+    @Override
+    public String getUploaderUrl() throws ParsingException {
+        String uploaderUrl = getUrlFromObject(longBylineText);
+        if (uploaderUrl == null) {
+            uploaderUrl = getUrlFromObject(shortBylineText);
+            if (uploaderUrl == null) {
+                throw new ParsingException("Could not get uploader URL");
+            }
+        }
+        return uploaderUrl;
+    }
+
+    @Override
+    public boolean isUploaderVerified() throws ParsingException {
+        // We do not have this information in showRenderers
+        return false;
+    }
+}

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java

@@ -22,15 +22,15 @@ package org.schabi.newpipe.extractor.services.youtube.extractors;
 
 import static org.schabi.newpipe.extractor.services.youtube.ItagItem.APPROX_DURATION_MS_UNKNOWN;
 import static org.schabi.newpipe.extractor.services.youtube.ItagItem.CONTENT_LENGTH_UNKNOWN;
+import static org.schabi.newpipe.extractor.services.youtube.YoutubeDescriptionHelper.attributedDescriptionToHtml;
 import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.CONTENT_CHECK_OK;
 import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.CPN;
 import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.RACY_CHECK_OK;
 import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.VIDEO_ID;
-import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.createDesktopPlayerBody;
+import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.createTvHtml5EmbedPlayerBody;
 import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.fixThumbnailUrl;
 import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.generateContentPlaybackNonce;
 import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.generateTParameter;
-import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getAttributedDescription;
 import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getImagesFromThumbnailsArray;
 import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonAndroidPostResponse;
 import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonIosPostResponse;
@@ -67,6 +67,7 @@ import org.schabi.newpipe.extractor.localization.TimeAgoParser;
 import org.schabi.newpipe.extractor.localization.TimeAgoPatternsManager;
 import org.schabi.newpipe.extractor.services.youtube.ItagItem;
 import org.schabi.newpipe.extractor.services.youtube.YoutubeJavaScriptPlayerManager;
+import org.schabi.newpipe.extractor.services.youtube.YoutubeMetaInfoHelper;
 import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper;
 import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeChannelLinkHandlerFactory;
 import org.schabi.newpipe.extractor.stream.AudioStream;
@@ -95,7 +96,6 @@ import java.util.Arrays;
 import java.util.Collections;
 import java.util.List;
 import java.util.Locale;
-import java.util.Map;
 import java.util.Objects;
 import java.util.stream.Collectors;
 
@@ -103,22 +103,21 @@ import javax.annotation.Nonnull;
 import javax.annotation.Nullable;
 
 public class YoutubeStreamExtractor extends StreamExtractor {
-    private static boolean isAndroidClientFetchForced = false;
-    private static boolean isIosClientFetchForced = false;
 
     private JsonObject playerResponse;
     private JsonObject nextResponse;
 
     @Nullable
-    private JsonObject html5StreamingData;
+    private JsonObject iosStreamingData;
     @Nullable
     private JsonObject androidStreamingData;
     @Nullable
-    private JsonObject iosStreamingData;
+    private JsonObject tvHtml5SimplyEmbedStreamingData;
 
     private JsonObject videoPrimaryInfoRenderer;
     private JsonObject videoSecondaryInfoRenderer;
     private JsonObject playerMicroFormatRenderer;
+    private JsonObject playerCaptionsTracklistRenderer;
     private int ageLimit = -1;
     private StreamType streamType;
 
@@ -126,9 +125,9 @@ public class YoutubeStreamExtractor extends StreamExtractor {
     // URLs (with the cpn parameter).
     // Also because a nonce should be unique, it should be different between clients used, so
     // three different strings are used.
-    private String html5Cpn;
-    private String androidCpn;
     private String iosCpn;
+    private String androidCpn;
+    private String tvHtml5SimplyEmbedCpn;
 
     public YoutubeStreamExtractor(final StreamingService service, final LinkHandler linkHandler) {
         super(service, linkHandler);
@@ -189,7 +188,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
 
                 try { // Premiered 20 hours ago
                     final TimeAgoParser timeAgoParser = TimeAgoPatternsManager.getTimeAgoParserFor(
-                            Localization.fromLocalizationCode("en"));
+                            new Localization("en"));
                     final OffsetDateTime parsedTime = timeAgoParser.parse(time).offsetDateTime();
                     return DateTimeFormatter.ISO_LOCAL_DATE.format(parsedTime);
                 } catch (final Exception ignored) {
@@ -260,7 +259,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
             return new Description(videoSecondaryInfoRendererDescription, Description.HTML);
         }
 
-        final String attributedDescription = getAttributedDescription(
+        final String attributedDescription = attributedDescriptionToHtml(
                 getVideoSecondaryInfoRenderer().getObject("attributedDescription"));
         if (!isNullOrEmpty(attributedDescription)) {
             return new Description(attributedDescription, Description.HTML);
@@ -322,7 +321,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
             return Long.parseLong(duration);
         } catch (final Exception e) {
             return getDurationFromFirstAdaptiveFormat(Arrays.asList(
-                    html5StreamingData, androidStreamingData, iosStreamingData));
+                    iosStreamingData, androidStreamingData, tvHtml5SimplyEmbedStreamingData));
         }
     }
 
@@ -584,7 +583,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
         // Android client doesn't contain all available streams (mainly the WEBM ones)
         return getManifestUrl(
                 "dash",
-                Arrays.asList(html5StreamingData, androidStreamingData));
+                Arrays.asList(androidStreamingData, tvHtml5SimplyEmbedStreamingData));
     }
 
     @Nonnull
@@ -597,7 +596,8 @@ public class YoutubeStreamExtractor extends StreamExtractor {
         // Also, on videos, non-iOS clients don't have an HLS manifest URL in their player response
         return getManifestUrl(
                 "hls",
-                Arrays.asList(iosStreamingData, html5StreamingData, androidStreamingData));
+                Arrays.asList(
+                        iosStreamingData, androidStreamingData, tvHtml5SimplyEmbedStreamingData));
     }
 
     @Nonnull
@@ -634,28 +634,6 @@ public class YoutubeStreamExtractor extends StreamExtractor {
                 getVideoStreamBuilderHelper(true), "video-only");
     }
 
-    /**
-     * Try to deobfuscate a streaming URL and fall back to the given URL, because decryption may
-     * fail if YouTube changes break something.
-     *
-     * <p>
-     * This way a breaking change from YouTube does not result in a broken extractor.
-     * </p>
-     *
-     * @param streamingUrl the streaming URL to which deobfuscating its throttling parameter if
-     *                     there is one
-     * @param videoId      the video ID to use when extracting JavaScript player code, if needed
-     */
-    private String tryDeobfuscateThrottlingParameterOfUrl(@Nonnull final String streamingUrl,
-                                                          @Nonnull final String videoId) {
-        try {
-            return YoutubeJavaScriptPlayerManager.getUrlWithThrottlingParameterDeobfuscated(
-                    videoId, streamingUrl);
-        } catch (final ParsingException e) {
-            return streamingUrl;
-        }
-    }
-
     @Override
     @Nonnull
     public List<SubtitlesStream> getSubtitlesDefault() throws ParsingException {
@@ -669,9 +647,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
 
         // We cannot store the subtitles list because the media format may change
         final List<SubtitlesStream> subtitlesToReturn = new ArrayList<>();
-        final JsonObject renderer = playerResponse.getObject("captions")
-                .getObject("playerCaptionsTracklistRenderer");
-        final JsonArray captionsArray = renderer.getArray("captionTracks");
+        final JsonArray captionsArray = playerCaptionsTracklistRenderer.getArray("captionTracks");
         // TODO: use this to apply auto translation to different language from a source language
         // final JsonArray autoCaptionsArray = renderer.getArray("translationLanguages");
 
@@ -750,6 +726,13 @@ public class YoutubeStreamExtractor extends StreamExtractor {
                         } else if (result.has("compactPlaylistRenderer")) {
                             return new YoutubeMixOrPlaylistInfoItemExtractor(
                                     result.getObject("compactPlaylistRenderer"));
+                        } else if (result.has("lockupViewModel")) {
+                            final JsonObject lockupViewModel = result.getObject("lockupViewModel");
+                            if ("LOCKUP_CONTENT_TYPE_PLAYLIST".equals(
+                                    lockupViewModel.getString("contentType"))) {
+                                return new YoutubeMixOrPlaylistLockupInfoItemExtractor(
+                                        lockupViewModel);
+                            }
                         }
                         return null;
                     })
@@ -795,64 +778,70 @@ public class YoutubeStreamExtractor extends StreamExtractor {
 
         final Localization localization = getExtractorLocalization();
         final ContentCountry contentCountry = getExtractorContentCountry();
-        html5Cpn = generateContentPlaybackNonce();
 
-        playerResponse = getJsonPostResponse(PLAYER,
-                createDesktopPlayerBody(
-                        localization,
-                        contentCountry,
-                        videoId,
-                        YoutubeJavaScriptPlayerManager.getSignatureTimestamp(videoId),
-                        false,
-                        html5Cpn),
-                localization);
+        final JsonObject webPlayerResponse = YoutubeParsingHelper.getWebPlayerResponse(
+                localization, contentCountry, videoId);
 
-        // Save the playerResponse from the player endpoint of the desktop internal API because
-        // there can be restrictions on the embedded player.
-        // E.g. if a video is age-restricted, the embedded player's playabilityStatus says that
-        // the video cannot be played outside of YouTube, but does not show the original message.
-        final JsonObject youtubePlayerResponse = playerResponse;
-
-        if (playerResponse == null) {
-            throw new ExtractionException("Could not get playerResponse");
+        if (isPlayerResponseNotValid(webPlayerResponse, videoId)) {
+            // Check the playability status, as private and deleted videos and invalid video IDs do
+            // not return the ID provided in the player response
+            // When the requested video is playable and a different video ID is returned, it has
+            // the OK playability status, meaning the ExtractionException after this check will be
+            // thrown
+            checkPlayabilityStatus(
+                    webPlayerResponse, webPlayerResponse.getObject("playabilityStatus"));
+            throw new ExtractionException("Initial WEB player response is not valid");
         }
 
-        final JsonObject playabilityStatus = playerResponse.getObject("playabilityStatus");
+        // Save the webPlayerResponse into playerResponse in the case the video cannot be played,
+        // so some metadata can be retrieved
+        playerResponse = webPlayerResponse;
 
-        final boolean isAgeRestricted = playabilityStatus.getString("reason", "")
+        // Use the player response from the player endpoint of the desktop internal API because
+        // there can be restrictions on videos in the embedded player.
+        // E.g. if a video is age-restricted, the embedded player's playabilityStatus says that
+        // the video cannot be played outside of YouTube, but does not show the original message.
+        final JsonObject playabilityStatus = webPlayerResponse.getObject("playabilityStatus");
+
+        final boolean isAgeRestricted = "login_required".equalsIgnoreCase(
+                playabilityStatus.getString("status"))
+                && playabilityStatus.getString("reason", "")
                 .contains("age");
 
         setStreamType();
 
-        if (!playerResponse.has(STREAMING_DATA)) {
+        if (isAgeRestricted) {
+            fetchTvHtml5EmbedJsonPlayer(contentCountry, localization, videoId);
+
+            // If no streams can be fetched in the TVHTML5 simply embed client, the video should be
+            // age-restricted, therefore throw an AgeRestrictedContentException explicitly.
+            if (tvHtml5SimplyEmbedStreamingData == null) {
+                throw new AgeRestrictedContentException(
+                        "This age-restricted video cannot be watched.");
+            }
+
+            // Refresh the stream type because the stream type may be not properly known for
+            // age-restricted videos
+            setStreamType();
+        } else {
+            checkPlayabilityStatus(webPlayerResponse, playabilityStatus);
+
+            // Fetching successfully the iOS player is mandatory to get streams
+            fetchIosMobileJsonPlayer(contentCountry, localization, videoId);
+
             try {
-                fetchTvHtml5EmbedJsonPlayer(contentCountry, localization, videoId);
+                fetchAndroidMobileJsonPlayer(contentCountry, localization, videoId);
             } catch (final Exception ignored) {
+                // Ignore exceptions related to ANDROID client fetch or parsing, as it is not
+                // compulsory to play contents
             }
         }
 
-        // Refresh the stream type because the stream type may be not properly known for
-        // age-restricted videos
-        setStreamType();
-
-        if (html5StreamingData == null && playerResponse.has(STREAMING_DATA)) {
-            html5StreamingData = playerResponse.getObject(STREAMING_DATA);
-        }
-
-        if (html5StreamingData == null) {
-            checkPlayabilityStatus(youtubePlayerResponse, playabilityStatus);
-        }
-
-        // The microformat JSON object of the content is not returned on the client we use to
-        // try to get streams of unavailable contents but is still returned on the WEB client,
+        // The microformat JSON object of the content is only returned on the WEB client,
         // so we need to store it instead of getting it directly from the playerResponse
-        playerMicroFormatRenderer = youtubePlayerResponse.getObject("microformat")
+        playerMicroFormatRenderer = webPlayerResponse.getObject("microformat")
                 .getObject("playerMicroformatRenderer");
 
-        if (isPlayerResponseNotValid(playerResponse, videoId)) {
-            throw new ExtractionException("Initial player response is not valid");
-        }
-
         final byte[] body = JsonWriter.string(
                 prepareDesktopJsonBuilder(localization, contentCountry)
                         .value(VIDEO_ID, videoId)
@@ -861,29 +850,6 @@ public class YoutubeStreamExtractor extends StreamExtractor {
                         .done())
                 .getBytes(StandardCharsets.UTF_8);
         nextResponse = getJsonPostResponse(NEXT, body, localization);
-
-        // streamType can only have LIVE_STREAM, POST_LIVE_STREAM and VIDEO_STREAM values (see
-        // setStreamType()), so this block will be run only for POST_LIVE_STREAM and VIDEO_STREAM
-        // values if fetching of the ANDROID client is not forced
-        if ((!isAgeRestricted && streamType != StreamType.LIVE_STREAM)
-                || isAndroidClientFetchForced) {
-            try {
-                fetchAndroidMobileJsonPlayer(contentCountry, localization, videoId);
-            } catch (final Exception ignored) {
-                // Ignore exceptions related to ANDROID client fetch or parsing, as it is not
-                // compulsory to play contents
-            }
-        }
-
-        if ((!isAgeRestricted && streamType == StreamType.LIVE_STREAM)
-                || isIosClientFetchForced) {
-            try {
-                fetchIosMobileJsonPlayer(contentCountry, localization, videoId);
-            } catch (final Exception ignored) {
-                // Ignore exceptions related to IOS client fetch or parsing, as it is not
-                // compulsory to play contents
-            }
-        }
     }
 
     private void checkPlayabilityStatus(final JsonObject youtubePlayerResponse,
@@ -901,17 +867,10 @@ public class YoutubeStreamExtractor extends StreamExtractor {
         status = newPlayabilityStatus.getString("status");
         final String reason = newPlayabilityStatus.getString("reason");
 
-        if (status.equalsIgnoreCase("login_required")) {
-            if (reason == null) {
-                final String message = newPlayabilityStatus.getArray("messages").getString(0);
-                if (message != null && message.contains("private")) {
-                    throw new PrivateContentException("This video is private.");
-                }
-            } else if (reason.contains("age")) {
-                // No streams can be fetched, therefore throw an AgeRestrictedContentException
-                // explicitly.
-                throw new AgeRestrictedContentException(
-                        "This age-restricted video cannot be watched.");
+        if (status.equalsIgnoreCase("login_required") && reason == null) {
+            final String message = newPlayabilityStatus.getArray("messages").getString(0);
+            if (message != null && message.contains("private")) {
+                throw new PrivateContentException("This video is private.");
             }
         }
 
@@ -936,10 +895,9 @@ public class YoutubeStreamExtractor extends StreamExtractor {
                 if (detailedErrorMessage != null && detailedErrorMessage.contains("country")) {
                     throw new GeographicRestrictionException(
                             "This video is not available in client's country.");
-                } else if (detailedErrorMessage != null) {
-                    throw new ContentNotAvailableException(detailedErrorMessage);
                 } else {
-                    throw new ContentNotAvailableException(reason);
+                    throw new ContentNotAvailableException(
+                            Objects.requireNonNullElse(detailedErrorMessage, reason));
                 }
             }
         }
@@ -958,29 +916,34 @@ public class YoutubeStreamExtractor extends StreamExtractor {
         androidCpn = generateContentPlaybackNonce();
         final byte[] mobileBody = JsonWriter.string(
                 prepareAndroidMobileJsonBuilder(localization, contentCountry)
+                        .object("playerRequest")
+                            .value(VIDEO_ID, videoId)
+                        .end()
+                        .value("disablePlayerResponse", false)
                         .value(VIDEO_ID, videoId)
                         .value(CPN, androidCpn)
                         .value(CONTENT_CHECK_OK, true)
                         .value(RACY_CHECK_OK, true)
-                        // Workaround getting streaming URLs which return 403 HTTP response code by
-                        // using some parameters for Android client requests
-                        .value("params", "CgIQBg")
                         .done())
                 .getBytes(StandardCharsets.UTF_8);
 
-        final JsonObject androidPlayerResponse = getJsonAndroidPostResponse(PLAYER,
-                mobileBody, localization, "&t=" + generateTParameter()
-                        + "&id=" + videoId);
+        final JsonObject androidPlayerResponse = getJsonAndroidPostResponse(
+                "reel/reel_item_watch",
+                mobileBody,
+                localization,
+                "&t=" + generateTParameter() + "&id=" + videoId + "&$fields=playerResponse");
 
-        if (isPlayerResponseNotValid(androidPlayerResponse, videoId)) {
+        final JsonObject playerResponseObject = androidPlayerResponse.getObject("playerResponse");
+        if (isPlayerResponseNotValid(playerResponseObject, videoId)) {
             return;
         }
 
-        final JsonObject streamingData = androidPlayerResponse.getObject(STREAMING_DATA);
+        final JsonObject streamingData = playerResponseObject.getObject(STREAMING_DATA);
         if (!isNullOrEmpty(streamingData)) {
             androidStreamingData = streamingData;
-            if (html5StreamingData == null) {
-                playerResponse = androidPlayerResponse;
+            if (isNullOrEmpty(playerCaptionsTracklistRenderer)) {
+                playerCaptionsTracklistRenderer = playerResponseObject.getObject("captions")
+                        .getObject("playerCaptionsTracklistRenderer");
             }
         }
     }
@@ -1008,15 +971,14 @@ public class YoutubeStreamExtractor extends StreamExtractor {
                         + "&id=" + videoId);
 
         if (isPlayerResponseNotValid(iosPlayerResponse, videoId)) {
-            return;
+            throw new ExtractionException("IOS player response is not valid");
         }
 
         final JsonObject streamingData = iosPlayerResponse.getObject(STREAMING_DATA);
         if (!isNullOrEmpty(streamingData)) {
             iosStreamingData = streamingData;
-            if (html5StreamingData == null) {
-                playerResponse = iosPlayerResponse;
-            }
+            playerCaptionsTracklistRenderer = iosPlayerResponse.getObject("captions")
+                    .getObject("playerCaptionsTracklistRenderer");
         }
     }
 
@@ -1033,26 +995,25 @@ public class YoutubeStreamExtractor extends StreamExtractor {
                                              @Nonnull final Localization localization,
                                              @Nonnull final String videoId)
             throws IOException, ExtractionException {
-        // Because a cpn is unique to each request, we need to generate it again
-        html5Cpn = generateContentPlaybackNonce();
+        tvHtml5SimplyEmbedCpn = generateContentPlaybackNonce();
 
         final JsonObject tvHtml5EmbedPlayerResponse = getJsonPostResponse(PLAYER,
-                createDesktopPlayerBody(localization,
+                createTvHtml5EmbedPlayerBody(localization,
                         contentCountry,
                         videoId,
                         YoutubeJavaScriptPlayerManager.getSignatureTimestamp(videoId),
-                        true,
-                        html5Cpn), localization);
+                        tvHtml5SimplyEmbedCpn), localization);
 
         if (isPlayerResponseNotValid(tvHtml5EmbedPlayerResponse, videoId)) {
-            return;
+            throw new ExtractionException("TVHTML5 embed player response is not valid");
         }
 
-        final JsonObject streamingData = tvHtml5EmbedPlayerResponse.getObject(
-                STREAMING_DATA);
+        final JsonObject streamingData = tvHtml5EmbedPlayerResponse.getObject(STREAMING_DATA);
         if (!isNullOrEmpty(streamingData)) {
             playerResponse = tvHtml5EmbedPlayerResponse;
-            html5StreamingData = streamingData;
+            tvHtml5SimplyEmbedStreamingData = streamingData;
+            playerCaptionsTracklistRenderer = playerResponse.getObject("captions")
+                    .getObject("playerCaptionsTracklistRenderer");
         }
     }
 
@@ -1144,14 +1105,20 @@ public class YoutubeStreamExtractor extends StreamExtractor {
             final List<T> streamList = new ArrayList<>();
 
             java.util.stream.Stream.of(
-                    // Use the androidStreamingData object first because there is no n param and no
-                    // signatureCiphers in streaming URLs of the Android client
+                    /*
+                    Use the iosStreamingData object first because there is no n param and no
+                    signatureCiphers in streaming URLs of the iOS client
+
+                    The androidStreamingData is used as second way as it isn't used on livestreams,
+                    it doesn't return all available streams, and the Android client extraction is
+                    more likely to break
+
+                    As age-restricted videos are not common, use tvHtml5SimplyEmbedStreamingData
+                    last, which will be the only one not empty for age-restricted content
+                     */
+                    new Pair<>(iosStreamingData, iosCpn),
                     new Pair<>(androidStreamingData, androidCpn),
-                    new Pair<>(html5StreamingData, html5Cpn),
-                    // Use the iosStreamingData object in the last position because most of the
-                    // available streams can be extracted with the Android and web clients and also
-                    // because the iOS client is only enabled by default on livestreams
-                    new Pair<>(iosStreamingData, iosCpn)
+                    new Pair<>(tvHtml5SimplyEmbedStreamingData, tvHtml5SimplyEmbedCpn)
             )
                     .flatMap(pair -> getStreamsFromStreamingDataKey(videoId, pair.getFirst(),
                             streamingDataKey, itagTypeWanted, pair.getSecond()))
@@ -1303,8 +1270,9 @@ public class YoutubeStreamExtractor extends StreamExtractor {
                             return buildAndAddItagInfoToList(videoId, formatData, itagItem,
                                     itagItem.itagType, contentPlaybackNonce);
                         }
-                    } catch (final IOException | ExtractionException ignored) {
-                        // if the itag is not supported and getItag fails, we end up here
+                    } catch (final ExtractionException ignored) {
+                        // If the itag is not supported, the n parameter of HTML5 clients cannot be
+                        // decoded or buildAndAddItagInfoToList fails, we end up here
                     }
                     return null;
                 })
@@ -1316,26 +1284,31 @@ public class YoutubeStreamExtractor extends StreamExtractor {
             @Nonnull final JsonObject formatData,
             @Nonnull final ItagItem itagItem,
             @Nonnull final ItagItem.ItagType itagType,
-            @Nonnull final String contentPlaybackNonce) throws IOException, ExtractionException {
+            @Nonnull final String contentPlaybackNonce) throws ExtractionException {
         String streamUrl;
         if (formatData.has("url")) {
             streamUrl = formatData.getString("url");
         } else {
             // This url has an obfuscated signature
-            final String cipherString = formatData.has(CIPHER)
-                    ? formatData.getString(CIPHER)
-                    : formatData.getString(SIGNATURE_CIPHER);
-            final Map<String, String> cipher = Parser.compatParseMap(
-                    cipherString);
-            streamUrl = cipher.get("url") + "&" + cipher.get("sp") + "="
-                    + YoutubeJavaScriptPlayerManager.deobfuscateSignature(videoId, cipher.get("s"));
+            final String cipherString = formatData.getString(CIPHER,
+                    formatData.getString(SIGNATURE_CIPHER));
+            final var cipher = Parser.compatParseMap(cipherString);
+            final String signature = YoutubeJavaScriptPlayerManager.deobfuscateSignature(videoId,
+                    cipher.getOrDefault("s", ""));
+            streamUrl = cipher.get("url") + "&" + cipher.get("sp") + "=" + signature;
         }
 
         // Add the content playback nonce to the stream URL
         streamUrl += "&" + CPN + "=" + contentPlaybackNonce;
 
-        // Decrypt the n parameter if it is present
-        streamUrl = tryDeobfuscateThrottlingParameterOfUrl(streamUrl, videoId);
+        // Decode the n parameter if it is present
+        // If it cannot be decoded, the stream cannot be used as streaming URLs return HTTP 403
+        // responses if it has not the right value
+        // Exceptions thrown by
+        // YoutubeJavaScriptPlayerManager.getUrlWithThrottlingParameterDeobfuscated are so
+        // propagated to the parent which ignores streams in this case
+        streamUrl = YoutubeJavaScriptPlayerManager.getUrlWithThrottlingParameterDeobfuscated(
+                videoId, streamUrl);
 
         final JsonObject initRange = formatData.getObject("initRange");
         final JsonObject indexRange = formatData.getObject("indexRange");
@@ -1377,8 +1350,9 @@ public class YoutubeStreamExtractor extends StreamExtractor {
                 final int audioTrackIdLastLocaleCharacter = audioTrackId.indexOf(".");
                 if (audioTrackIdLastLocaleCharacter != -1) {
                     // Audio tracks IDs are in the form LANGUAGE_CODE.TRACK_NUMBER
-                    itagItem.setAudioLocale(LocaleCompat.forLanguageTag(
-                            audioTrackId.substring(0, audioTrackIdLastLocaleCharacter)));
+                    LocaleCompat.forLanguageTag(
+                            audioTrackId.substring(0, audioTrackIdLastLocaleCharacter)
+                    ).ifPresent(itagItem::setAudioLocale);
                 }
                 itagItem.setAudioTrackType(YoutubeParsingHelper.extractAudioTrackType(streamUrl));
             }
@@ -1410,6 +1384,20 @@ public class YoutubeStreamExtractor extends StreamExtractor {
         return itagInfo;
     }
 
+
+    /**
+     * {@inheritDoc}
+     * Should return a list of Frameset object that contains preview of stream frames
+     *
+     * <p><b>Warning:</b> When using this method be aware
+     * that the YouTube API very rarely returns framesets,
+     * that are slightly too small e.g. framesPerPageX = 5, frameWidth = 160, but the url contains
+     * a storyboard that is only 795 pixels wide (5*160 &gt; 795). You will need to handle this
+     * "manually" to avoid errors.</p>
+     *
+     * @see <a href="https://github.com/TeamNewPipe/NewPipe/pull/11596">
+     *     TeamNewPipe/NewPipe#11596</a>
+     */
     @Nonnull
     @Override
     public List<Frameset> getFrames() throws ExtractionException {
@@ -1592,49 +1580,11 @@ public class YoutubeStreamExtractor extends StreamExtractor {
     @Nonnull
     @Override
     public List<MetaInfo> getMetaInfo() throws ParsingException {
-        return YoutubeParsingHelper.getMetaInfo(nextResponse
+        return YoutubeMetaInfoHelper.getMetaInfo(nextResponse
                 .getObject("contents")
                 .getObject("twoColumnWatchNextResults")
                 .getObject("results")
                 .getObject("results")
                 .getArray("contents"));
     }
-
-    /**
-     * Enable or disable the fetch of the Android client for all stream types.
-     *
-     * <p>
-     * By default, the fetch of the Android client will be made only on videos, in order to reduce
-     * data usage, because available streams of the Android client will be almost equal to the ones
-     * available on the {@code WEB} client: you can get exclusively a 48kbps audio stream and a
-     * 3GPP very low stream (which is, most of times, a 144p8 stream).
-     * </p>
-     *
-     * @param forceFetchAndroidClientValue whether to always fetch the Android client and not only
-     *                                     for videos
-     */
-    public static void forceFetchAndroidClient(final boolean forceFetchAndroidClientValue) {
-        isAndroidClientFetchForced = forceFetchAndroidClientValue;
-    }
-
-    /**
-     * Enable or disable the fetch of the iOS client for all stream types.
-     *
-     * <p>
-     * By default, the fetch of the iOS client will be made only on livestreams, in order to get an
-     * HLS manifest with separated audio and video which has also an higher replay time (up to one
-     * hour, depending of the content instead of 30 seconds with non-iOS clients).
-     * </p>
-     *
-     * <p>
-     * Enabling this option will allow you to get an HLS manifest also for regular videos, which
-     * contains resolutions up to 1080p60.
-     * </p>
-     *
-     * @param forceFetchIosClientValue whether to always fetch the iOS client and not only for
-     *                                 livestreams
-     */
-    public static void forceFetchIosClient(final boolean forceFetchIosClientValue) {
-        isIosClientFetchForced = forceFetchIosClientValue;
-    }
 }

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeChannelTabLinkHandlerFactory.java

@@ -29,6 +29,8 @@ public final class YoutubeChannelTabLinkHandlerFactory extends ListLinkHandlerFa
                 return "/shorts";
             case ChannelTabs.LIVESTREAMS:
                 return "/streams";
+            case ChannelTabs.ALBUMS:
+                return "/releases";
             case ChannelTabs.PLAYLISTS:
                 return "/playlists";
             default:
@@ -65,6 +67,7 @@ public final class YoutubeChannelTabLinkHandlerFactory extends ListLinkHandlerFa
                 ChannelTabs.VIDEOS,
                 ChannelTabs.SHORTS,
                 ChannelTabs.LIVESTREAMS,
+                ChannelTabs.ALBUMS,
                 ChannelTabs.PLAYLISTS
         };
     }

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeSearchQueryHandlerFactory.java

@@ -6,7 +6,6 @@ import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
 import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandlerFactory;
 
-import java.io.UnsupportedEncodingException;
 import java.util.List;
 
 import javax.annotation.Nonnull;
@@ -40,31 +39,22 @@ public final class YoutubeSearchQueryHandlerFactory extends SearchQueryHandlerFa
                          @Nonnull final List<String> contentFilters,
                          final String sortFilter)
             throws ParsingException, UnsupportedOperationException {
-        try {
-            if (!contentFilters.isEmpty()) {
-                final String contentFilter = contentFilters.get(0);
-                switch (contentFilter) {
-                    case VIDEOS:
-                        return SEARCH_URL + encodeUrlUtf8(searchString)
-                                + "&sp=EgIQAfABAQ%253D%253D";
-                    case CHANNELS:
-                        return SEARCH_URL + encodeUrlUtf8(searchString)
-                                + "&sp=EgIQAvABAQ%253D%253D";
-                    case PLAYLISTS:
-                        return SEARCH_URL + encodeUrlUtf8(searchString)
-                                + "&sp=EgIQA_ABAQ%253D%253D";
-                    case MUSIC_SONGS:
-                    case MUSIC_VIDEOS:
-                    case MUSIC_ALBUMS:
-                    case MUSIC_PLAYLISTS:
-                    case MUSIC_ARTISTS:
-                        return MUSIC_SEARCH_URL + encodeUrlUtf8(searchString);
-                }
-            }
-
-            return SEARCH_URL + encodeUrlUtf8(searchString) + "&sp=8AEB";
-        } catch (final UnsupportedEncodingException e) {
-            throw new ParsingException("Could not encode query", e);
+        final String contentFilter = !contentFilters.isEmpty() ? contentFilters.get(0) : "";
+        switch (contentFilter) {
+            case VIDEOS:
+                return SEARCH_URL + encodeUrlUtf8(searchString) + "&sp=EgIQAfABAQ%253D%253D";
+            case CHANNELS:
+                return SEARCH_URL + encodeUrlUtf8(searchString) + "&sp=EgIQAvABAQ%253D%253D";
+            case PLAYLISTS:
+                return SEARCH_URL + encodeUrlUtf8(searchString) + "&sp=EgIQA_ABAQ%253D%253D";
+            case MUSIC_SONGS:
+            case MUSIC_VIDEOS:
+            case MUSIC_ALBUMS:
+            case MUSIC_PLAYLISTS:
+            case MUSIC_ARTISTS:
+                return MUSIC_SEARCH_URL + encodeUrlUtf8(searchString);
+            default:
+                return SEARCH_URL + encodeUrlUtf8(searchString) + "&sp=8AEB";
         }
     }
 

extractor/src/main/java/org/schabi/newpipe/extractor/stream/AudioTrackType.java

@@ -1,12 +1,13 @@
 package org.schabi.newpipe.extractor.stream;
 
 /**
- * An enum representing the track type of an {@link AudioStream} extracted by a {@link
+ * An enum representing the track type of {@link AudioStream}s extracted by a {@link
  * StreamExtractor}.
  */
 public enum AudioTrackType {
+
     /**
-     * An original audio track of the video.
+     * An original audio track of a video.
      */
     ORIGINAL,
 
@@ -20,6 +21,7 @@ public enum AudioTrackType {
 
     /**
      * A descriptive audio track.
+     *
      * <p>
      * A descriptive audio track is an audio track in which descriptions of visual elements of
      * a video are added to the original audio, with the goal to make a video more accessible to
@@ -29,5 +31,15 @@ public enum AudioTrackType {
      * @see <a href="https://en.wikipedia.org/wiki/Audio_description">
      * https://en.wikipedia.org/wiki/Audio_description</a>
      */
-    DESCRIPTIVE
+    DESCRIPTIVE,
+
+    /**
+     * A secondary audio track.
+     *
+     * <p>
+     * A secondary audio track can be an alternate audio track from the original language of a
+     * video or an alternate language.
+     * </p>
+     */
+    SECONDARY
 }

extractor/src/main/java/org/schabi/newpipe/extractor/stream/DeliveryMethod.java

@@ -34,8 +34,8 @@ public enum DeliveryMethod {
     /**
      * Used for {@link Stream}s served using the SmoothStreaming adaptive streaming method.
      *
-     * @see <a href="https://en.wikipedia.org/wiki/Adaptive_bitrate_streaming
-     * #Microsoft_Smooth_Streaming_(MSS)">Wikipedia's page about adaptive bitrate streaming,
+     * @see <a href="https://en.wikipedia.org/wiki/Adaptive_bitrate_streaming#Microsoft_Smooth_Streaming_(MSS)">
+     * Wikipedia's page about adaptive bitrate streaming,
      * section <i>Microsoft Smooth Streaming (MSS)</i></a> for more information about the
      * SmoothStreaming delivery method
      */

extractor/src/main/java/org/schabi/newpipe/extractor/stream/Description.java

@@ -3,6 +3,8 @@ package org.schabi.newpipe.extractor.stream;
 import java.io.Serializable;
 import java.util.Objects;
 
+import javax.annotation.Nullable;
+
 public class Description implements Serializable {
 
     public static final int HTML = 1;
@@ -13,7 +15,7 @@ public class Description implements Serializable {
     private final String content;
     private final int type;
 
-    public Description(final String content, final int type) {
+    public Description(@Nullable final String content, final int type) {
         this.type = type;
         if (content == null) {
             this.content = "";

extractor/src/main/java/org/schabi/newpipe/extractor/stream/Frameset.java

@@ -3,6 +3,9 @@ package org.schabi.newpipe.extractor.stream;
 import java.io.Serializable;
 import java.util.List;
 
+/**
+ * Class to handle framesets / storyboards which summarize the stream content.
+ */
 public final class Frameset implements Serializable {
 
     private final List<String> urls;
@@ -13,6 +16,17 @@ public final class Frameset implements Serializable {
     private final int framesPerPageX;
     private final int framesPerPageY;
 
+    /**
+     * Creates a new Frameset or set of storyboards.
+     * @param urls the URLs to the images with frames / storyboards
+     * @param frameWidth the width of a single frame, in pixels
+     * @param frameHeight the height of a single frame, in pixels
+     * @param totalCount the total count of frames
+     * @param durationPerFrame the duration per frame in milliseconds
+     * @param framesPerPageX the maximum count of frames per page by x / over the width of the image
+     * @param framesPerPageY the maximum count of frames per page by y / over the height
+     *                       of the image
+     */
     public Frameset(
             final List<String> urls,
             final int frameWidth,
@@ -32,7 +46,7 @@ public final class Frameset implements Serializable {
     }
 
     /**
-     * @return list of urls to images with frames
+     * @return list of URLs to images with frames
      */
     public List<String> getUrls() {
         return urls;

extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfoItemExtractor.java

@@ -50,7 +50,7 @@ public interface StreamInfoItemExtractor extends InfoItemExtractor {
     /**
      * Get the stream duration in seconds
      *
-     * @return the stream duration in seconds
+     * @return the stream duration in seconds or -1 if no duration is available
      * @throws ParsingException if there is an error in the extraction
      */
     long getDuration() throws ParsingException;

extractor/src/main/java/org/schabi/newpipe/extractor/stream/SubtitlesStream.java

@@ -1,6 +1,7 @@
 package org.schabi.newpipe.extractor.stream;
 
 import org.schabi.newpipe.extractor.MediaFormat;
+import org.schabi.newpipe.extractor.exceptions.ParsingException;
 import org.schabi.newpipe.extractor.services.youtube.ItagItem;
 import org.schabi.newpipe.extractor.utils.LocaleCompat;
 
@@ -170,7 +171,7 @@ public final class SubtitlesStream extends Stream {
          * not set, or have been set as {@code null}
          */
         @Nonnull
-        public SubtitlesStream build() {
+        public SubtitlesStream build() throws ParsingException {
             if (content == null) {
                 throw new IllegalStateException("No valid content was specified. Please specify a "
                         + "valid one with setContent.");
@@ -229,9 +230,11 @@ public final class SubtitlesStream extends Stream {
                             @Nonnull final DeliveryMethod deliveryMethod,
                             @Nonnull final String languageCode,
                             final boolean autoGenerated,
-                            @Nullable final String manifestUrl) {
+                            @Nullable final String manifestUrl) throws ParsingException {
         super(id, content, isUrl, mediaFormat, deliveryMethod, manifestUrl);
-        this.locale = LocaleCompat.forLanguageTag(languageCode);
+        this.locale = LocaleCompat.forLanguageTag(languageCode).orElseThrow(
+                () -> new ParsingException(
+                        "not a valid locale language code: " + languageCode));
         this.code = languageCode;
         this.format = mediaFormat;
         this.autoGenerated = autoGenerated;

extractor/src/main/java/org/schabi/newpipe/extractor/utils/LocaleCompat.java

@@ -1,6 +1,7 @@
 package org.schabi.newpipe.extractor.utils;
 
 import java.util.Locale;
+import java.util.Optional;
 
 /**
  * This class contains a simple implementation of {@link Locale#forLanguageTag(String)} for Android
@@ -15,29 +16,29 @@ public final class LocaleCompat {
 
     // Source: The AndroidX LocaleListCompat class's private forLanguageTagCompat() method.
     // Use Locale.forLanguageTag() on Android API level >= 21 / Java instead.
-    public static Locale forLanguageTag(final String str) {
+    public static Optional<Locale> forLanguageTag(final String str) {
         if (str.contains("-")) {
             final String[] args = str.split("-", -1);
             if (args.length > 2) {
-                return new Locale(args[0], args[1], args[2]);
+                return Optional.of(new Locale(args[0], args[1], args[2]));
             } else if (args.length > 1) {
-                return new Locale(args[0], args[1]);
+                return Optional.of(new Locale(args[0], args[1]));
             } else if (args.length == 1) {
-                return new Locale(args[0]);
+                return Optional.of(new Locale(args[0]));
             }
         } else if (str.contains("_")) {
             final String[] args = str.split("_", -1);
             if (args.length > 2) {
-                return new Locale(args[0], args[1], args[2]);
+                return Optional.of(new Locale(args[0], args[1], args[2]));
             } else if (args.length > 1) {
-                return new Locale(args[0], args[1]);
+                return Optional.of(new Locale(args[0], args[1]));
             } else if (args.length == 1) {
-                return new Locale(args[0]);
+                return Optional.of(new Locale(args[0]));
             }
         } else {
-            return new Locale(str);
+            return Optional.of(new Locale(str));
         }
 
-        throw new IllegalArgumentException("Can not parse language tag: [" + str + "]");
+        return Optional.empty();
     }
 }

extractor/src/main/java/org/schabi/newpipe/extractor/utils/Parser.java

@@ -22,11 +22,11 @@ package org.schabi.newpipe.extractor.utils;
 
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
 
-import java.io.UnsupportedEncodingException;
-import java.util.HashMap;
+import java.util.Arrays;
 import java.util.Map;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
+import java.util.stream.Collectors;
 
 import javax.annotation.Nonnull;
 
@@ -78,6 +78,37 @@ public final class Parser {
         }
     }
 
+    public static String matchGroup1MultiplePatterns(final Pattern[] patterns, final String input)
+            throws RegexException {
+        return matchMultiplePatterns(patterns, input).group(1);
+    }
+
+    public static Matcher matchMultiplePatterns(final Pattern[] patterns, final String input)
+            throws RegexException {
+        Parser.RegexException exception = null;
+        for (final Pattern pattern : patterns) {
+            final Matcher matcher = pattern.matcher(input);
+            if (matcher.find()) {
+                return matcher;
+            } else if (exception == null) {
+                // only pass input to exception message when it is not too long
+                if (input.length() > 1024) {
+                    exception = new RegexException("Failed to find pattern \"" + pattern.pattern()
+                            + "\"");
+                } else {
+                    exception = new RegexException("Failed to find pattern \"" + pattern.pattern()
+                            + "\" inside of \"" + input + "\"");
+                }
+            }
+        }
+
+        if (exception == null) {
+            throw new RegexException("Empty patterns array passed to matchMultiplePatterns");
+        } else {
+            throw exception;
+        }
+    }
+
     public static boolean isMatch(final String pattern, final String input) {
         final Pattern pat = Pattern.compile(pattern);
         final Matcher mat = pat.matcher(input);
@@ -90,17 +121,12 @@ public final class Parser {
     }
 
     @Nonnull
-    public static Map<String, String> compatParseMap(@Nonnull final String input)
-            throws UnsupportedEncodingException {
-        final Map<String, String> map = new HashMap<>();
-        for (final String arg : input.split("&")) {
-            final String[] splitArg = arg.split("=");
-            if (splitArg.length > 1) {
-                map.put(splitArg[0], Utils.decodeUrlUtf8(splitArg[1]));
-            } else {
-                map.put(splitArg[0], "");
-            }
-        }
-        return map;
+    public static Map<String, String> compatParseMap(@Nonnull final String input) {
+        return Arrays.stream(input.split("&"))
+                .map(arg -> arg.split("="))
+                .filter(splitArg -> splitArg.length > 1)
+                .collect(Collectors.toMap(splitArg -> splitArg[0],
+                        splitArg -> Utils.decodeUrlUtf8(splitArg[1]),
+                        (existing, replacement) -> replacement));
     }
 }

extractor/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java

@@ -2,7 +2,6 @@ package org.schabi.newpipe.extractor.utils;
 
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
 
-import java.io.UnsupportedEncodingException;
 import java.net.MalformedURLException;
 import java.net.URL;
 import java.net.URLDecoder;
@@ -33,22 +32,18 @@ public final class Utils {
      *
      * @param string The string to be encoded.
      * @return The encoded URL.
-     * @throws UnsupportedEncodingException This shouldn't be thrown, as UTF-8 should be supported.
      */
-    public static String encodeUrlUtf8(final String string) throws UnsupportedEncodingException {
-        // TODO: Switch to URLEncoder.encode(String, Charset) in Java 10.
-        return URLEncoder.encode(string, StandardCharsets.UTF_8.name());
+    public static String encodeUrlUtf8(final String string) {
+        return URLEncoder.encode(string, StandardCharsets.UTF_8);
     }
 
     /**
      * Decodes a URL using the UTF-8 character set.
      * @param url The URL to be decoded.
      * @return The decoded URL.
-     * @throws UnsupportedEncodingException This shouldn't be thrown, as UTF-8 should be supported.
      */
-    public static String decodeUrlUtf8(final String url) throws UnsupportedEncodingException {
-        // TODO: Switch to URLDecoder.decode(String, Charset) in Java 10.
-        return URLDecoder.decode(url, StandardCharsets.UTF_8.name());
+    public static String decodeUrlUtf8(final String url) {
+        return URLDecoder.decode(url, StandardCharsets.UTF_8);
     }
 
     /**
@@ -155,22 +150,10 @@ public final class Utils {
         if (urlQuery != null) {
             for (final String param : urlQuery.split("&")) {
                 final String[] params = param.split("=", 2);
-
-                String query;
-                try {
-                    query = decodeUrlUtf8(params[0]);
-                } catch (final UnsupportedEncodingException e) {
-                    // Cannot decode string with UTF-8, using the string without decoding
-                    query = params[0];
-                }
+                final String query = decodeUrlUtf8(params[0]);
 
                 if (query.equals(parameterName)) {
-                    try {
-                        return decodeUrlUtf8(params[1]);
-                    } catch (final UnsupportedEncodingException e) {
-                        // Cannot decode string with UTF-8, using the string without decoding
-                        return params[1];
-                    }
+                    return decodeUrlUtf8(params[1]);
                 }
             }
         }

extractor/src/test/java/org/schabi/newpipe/downloader/DownloaderTestImpl.java

@@ -18,8 +18,11 @@ import okhttp3.RequestBody;
 import okhttp3.ResponseBody;
 
 public final class DownloaderTestImpl extends Downloader {
+    /**
+     * Should be the latest Firefox ESR version.
+     */
     private static final String USER_AGENT
-            = "Mozilla/5.0 (Windows NT 10.0; rv:91.0) Gecko/20100101 Firefox/91.0";
+            = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:128.0) Gecko/20100101 Firefox/128.0";
     private static DownloaderTestImpl instance;
     private final OkHttpClient client;
 

extractor/src/test/java/org/schabi/newpipe/downloader/RecordingDownloader.java

@@ -15,6 +15,7 @@ import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
+import java.util.Random;
 
 import javax.annotation.Nonnull;
 
@@ -37,17 +38,31 @@ import javax.annotation.Nonnull;
  */
 class RecordingDownloader extends Downloader {
 
-    public final static String FILE_NAME_PREFIX = "generated_mock_";
+    public static final String FILE_NAME_PREFIX = "generated_mock_";
 
     // From https://stackoverflow.com/a/15875500/13516981
-    private final static String IP_V4_PATTERN =
+    private static final String IP_V4_PATTERN =
             "(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)";
 
     private int index = 0;
     private final String path;
 
+    // try to prevent ReCaptchaExceptions / rate limits by tracking and throttling the requests
     /**
-     * Creates the folder described by {@code stringPath} if it does not exists.
+     * The maximum number of requests per 20 seconds which are executed
+     * by the {@link RecordingDownloader}.
+     * 20 seconds is used as upper bound because the rate limit can be triggered within 30 seconds
+     * and hitting the rate limit should be prevented because it comes with a bigger delay.
+     * The values can be adjusted when executing the downloader and running into problems.
+     * <p>TODO: Allow adjusting the value by setting a param in the gradle command</p>
+     */
+    private static final int MAX_REQUESTS_PER_20_SECONDS = 30;
+    private static final long[] requestTimes = new long[MAX_REQUESTS_PER_20_SECONDS];
+    private static int requestTimesCursor = -1;
+    private static final Random throttleRandom = new Random();
+
+    /**
+     * Creates the folder described by {@code stringPath} if it does not exist.
      * Deletes existing files starting with {@link RecordingDownloader#FILE_NAME_PREFIX}.
      * @param stringPath Path to the folder where the json files will be saved to.
      */
@@ -69,6 +84,48 @@ class RecordingDownloader extends Downloader {
     @Override
     public Response execute(@Nonnull final Request request) throws IOException,
             ReCaptchaException {
+
+        // Delay the execution if the max number of requests per minute is reached
+        final long currentTime = System.currentTimeMillis();
+        // the cursor points to the latest request time and the next position is the oldest one
+        final int oldestRequestTimeCursor = (requestTimesCursor + 1) % requestTimes.length;
+        final long oldestRequestTime = requestTimes[oldestRequestTimeCursor];
+        if (oldestRequestTime + 20_000 >= currentTime) {
+            try {
+                // sleep at least until the oldest request is 20s old, but not more than 20s
+                final int minSleepTime = (int) (currentTime - oldestRequestTime);
+                Thread.sleep(minSleepTime + throttleRandom.nextInt(20_000 - minSleepTime));
+            } catch (InterruptedException e) {
+                // handle the exception gracefully because it's not critical for the test
+                System.err.println("Error while throttling the RecordingDownloader.");
+                e.printStackTrace();
+            }
+        }
+        requestTimesCursor = oldestRequestTimeCursor; // the oldest value needs to be overridden
+        requestTimes[requestTimesCursor] = System.currentTimeMillis();
+
+        // Handle ReCaptchaExceptions by retrying the request once after a while
+        try {
+            return executeRequest(request);
+        } catch (ReCaptchaException e) {
+            try {
+                // sleep for 35-60 seconds to circumvent the rate limit
+                System.out.println("Throttling the RecordingDownloader to handle a ReCaptcha."
+                        + " Sleeping for 35-60 seconds.");
+                Thread.sleep(35_000 + throttleRandom.nextInt(25_000));
+            } catch (InterruptedException ie) {
+                // handle the exception gracefully because it's not critical for the test
+                System.err.println("Error while throttling the RecordingDownloader.");
+                ie.printStackTrace();
+                e.printStackTrace();
+            }
+            return executeRequest(request);
+        }
+    }
+
+    @Nonnull
+    private Response executeRequest(@Nonnull final Request request) throws IOException,
+            ReCaptchaException {
         final Downloader downloader = DownloaderTestImpl.getInstance();
         Response response = downloader.execute(request);
         String cleanedResponseBody = response.responseBody().replaceAll(IP_V4_PATTERN, "127.0.0.1");

extractor/src/test/java/org/schabi/newpipe/extractor/ExtractorAsserts.java

@@ -171,8 +171,17 @@ public class ExtractorAsserts {
     public static void assertTabsContain(@Nonnull final List<ListLinkHandler> tabs,
                                          @Nonnull final String... expectedTabs) {
         final Set<String> tabSet = tabs.stream()
-                .map(linkHandler -> linkHandler.getContentFilters().get(0))
+                .map(linkHandler -> {
+                    assertEquals(1, linkHandler.getContentFilters().size(),
+                            "Unexpected content filters for channel tab: "
+                                    + linkHandler.getContentFilters());
+                    return linkHandler.getContentFilters().get(0);
+                })
                 .collect(Collectors.toUnmodifiableSet());
+
+        assertEquals(expectedTabs.length, tabSet.size(),
+                "Different amount of tabs returned:\nExpected: "
+                + Arrays.toString(expectedTabs) + "\nActual: " + tabSet);
         Arrays.stream(expectedTabs)
                 .forEach(expectedTab -> assertTrue(tabSet.contains(expectedTab),
                         "Missing " + expectedTab + " tab (got " + tabSet + ")"));

extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampCommentsLinkHandlerFactoryTest.java

@@ -39,6 +39,5 @@ public class BandcampCommentsLinkHandlerFactoryTest {
         assertTrue(linkHandler.acceptUrl("http://ZachBenson.Bandcamp.COM/Track/U-I-Tonite/"));
         assertTrue(linkHandler.acceptUrl("https://interovgm.bandcamp.com/track/title"));
         assertTrue(linkHandler.acceptUrl("https://goodgoodblood-tl.bandcamp.com/track/when-it-all-wakes-up"));
-        assertTrue(linkHandler.acceptUrl("https://lobstertheremin.com/track/unfinished"));
     }
 }

extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampFeaturedLinkHandlerFactoryTest.java

@@ -28,7 +28,7 @@ public class BandcampFeaturedLinkHandlerFactoryTest {
         assertTrue(linkHandler.acceptUrl("https://bandcamp.com/?show=1"));
         assertTrue(linkHandler.acceptUrl("http://bandcamp.com/?show=2"));
         assertTrue(linkHandler.acceptUrl("https://bandcamp.com/api/mobile/24/bootstrap_data"));
-        assertTrue(linkHandler.acceptUrl("https://bandcamp.com/api/bcweekly/1/list"));
+        assertTrue(linkHandler.acceptUrl("https://bandcamp.com/api/bcweekly/3/list"));
 
         assertFalse(linkHandler.acceptUrl("https://bandcamp.com/?show="));
         assertFalse(linkHandler.acceptUrl("https://bandcamp.com/?show=a"));
@@ -38,7 +38,7 @@ public class BandcampFeaturedLinkHandlerFactoryTest {
     @Test
     public void testGetUrl() throws ParsingException {
         assertEquals("https://bandcamp.com/api/mobile/24/bootstrap_data", linkHandler.getUrl("Featured"));
-        assertEquals("https://bandcamp.com/api/bcweekly/1/list", linkHandler.getUrl("Radio"));
+        assertEquals("https://bandcamp.com/api/bcweekly/3/list", linkHandler.getUrl("Radio"));
     }
 
     @Test
@@ -46,7 +46,7 @@ public class BandcampFeaturedLinkHandlerFactoryTest {
         assertEquals("Featured", linkHandler.getId("http://bandcamp.com/api/mobile/24/bootstrap_data"));
         assertEquals("Featured", linkHandler.getId("https://bandcamp.com/api/mobile/24/bootstrap_data"));
         assertEquals("Radio", linkHandler.getId("http://bandcamp.com/?show=1"));
-        assertEquals("Radio", linkHandler.getId("https://bandcamp.com/api/bcweekly/1/list"));
+        assertEquals("Radio", linkHandler.getId("https://bandcamp.com/api/bcweekly/3/list"));
     }
 
 }

extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampRadioExtractorTest.java

@@ -8,6 +8,7 @@ import org.schabi.newpipe.downloader.DownloaderTestImpl;
 import org.schabi.newpipe.extractor.NewPipe;
 import org.schabi.newpipe.extractor.exceptions.ExtractionException;
 import org.schabi.newpipe.extractor.services.BaseListExtractorTest;
+import org.schabi.newpipe.extractor.services.DefaultTests;
 import org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampRadioExtractor;
 import org.schabi.newpipe.extractor.stream.StreamInfoItem;
 
@@ -35,15 +36,14 @@ public class BandcampRadioExtractorTest implements BaseListExtractorTest {
     }
 
     @Test
-    public void testRadioCount() throws ExtractionException, IOException {
+    public void testRadioCount() {
         final List<StreamInfoItem> list = extractor.getInitialPage().getItems();
         assertTrue(list.size() > 300);
     }
 
     @Test
     public void testRelatedItems() throws Exception {
-        // DefaultTests.defaultTestRelatedItems(extractor);
-        // Would fail because BandcampRadioInfoItemExtractor.getUploaderName() returns an empty String
+        DefaultTests.defaultTestRelatedItems(extractor);
     }
 
     @Test
@@ -68,11 +68,11 @@ public class BandcampRadioExtractorTest implements BaseListExtractorTest {
 
     @Test
     public void testUrl() throws Exception {
-        assertEquals("https://bandcamp.com/api/bcweekly/1/list", extractor.getUrl());
+        assertEquals("https://bandcamp.com/api/bcweekly/3/list", extractor.getUrl());
     }
 
     @Test
     public void testOriginalUrl() throws Exception {
-        assertEquals("https://bandcamp.com/api/bcweekly/1/list", extractor.getOriginalUrl());
+        assertEquals("https://bandcamp.com/api/bcweekly/3/list", extractor.getOriginalUrl());
     }
 }

extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampStreamLinkHandlerFactoryTest.java

@@ -49,6 +49,5 @@ public class BandcampStreamLinkHandlerFactoryTest {
         assertTrue(linkHandler.acceptUrl("https://interovgm.bandcamp.com/track/title"));
         assertTrue(linkHandler.acceptUrl("http://bandcamP.com/?show=38"));
         assertTrue(linkHandler.acceptUrl("https://goodgoodblood-tl.bandcamp.com/track/when-it-all-wakes-up"));
-        assertTrue(linkHandler.acceptUrl("https://lobstertheremin.com/track/unfinished"));
     }
 }

extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCChannelTabExtractorTest.java

@@ -0,0 +1,49 @@
+package org.schabi.newpipe.extractor.services.media_ccc;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.schabi.newpipe.extractor.ServiceList.MediaCCC;
+
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Test;
+import org.schabi.newpipe.downloader.DownloaderTestImpl;
+import org.schabi.newpipe.extractor.NewPipe;
+import org.schabi.newpipe.extractor.channel.tabs.ChannelTabExtractor;
+import org.schabi.newpipe.extractor.channel.tabs.ChannelTabs;
+
+/**
+ * Test that it is possible to create and use a channel tab extractor ({@link
+ * org.schabi.newpipe.extractor.services.media_ccc.extractors.MediaCCCChannelTabExtractor}) without
+ * passing through the conference extractor
+ */
+public class MediaCCCChannelTabExtractorTest {
+    public static class CCCamp2023 {
+        private static ChannelTabExtractor extractor;
+
+        @BeforeAll
+        public static void setUpClass() throws Exception {
+            NewPipe.init(DownloaderTestImpl.getInstance());
+            extractor = MediaCCC.getChannelTabExtractorFromId("camp2023", ChannelTabs.VIDEOS);
+            extractor.fetchPage();
+        }
+
+        @Test
+        void testName() {
+            assertEquals(ChannelTabs.VIDEOS, extractor.getName());
+        }
+
+        @Test
+        void testGetUrl() throws Exception {
+            assertEquals("https://media.ccc.de/c/camp2023", extractor.getUrl());
+        }
+
+        @Test
+        void testGetOriginalUrl() throws Exception {
+            assertEquals("https://media.ccc.de/c/camp2023", extractor.getOriginalUrl());
+        }
+
+        @Test
+        void testGetInitalPage() throws Exception {
+            assertEquals(177, extractor.getInitialPage().getItems().size());
+        }
+    }
+}

extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCConferenceExtractorTest.java

@@ -13,7 +13,8 @@ import static org.schabi.newpipe.extractor.ExtractorAsserts.assertContainsImageU
 import static org.schabi.newpipe.extractor.ServiceList.MediaCCC;
 
 /**
- * Test {@link MediaCCCConferenceExtractor}
+ * Test {@link MediaCCCConferenceExtractor} and {@link
+ * org.schabi.newpipe.extractor.services.media_ccc.extractors.MediaCCCChannelTabExtractor}
  */
 public class MediaCCCConferenceExtractorTest {
     public static class FrOSCon2017 {

extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeChannelExtractorTest.java

@@ -234,4 +234,111 @@ public class PeertubeChannelExtractorTest {
             assertTrue(extractor.getTags().isEmpty());
         }
     }
+
+    public static class DocumentaryChannel implements BaseChannelExtractorTest {
+        // https://github.com/TeamNewPipe/NewPipe/issues/11369
+
+        private static PeertubeChannelExtractor extractor;
+
+        @BeforeAll
+        public static void setUp() throws Exception {
+            NewPipe.init(DownloaderTestImpl.getInstance());
+            // setting instance might break test when running in parallel
+            PeerTube.setInstance(new PeertubeInstance("https://kolektiva.media", "kolektiva.media"));
+            extractor = (PeertubeChannelExtractor) PeerTube
+                    .getChannelExtractor("https://kolektiva.media/video-channels/documentary_channel");
+            extractor.fetchPage();
+        }
+
+        /*//////////////////////////////////////////////////////////////////////////
+        // Extractor
+        //////////////////////////////////////////////////////////////////////////*/
+
+        @Test
+        public void testServiceId() {
+            assertEquals(PeerTube.getServiceId(), extractor.getServiceId());
+        }
+
+        @Test
+        public void testName() throws ParsingException {
+            assertEquals("Documentary Channel", extractor.getName());
+        }
+
+        @Test
+        public void testId() throws ParsingException {
+            assertEquals("video-channels/documentary_channel", extractor.getId());
+        }
+
+        @Test
+        public void testUrl() throws ParsingException {
+            assertEquals("https://kolektiva.media/video-channels/documentary_channel", extractor.getUrl());
+        }
+
+        @Test
+        public void testOriginalUrl() throws ParsingException {
+            assertEquals("https://kolektiva.media/video-channels/documentary_channel", extractor.getOriginalUrl());
+        }
+
+        /*//////////////////////////////////////////////////////////////////////////
+        // ChannelExtractor
+        //////////////////////////////////////////////////////////////////////////*/
+
+        @Test
+        public void testDescription() {
+            assertNotNull(extractor.getDescription());
+        }
+
+        @Test
+        void testParentChannelName() throws ParsingException {
+            assertEquals("consumedmind", extractor.getParentChannelName());
+        }
+
+        @Test
+        void testParentChannelUrl() throws ParsingException {
+            assertEquals("https://kolektiva.media/accounts/consumedmind", extractor.getParentChannelUrl());
+        }
+
+        @Test
+        void testParentChannelAvatars() {
+            defaultTestImageCollection(extractor.getParentChannelAvatars());
+        }
+
+        @Test
+        public void testAvatars() {
+            defaultTestImageCollection(extractor.getAvatars());
+        }
+
+        @Test
+        public void testBanners() throws ParsingException {
+            assertGreaterOrEqual(1, extractor.getBanners().size());
+        }
+
+        @Test
+        public void testFeedUrl() throws ParsingException {
+            assertEquals("https://kolektiva.media/feeds/videos.xml?videoChannelId=2994", extractor.getFeedUrl());
+        }
+
+        @Test
+        public void testSubscriberCount() {
+            assertGreaterOrEqual(25, extractor.getSubscriberCount());
+        }
+
+        @Test
+        @Override
+        public void testVerified() throws Exception {
+            assertFalse(extractor.isVerified());
+        }
+
+        @Test
+        @Override
+        public void testTabs() throws Exception {
+            assertTabsContain(extractor.getTabs(), ChannelTabs.VIDEOS, ChannelTabs.PLAYLISTS);
+        }
+
+        @Test
+        @Override
+        public void testTags() throws Exception {
+            assertTrue(extractor.getTags().isEmpty());
+        }
+    }
 }

extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeChannelLinkHandlerFactoryTest.java

@@ -7,9 +7,9 @@ import org.schabi.newpipe.extractor.NewPipe;
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
 import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeChannelLinkHandlerFactory;
 
-import static org.junit.jupiter.api.Assertions.assertEquals;
-import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.junit.jupiter.api.Assertions.*;
 import static org.schabi.newpipe.extractor.ServiceList.PeerTube;
+import static org.schabi.newpipe.extractor.services.peertube.PeertubeLinkHandlerFactoryTestHelper.assertDoNotAcceptNonURLs;
 
 /**
  * Test for {@link PeertubeChannelLinkHandlerFactory}
@@ -33,6 +33,9 @@ public class PeertubeChannelLinkHandlerFactoryTest {
         assertTrue(linkHandler.acceptUrl("https://peertube.stream/video-channels/kranti_channel@videos.squat.net/videos"));
         assertTrue(linkHandler.acceptUrl("https://peertube.stream/c/kranti_channel@videos.squat.net/videos"));
         assertTrue(linkHandler.acceptUrl("https://peertube.stream/api/v1/video-channels/7682d9f2-07be-4622-862e-93ec812e2ffa"));
+        assertTrue(linkHandler.acceptUrl("https://kolektiva.media/video-channels/documentary_channel"));
+
+        assertDoNotAcceptNonURLs(linkHandler);
     }
 
     @Test
@@ -58,6 +61,16 @@ public class PeertubeChannelLinkHandlerFactoryTest {
                 linkHandler.fromUrl("https://peertube.stream/c/kranti_channel@videos.squat.net/video-playlists").getId());
         assertEquals("video-channels/kranti_channel@videos.squat.net",
                 linkHandler.fromUrl("https://peertube.stream/api/v1/video-channels/kranti_channel@videos.squat.net").getId());
+
+        // instance URL ending with an "a": https://kolektiva.media
+        assertEquals("video-channels/documentary_channel",
+                linkHandler.fromUrl("https://kolektiva.media/video-channels/documentary_channel/videos").getId());
+        assertEquals("video-channels/documentary_channel",
+                linkHandler.fromUrl("https://kolektiva.media/c/documentary_channel/videos").getId());
+        assertEquals("video-channels/documentary_channel",
+                linkHandler.fromUrl("https://kolektiva.media/c/documentary_channel/video-playlists").getId());
+        assertEquals("video-channels/documentary_channel",
+                linkHandler.fromUrl("https://kolektiva.media/api/v1/video-channels/documentary_channel").getId());
     }
 
     @Test

extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeCommentsExtractorTest.java

@@ -48,7 +48,7 @@ public class PeertubeCommentsExtractorTest {
 
         @Test
         void testGetCommentsFromCommentsInfo() throws IOException, ExtractionException {
-            final String comment = "Thanks for creating such an informative video";
+            final String comment = "I love this. ❤";
 
             final CommentsInfo commentsInfo =
                     CommentsInfo.getInfo("https://framatube.org/w/kkGMgK9ZtnKfYAgnEtQxbv");

extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeCommentsLinkHandlerFactoryTest.java

@@ -9,6 +9,7 @@ import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeCommen
 
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.schabi.newpipe.extractor.services.peertube.PeertubeLinkHandlerFactoryTestHelper.assertDoNotAcceptNonURLs;
 
 /**
  * Test for {@link PeertubeCommentsLinkHandlerFactory}
@@ -31,6 +32,8 @@ public class PeertubeCommentsLinkHandlerFactoryTest {
         assertTrue(linkHandler.acceptUrl("https://framatube.org/videos/watch/9c9de5e8-0a1e-484a-b099-e80766180a6d"));
         assertTrue(linkHandler.acceptUrl("https://framatube.org/w/9c9de5e8-0a1e-484a-b099-e80766180a6d"));
         assertTrue(linkHandler.acceptUrl("https://framatube.org/api/v1/videos/9c9de5e8-0a1e-484a-b099-e80766180a6d/comment-threads?start=0&count=10&sort=-createdAt"));
+
+        assertDoNotAcceptNonURLs(linkHandler);
     }
 
     @Test

extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeLinkHandlerFactoryTestHelper.java

@@ -0,0 +1,40 @@
+package org.schabi.newpipe.extractor.services.peertube;
+
+import org.schabi.newpipe.extractor.exceptions.ParsingException;
+import org.schabi.newpipe.extractor.linkhandler.LinkHandlerFactory;
+import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory;
+
+import static org.junit.jupiter.api.Assertions.assertFalse;
+
+public class PeertubeLinkHandlerFactoryTestHelper {
+
+    public static void assertDoNotAcceptNonURLs(LinkHandlerFactory linkHandler)
+            throws ParsingException {
+        assertFalse(linkHandler.acceptUrl("orchestr/a/"));
+        assertFalse(linkHandler.acceptUrl("/a/"));
+        assertFalse(linkHandler.acceptUrl("something/c/"));
+        assertFalse(linkHandler.acceptUrl("/c/"));
+        assertFalse(linkHandler.acceptUrl("videos/"));
+        assertFalse(linkHandler.acceptUrl("I-hate-videos/"));
+        assertFalse(linkHandler.acceptUrl("/w/"));
+        assertFalse(linkHandler.acceptUrl("ksmg/w/"));
+        assertFalse(linkHandler.acceptUrl("a reandom search query"));
+        assertFalse(linkHandler.acceptUrl("test 230 "));
+        assertFalse(linkHandler.acceptUrl("986513"));
+    }
+
+    public static void assertDoNotAcceptNonURLs(ListLinkHandlerFactory linkHandler)
+            throws ParsingException {
+        assertFalse(linkHandler.acceptUrl("orchestr/a/"));
+        assertFalse(linkHandler.acceptUrl("/a/"));
+        assertFalse(linkHandler.acceptUrl("something/c/"));
+        assertFalse(linkHandler.acceptUrl("/c/"));
+        assertFalse(linkHandler.acceptUrl("videos/"));
+        assertFalse(linkHandler.acceptUrl("I-hate-videos/"));
+        assertFalse(linkHandler.acceptUrl("/w/"));
+        assertFalse(linkHandler.acceptUrl("ksmg/w/"));
+        assertFalse(linkHandler.acceptUrl("a reandom search query"));
+        assertFalse(linkHandler.acceptUrl("test 230 "));
+        assertFalse(linkHandler.acceptUrl("986513"));
+    }
+}

extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubePlaylistLinkHandlerFactoryTest.java

@@ -10,6 +10,7 @@ import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubePlayli
 import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.schabi.newpipe.extractor.services.peertube.PeertubeLinkHandlerFactoryTestHelper.assertDoNotAcceptNonURLs;
 
 /**
  * Test for {@link PeertubePlaylistLinkHandlerFactory}
@@ -33,6 +34,8 @@ public class PeertubePlaylistLinkHandlerFactoryTest {
         assertTrue(linkHandler.acceptUrl("https://framatube.org/w/p/dacdc4ef-5160-4846-9b70-a655880da667"));
         assertTrue(linkHandler.acceptUrl("https://framatube.org/videos/watch/playlist/96b0ee2b-a5a7-4794-8769-58d8ccb79ab7"));
         assertTrue(linkHandler.acceptUrl("https://framatube.org/w/p/96b0ee2b-a5a7-4794-8769-58d8ccb79ab7"));
+
+        assertDoNotAcceptNonURLs(linkHandler);
     }
 
     @Test

extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorTest.java

@@ -26,7 +26,6 @@ public abstract class PeertubeStreamExtractorTest extends DefaultStreamExtractor
     private static final String BASE_URL = "/videos/watch/";
 
     @Override public boolean expectedHasAudioStreams() { return false; }
-    @Override public boolean expectedHasFrames() { return false; }
 
     public static class WhatIsPeertube extends PeertubeStreamExtractorTest {
         private static final String ID = "9c9de5e8-0a1e-484a-b099-e80766180a6d";
@@ -86,7 +85,7 @@ public abstract class PeertubeStreamExtractorTest extends DefaultStreamExtractor
         @Override public long expectedViewCountAtLeast() { return 38600; }
         @Nullable @Override public String expectedUploadDate() { return "2018-10-01 10:52:46.396"; }
         @Nullable @Override public String expectedTextualUploadDate() { return "2018-10-01T10:52:46.396Z"; }
-        @Override public long expectedLikeCountAtLeast() { return 50; }
+        @Override public long expectedLikeCountAtLeast() { return 18; }
         @Override public long expectedDislikeCountAtLeast() { return 0; }
         @Override public String expectedHost() { return "framatube.org"; }
         @Override public String expectedCategory() { return "Science & Technology"; }
@@ -138,6 +137,7 @@ public abstract class PeertubeStreamExtractorTest extends DefaultStreamExtractor
         @Override public String expectedLicence() { return "Unknown"; }
         @Override public Locale expectedLanguageInfo() { return null; }
         @Override public List<String> expectedTags() { return Arrays.asList("Marinauts", "adobe flash", "adobe flash player", "flash games", "the marinauts"); }
+        @Override public boolean expectedHasFrames() { return false; } // not yet supported by instance
     }
 
     @Disabled("Test broken, SSL problem")
@@ -185,6 +185,50 @@ public abstract class PeertubeStreamExtractorTest extends DefaultStreamExtractor
         @Override public List<String> expectedTags() { return Arrays.asList("Covid-19", "Gérôme-Mary trebor", "Horreur et beauté", "court-métrage", "nue artistique"); }
     }
 
+    public static class Segments extends PeertubeStreamExtractorTest {
+        private static final String ID = "vqABGP97fEjo7RhPuDnSZk";
+        private static final String INSTANCE = "https://tube.tchncs.de";
+
+        private static final String URL = INSTANCE + BASE_URL + ID;
+        private static StreamExtractor extractor;
+
+        @BeforeAll
+        public static void setUp() throws Exception {
+            NewPipe.init(DownloaderTestImpl.getInstance());
+            // setting instance might break test when running in parallel (!)
+            PeerTube.setInstance(new PeertubeInstance(INSTANCE, "tchncs.de"));
+            extractor = PeerTube.getStreamExtractor(URL);
+            extractor.fetchPage();
+        }
+
+        @Override public StreamExtractor extractor() { return extractor; }
+        @Override public StreamingService expectedService() { return PeerTube; }
+        @Override public String expectedName() { return "Bauinformatik 11 – Objekte und Methoden"; }
+        @Override public String expectedId() { return ID; }
+        @Override public String expectedUrlContains() { return INSTANCE + BASE_URL + ID; }
+        @Override public String expectedOriginalUrlContains() { return URL; }
+
+        @Override public StreamType expectedStreamType() { return StreamType.VIDEO_STREAM; }
+        @Override public String expectedUploaderName() { return "Martin Vogel"; }
+        @Override public String expectedUploaderUrl() { return "https://tube.tchncs.de/accounts/martin_vogel@tube.tchncs.de"; }
+        @Override public String expectedSubChannelName() { return "Bauinformatik mit Python"; }
+        @Override public String expectedSubChannelUrl() { return "https://tube.tchncs.de/video-channels/python"; }
+        @Override public List<String> expectedDescriptionContains() { // CRLF line ending
+            return Arrays.asList("Um", "Programme", "Variablen", "Funktionen", "Objekte", "Skript", "Wiederholung", "Listen");
+        }
+        @Override public long expectedLength() { return 1017; }
+        @Override public long expectedViewCountAtLeast() { return 20; }
+        @Nullable @Override public String expectedUploadDate() { return "2023-12-08 15:57:04.142"; }
+        @Nullable @Override public String expectedTextualUploadDate() { return "2023-12-08T15:57:04.142Z"; }
+        @Override public long expectedLikeCountAtLeast() { return 0; }
+        @Override public long expectedDislikeCountAtLeast() { return 0; }
+        @Override public boolean expectedHasSubtitles() { return false; }
+        @Override public String expectedHost() { return "tube.tchncs.de"; }
+        @Override public String expectedCategory() { return "Unknown"; }
+        @Override public String expectedLicence() { return "Unknown"; }
+        @Override public Locale expectedLanguageInfo() { return null; }
+        @Override public List<String> expectedTags() { return Arrays.asList("Attribute", "Bauinformatik", "Klassen", "Objekte", "Python"); }
+    }
 
     @BeforeAll
     public static void setUp() throws Exception {

extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamLinkHandlerFactoryTest.java

@@ -9,6 +9,7 @@ import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeStream
 
 import static org.junit.jupiter.api.Assertions.*;
 import static org.schabi.newpipe.extractor.ServiceList.PeerTube;
+import static org.schabi.newpipe.extractor.services.peertube.PeertubeLinkHandlerFactoryTestHelper.assertDoNotAcceptNonURLs;
 
 /**
  * Test for {@link PeertubeStreamLinkHandlerFactory}
@@ -71,5 +72,7 @@ public class PeertubeStreamLinkHandlerFactoryTest {
         // make sure playlists aren't accepted
         assertFalse(linkHandler.acceptUrl("https://framatube.org/w/p/dacdc4ef-5160-4846-9b70-a655880da667"));
         assertFalse(linkHandler.acceptUrl("https://framatube.org/videos/watch/playlist/dacdc4ef-5160-4846-9b70-a655880da667"));
+
+        PeertubeLinkHandlerFactoryTestHelper.assertDoNotAcceptNonURLs(linkHandler);
     }
 }

extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeTrendingLinkHandlerFactoryTest.java

@@ -11,6 +11,7 @@ import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeTrendi
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertTrue;
 import static org.schabi.newpipe.extractor.ServiceList.PeerTube;
+import static org.schabi.newpipe.extractor.services.peertube.PeertubeLinkHandlerFactoryTestHelper.assertDoNotAcceptNonURLs;
 
 /**
  * Test for {@link PeertubeTrendingLinkHandlerFactory}
@@ -32,7 +33,7 @@ public class PeertubeTrendingLinkHandlerFactoryTest {
         assertEquals(LinkHandlerFactory.fromId("Trending").getUrl(), "https://peertube.mastodon.host/api/v1/videos?sort=-trending");
         assertEquals(LinkHandlerFactory.fromId("Most liked").getUrl(), "https://peertube.mastodon.host/api/v1/videos?sort=-likes");
         assertEquals(LinkHandlerFactory.fromId("Recently added").getUrl(), "https://peertube.mastodon.host/api/v1/videos?sort=-publishedAt");
-        assertEquals(LinkHandlerFactory.fromId("Local").getUrl(), "https://peertube.mastodon.host/api/v1/videos?sort=-publishedAt&filter=local");
+        assertEquals(LinkHandlerFactory.fromId("Local").getUrl(), "https://peertube.mastodon.host/api/v1/videos?sort=-publishedAt&isLocal=true");
     }
 
     @Test
@@ -57,5 +58,7 @@ public class PeertubeTrendingLinkHandlerFactoryTest {
 
         assertTrue(LinkHandlerFactory.acceptUrl("https://peertube.mastodon.host/videos/local"));
         assertTrue(LinkHandlerFactory.acceptUrl("https://peertube.mastodon.host/videos/local?adsf=fjaj#fhe"));
+
+        PeertubeLinkHandlerFactoryTestHelper.assertDoNotAcceptNonURLs(LinkHandlerFactory);
     }
 }