be-gay-do-crimes
/
NewPipeExtractor
mirror of https://github.com/TeamNewPipe/NewPipeExtractor
1
0
Fork 0
Code Issues Releases Wiki Activity

Only store ajaxJson for initial page and eager fetch the initial continuation.

This commit is contained in:
Kavin 2022-12-08 11:44:47 +00:00
parent 64d24aa09e
commit 2974dfaa48
No known key found for this signature in database
GPG Key ID: 49451E4482CC5BCD
2 changed files with 43 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

63
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsExtractor.java
View File

@ -30,17 +30,16 @@ import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
public class YoutubeCommentsExtractor extends CommentsExtractor {
/**
* The initial request's continuation token.
* Since we need to make two requests to get the comments,
*/
private String initialToken;
/**
* Whether comments are disabled on video.
*/
private boolean commentsDisabled = true;
/**
* The total number of comments on video.
*/
private int commentsCount = (int) ITEM_COUNT_UNKNOWN;
/**
* The second ajax <b>/next</b> response.
*/
@ -62,7 +61,7 @@ public class YoutubeCommentsExtractor extends CommentsExtractor {
return getInfoItemsPageForDisabledComments();
}
return getPage(getNextPage(this.initialToken));
return extractComments(ajaxJson);
}
/**
@ -186,12 +185,17 @@ public class YoutubeCommentsExtractor extends CommentsExtractor {
.getBytes(StandardCharsets.UTF_8);
// @formatter:on
this.ajaxJson = getJsonPostResponse("next", body, localization);
final var jsonObject = getJsonPostResponse("next", body, localization);
return extractComments(jsonObject);
}
private InfoItemsPage<CommentsInfoItem> extractComments(final JsonObject jsonObject)
throws ExtractionException {
final CommentsInfoItemsCollector collector = new CommentsInfoItemsCollector(
getServiceId());
collectCommentsFrom(collector);
return new InfoItemsPage<>(collector, getNextPage(ajaxJson));
return new InfoItemsPage<>(collector, getNextPage(jsonObject));
}
private void collectCommentsFrom(final CommentsInfoItemsCollector collector)
@ -261,7 +265,18 @@ public class YoutubeCommentsExtractor extends CommentsExtractor {
.getBytes(StandardCharsets.UTF_8);
// @formatter:on
initialToken = findInitialCommentsToken(getJsonPostResponse("next", body, localization));
final String initialToken =
findInitialCommentsToken(getJsonPostResponse("next", body, localization));
// @formatter:off
final byte[] ajaxBody = JsonWriter.string(
prepareDesktopJsonBuilder(localization, getExtractorContentCountry())
.value("continuation", initialToken)
.done())
.getBytes(StandardCharsets.UTF_8);
// @formatter:on
ajaxJson = getJsonPostResponse("next", ajaxBody, localization);
}
@ -272,17 +287,25 @@ public class YoutubeCommentsExtractor extends CommentsExtractor {
@Override
public int getCommentsCount() throws ExtractionException {
final JsonObject countText = ajaxJson
.getArray("onResponseReceivedEndpoints").getObject(0)
.getObject("reloadContinuationItemsCommand")
.getArray("continuationItems").getObject(0)
.getObject("commentsHeaderRenderer")
.getObject("countText");
assertPageFetched();
try {
return Integer.parseInt(Utils.removeNonDigitCharacters(getTextFromObject(countText)));
} catch (final Exception e) {
throw new ExtractionException("Unable to get comments count", e);
if (commentsCount == ITEM_COUNT_UNKNOWN) {
final JsonObject countText = ajaxJson
.getArray("onResponseReceivedEndpoints").getObject(0)
.getObject("reloadContinuationItemsCommand")
.getArray("continuationItems").getObject(0)
.getObject("commentsHeaderRenderer")
.getObject("countText");
try {
commentsCount = Integer.parseInt(
Utils.removeNonDigitCharacters(getTextFromObject(countText))
);
} catch (final Exception e) {
throw new ExtractionException("Unable to get comments count", e);
}
}
return commentsCount;
}
}

1
extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeCommentsExtractorTest.java
View File

@ -348,7 +348,6 @@ public class YoutubeCommentsExtractorTest {
@Test
public void testCommentsCount() throws IOException, ExtractionException {
extractor.getInitialPage(); // Needs to be called first
assertTrue(extractor.getCommentsCount() > 18800);
}
}