Merge pull request #628 from litetex/fix-broken-yt-liked-comments

Fix broken yt likes in comments

extractor/src/main/java/org/schabi/newpipe/extractor/comments/CommentsInfoItem.java

@@ -17,6 +17,7 @@ public class CommentsInfoItem extends InfoItem {
     @Nullable
     private DateWrapper uploadDate;
     private int likeCount;
+    private String textualLikeCount;
     private boolean heartedByUploader;
     private boolean pinned;
 
@@ -89,6 +90,14 @@ public class CommentsInfoItem extends InfoItem {
         this.likeCount = likeCount;
     }
 
+    public String getTextualLikeCount() {
+        return textualLikeCount;
+    }
+
+    public void setTextualLikeCount(String textualLikeCount) {
+        this.textualLikeCount = textualLikeCount;
+    }
+
     public void setHeartedByUploader(boolean isHeartedByUploader) {
         this.heartedByUploader = isHeartedByUploader;
     }

extractor/src/main/java/org/schabi/newpipe/extractor/comments/CommentsInfoItemExtractor.java

@@ -3,30 +3,50 @@ package org.schabi.newpipe.extractor.comments;
 import org.schabi.newpipe.extractor.InfoItemExtractor;
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
 import org.schabi.newpipe.extractor.localization.DateWrapper;
+import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeCommentsInfoItemExtractor;
 import org.schabi.newpipe.extractor.stream.StreamExtractor;
+import org.schabi.newpipe.extractor.utils.Utils;
 
 import javax.annotation.Nullable;
 
 public interface CommentsInfoItemExtractor extends InfoItemExtractor {
 
     /**
-     * Return the like count of the comment, or -1 if it's unavailable
+     * Return the like count of the comment, or -1 if it's unavailable<br/>
+     *
+     * NOTE: Currently only implemented for YT {@link YoutubeCommentsInfoItemExtractor#getLikeCount()}
+     * with limitations (only approximate like count is returned)
      *
      * @see StreamExtractor#getLikeCount()
      */
-    int getLikeCount() throws ParsingException;
+    default int getLikeCount() throws ParsingException {
+        return -1;
+    }
+
+    /**
+     * The unmodified like count given by the service<br/>
+     *
+     * It may be language dependent
+     */
+    default String getTextualLikeCount() throws ParsingException {
+        return Utils.EMPTY_STRING;
+    }
 
     /**
      * The text of the comment
      */
-    String getCommentText() throws ParsingException;
+    default String getCommentText() throws ParsingException {
+        return Utils.EMPTY_STRING;
+    }
 
     /**
      * The upload date given by the service, unmodified
      *
      * @see StreamExtractor#getTextualUploadDate()
      */
-    String getTextualUploadDate() throws ParsingException;
+    default String getTextualUploadDate() throws ParsingException {
+        return Utils.EMPTY_STRING;
+    }
 
     /**
      * The upload date wrapped with DateWrapper class
@@ -34,28 +54,44 @@ public interface CommentsInfoItemExtractor extends InfoItemExtractor {
      * @see StreamExtractor#getUploadDate()
      */
     @Nullable
-    DateWrapper getUploadDate() throws ParsingException;
+    default DateWrapper getUploadDate() throws ParsingException {
+        return null;
+    }
 
-    String getCommentId() throws ParsingException;
+    default String getCommentId() throws ParsingException {
+        return Utils.EMPTY_STRING;
+    }
 
-    String getUploaderUrl() throws ParsingException;
+    default String getUploaderUrl() throws ParsingException {
+        return Utils.EMPTY_STRING;
+    }
 
-    String getUploaderName() throws ParsingException;
+    default String getUploaderName() throws ParsingException {
+        return Utils.EMPTY_STRING;
+    }
 
-    String getUploaderAvatarUrl() throws ParsingException;
+    default String getUploaderAvatarUrl() throws ParsingException {
+        return Utils.EMPTY_STRING;
+    }
 
     /**
      * Whether the comment has been hearted by the uploader
      */
-    boolean isHeartedByUploader() throws ParsingException;
+    default boolean isHeartedByUploader() throws ParsingException {
+        return false;
+    }
 
     /**
      * Whether the comment is pinned
      */
-    boolean isPinned() throws ParsingException;
+    default boolean isPinned() throws ParsingException {
+        return false;
+    }
 
     /**
      * Whether the uploader is verified by the service
      */
-    boolean isUploaderVerified() throws ParsingException;
+    default boolean isUploaderVerified() throws ParsingException {
+        return false;
+    }
 }

extractor/src/main/java/org/schabi/newpipe/extractor/comments/CommentsInfoItemsCollector.java

@@ -6,7 +6,6 @@ import org.schabi.newpipe.extractor.exceptions.ParsingException;
 
 import java.util.ArrayList;
 import java.util.List;
-import java.util.Vector;
 
 public class CommentsInfoItemsCollector extends InfoItemsCollector<CommentsInfoItem, CommentsInfoItemExtractor> {
 
@@ -65,6 +64,11 @@ public class CommentsInfoItemsCollector extends InfoItemsCollector<CommentsInfoI
         } catch (Exception e) {
             addError(e);
         }
+        try {
+            resultItem.setTextualLikeCount(extractor.getTextualLikeCount());
+        } catch (Exception e) {
+            addError(e);
+        }
         try {
             resultItem.setThumbnailUrl(extractor.getThumbnailUrl());
         } catch (Exception e) {

extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampCommentsInfoItemExtractor.java

@@ -3,9 +3,6 @@ package org.schabi.newpipe.extractor.services.bandcamp.extractors;
 import org.jsoup.nodes.Element;
 import org.schabi.newpipe.extractor.comments.CommentsInfoItemExtractor;
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
-import org.schabi.newpipe.extractor.localization.DateWrapper;
-
-import javax.annotation.Nullable;
 
 public class BandcampCommentsInfoItemExtractor implements CommentsInfoItemExtractor {
 
@@ -32,39 +29,11 @@ public class BandcampCommentsInfoItemExtractor implements CommentsInfoItemExtrac
         return writing.getElementsByClass("thumb").attr("src");
     }
 
-    @Override
-    public int getLikeCount() {
-        return -1;
-    }
-
     @Override
     public String getCommentText() {
         return writing.getElementsByClass("text").first().ownText();
     }
 
-    @Override
-    public String getTextualUploadDate() {
-        return "";
-    }
-
-    @Nullable
-    @Override
-    public DateWrapper getUploadDate() {
-        return null;
-    }
-
-    @Override
-    public String getCommentId() {
-        return "";
-    }
-
-    @Override
-    public String getUploaderUrl() {
-        //return writing.getElementsByClass("name").attr("href");
-        // Fan links cannot be opened
-        return "";
-    }
-
     @Override
     public String getUploaderName() throws ParsingException {
         return writing.getElementsByClass("name").first().text();
@@ -74,19 +43,4 @@ public class BandcampCommentsInfoItemExtractor implements CommentsInfoItemExtrac
     public String getUploaderAvatarUrl() {
         return writing.getElementsByClass("thumb").attr("src");
     }
-
-    @Override
-    public boolean isHeartedByUploader() {
-        return false;
-    }
-
-    @Override
-    public boolean isPinned() {
-        return false;
-    }
-
-    @Override
-    public boolean isUploaderVerified() throws ParsingException {
-        return false;
-    }
 }

extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeCommentsInfoItemExtractor.java

@@ -57,11 +57,6 @@ public class PeertubeCommentsInfoItemExtractor implements CommentsInfoItemExtrac
         return new DateWrapper(PeertubeParsingHelper.parseDateFrom(textualUploadDate));
     }
 
-    @Override
-    public int getLikeCount() {
-        return -1;
-    }
-
     @Override
     public String getCommentText() throws ParsingException {
         final String htmlText = JsonUtils.getString(item, "text");
@@ -89,21 +84,6 @@ public class PeertubeCommentsInfoItemExtractor implements CommentsInfoItemExtrac
         return baseUrl + value;
     }
 
-    @Override
-    public boolean isHeartedByUploader() throws ParsingException {
-        return false;
-    }
-
-    @Override
-    public boolean isPinned() throws ParsingException {
-        return false;
-    }
-
-    @Override
-    public boolean isUploaderVerified() throws ParsingException {
-        return false;
-    }
-
     @Override
     public String getUploaderName() throws ParsingException {
         return JsonUtils.getString(item, "account.name") + "@" + JsonUtils.getString(item, "account.host");

extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudCommentsInfoItemExtractor.java

@@ -38,16 +38,6 @@ public class SoundcloudCommentsInfoItemExtractor implements CommentsInfoItemExtr
         return json.getObject("user").getString("avatar_url");
     }
 
-    @Override
-    public boolean isHeartedByUploader() throws ParsingException {
-        return false;
-    }
-
-    @Override
-    public boolean isPinned() throws ParsingException {
-        return false;
-    }
-
     @Override
     public boolean isUploaderVerified() throws ParsingException {
         return json.getObject("user").getBoolean("verified");
@@ -69,11 +59,6 @@ public class SoundcloudCommentsInfoItemExtractor implements CommentsInfoItemExtr
         return new DateWrapper(SoundcloudParsingHelper.parseDateFrom(getTextualUploadDate()));
     }
 
-    @Override
-    public int getLikeCount() {
-        return -1;
-    }
-
     @Override
     public String getName() throws ParsingException {
         return json.getObject("user").getString("permalink");

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsInfoItemExtractor.java

@@ -2,6 +2,7 @@ package org.schabi.newpipe.extractor.services.youtube.extractors;
 
 import com.grack.nanojson.JsonArray;
 import com.grack.nanojson.JsonObject;
+
 import org.schabi.newpipe.extractor.comments.CommentsInfoItemExtractor;
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
 import org.schabi.newpipe.extractor.localization.DateWrapper;
@@ -70,12 +71,70 @@ public class YoutubeCommentsInfoItemExtractor implements CommentsInfoItemExtract
         }
     }
 
+    /**
+     * @implNote The method is parsing internally a localized string.<br/>
+     * <ul>
+     *     <li>
+     *         More than >1k likes will result in an inaccurate number
+     *     </li>
+     *     <li>
+     *         This will fail for other languages than English.
+     *         However as long as the Extractor only uses "en-GB"
+     *         (as seen in {@link org.schabi.newpipe.extractor.services.youtube.YoutubeService#SUPPORTED_LANGUAGES})
+     *         everything will work fine.
+     *      </li>
+     * </ul>
+     * <br/>
+     * Consider using {@link #getTextualLikeCount()}
+     */
     @Override
     public int getLikeCount() throws ParsingException {
+        // This may return a language dependent version, e.g. in German: 3,3 Mio
+        final String textualLikeCount = getTextualLikeCount();
         try {
-            return json.getInt("likeCount");
+            if (Utils.isBlank(textualLikeCount)) {
+                return 0;
+            }
+
+            return (int) Utils.mixedNumberWordToLong(textualLikeCount);
         } catch (Exception e) {
-            throw new ParsingException("Could not get like count", e);
+            throw new ParsingException("Unexpected error while converting textual like count to like count", e);
+        }
+    }
+
+    @Override
+    public String getTextualLikeCount() throws ParsingException {
+        /*
+         * Example results as of 2021-05-20:
+         * Language = English
+         * 3.3M
+         * 48K
+         * 1.4K
+         * 270K
+         * 19
+         * 6
+         *
+         * Language = German
+         * 3,3 Mio
+         * 48.189
+         * 1419
+         * 270.984
+         * 19
+         * 6
+         */
+        try {
+            // If a comment has no likes voteCount is not set
+            if (!json.has("voteCount")) {
+                return EMPTY_STRING;
+            }
+
+            final JsonObject voteCountObj = JsonUtils.getObject(json, "voteCount");
+            if (voteCountObj.isEmpty()) {
+                return EMPTY_STRING;
+            }
+            return getTextFromObject(voteCountObj);
+        } catch (Exception e) {
+            throw new ParsingException("Could not get vote count", e);
         }
     }
 

extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampCommentsExtractorTest.java

@@ -13,10 +13,10 @@ import org.schabi.newpipe.extractor.utils.Utils;
 
 import java.io.IOException;
 
-import static org.junit.Assert.*;
+import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
 import static org.schabi.newpipe.extractor.ServiceList.Bandcamp;
-import static org.schabi.newpipe.extractor.ServiceList.YouTube;
 
 public class BandcampCommentsExtractorTest {
 
@@ -47,6 +47,7 @@ public class BandcampCommentsExtractorTest {
             assertFalse(Utils.isBlank(c.getThumbnailUrl()));
             assertFalse(Utils.isBlank(c.getUrl()));
             assertEquals(-1, c.getLikeCount());
+            assertTrue(Utils.isBlank(c.getTextualLikeCount()));
         }
     }
 }

extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeCommentsExtractorTest.java

@@ -75,7 +75,8 @@ public class PeertubeCommentsExtractorTest {
                 assertFalse(Utils.isBlank(c.getTextualUploadDate()));
                 assertFalse(Utils.isBlank(c.getThumbnailUrl()));
                 assertFalse(Utils.isBlank(c.getUrl()));
-                assertFalse(c.getLikeCount() != -1);
+                assertEquals(-1, c.getLikeCount());
+                assertTrue(Utils.isBlank(c.getTextualLikeCount()));
             }
         }
 

extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeCommentsExtractorTest.java

@@ -9,12 +9,14 @@ import org.schabi.newpipe.extractor.Page;
 import org.schabi.newpipe.extractor.comments.CommentsInfo;
 import org.schabi.newpipe.extractor.comments.CommentsInfoItem;
 import org.schabi.newpipe.extractor.exceptions.ExtractionException;
+import org.schabi.newpipe.extractor.localization.Localization;
 import org.schabi.newpipe.extractor.services.DefaultTests;
 import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeCommentsExtractor;
 import org.schabi.newpipe.extractor.utils.Utils;
 
 import java.io.IOException;
 import java.util.List;
+import java.util.Locale;
 import java.util.Random;
 
 import static org.junit.Assert.assertEquals;
@@ -98,7 +100,7 @@ public class YoutubeCommentsExtractorTest {
                 assertNotNull(c.getUploadDate());
                 assertFalse(Utils.isBlank(c.getThumbnailUrl()));
                 assertFalse(Utils.isBlank(c.getUrl()));
-                assertFalse(c.getLikeCount() < 0);
+                assertTrue(c.getLikeCount() >= 0);
             }
         }
 
@@ -148,7 +150,7 @@ public class YoutubeCommentsExtractorTest {
                 assertNotNull(c.getUploadDate());
                 assertFalse(Utils.isBlank(c.getThumbnailUrl()));
                 assertFalse(Utils.isBlank(c.getUrl()));
-                assertFalse(c.getLikeCount() < 0);
+                assertTrue(c.getLikeCount() >= 0);
                 if (c.getCommentId().equals("Ugga_h1-EXdHB3gCoAEC")) { // comment without text
                     assertTrue(Utils.isBlank(c.getCommentText()));
                 } else {
@@ -191,7 +193,7 @@ public class YoutubeCommentsExtractorTest {
                 assertNotNull(c.getUploadDate());
                 assertFalse(Utils.isBlank(c.getThumbnailUrl()));
                 assertFalse(Utils.isBlank(c.getUrl()));
-                assertFalse(c.getLikeCount() < 0);
+                assertTrue(c.getLikeCount() >= 0);
                 assertFalse(Utils.isBlank(c.getCommentText()));
                 if (c.isHeartedByUploader()) {
                     heartedByUploader = true;
@@ -232,11 +234,76 @@ public class YoutubeCommentsExtractorTest {
                 assertNotNull(c.getUploadDate());
                 assertFalse(Utils.isBlank(c.getThumbnailUrl()));
                 assertFalse(Utils.isBlank(c.getUrl()));
-                assertFalse(c.getLikeCount() < 0);
+                assertTrue(c.getLikeCount() >= 0);
                 assertFalse(Utils.isBlank(c.getCommentText()));
             }
 
             assertTrue("First comment isn't pinned", comments.getItems().get(0).isPinned());
         }
     }
+
+    /**
+     * Checks if the likes/votes are handled correctly<br/>
+     * A pinned comment with >15K likes is used for the test
+     */
+    public static class LikesVotes {
+        private final static String url = "https://www.youtube.com/watch?v=QqsLTNkzvaY";
+        private static YoutubeCommentsExtractor extractor;
+
+        @BeforeClass
+        public static void setUp() throws Exception {
+            YoutubeParsingHelper.resetClientVersionAndKey();
+            YoutubeParsingHelper.setNumberGenerator(new Random(1));
+            NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "likes"));
+            extractor = (YoutubeCommentsExtractor) YouTube
+                    .getCommentsExtractor(url);
+            extractor.fetchPage();
+        }
+
+        @Test
+        public void testGetCommentsFirst() throws IOException, ExtractionException {
+            final InfoItemsPage<CommentsInfoItem> comments = extractor.getInitialPage();
+
+            DefaultTests.defaultTestListOfItems(YouTube, comments.getItems(), comments.getErrors());
+
+            CommentsInfoItem pinnedComment = comments.getItems().get(0);
+
+            assertTrue("First comment isn't pinned", pinnedComment.isPinned());
+            assertTrue("The first pinned comment has no likes", pinnedComment.getLikeCount() > 0);
+            assertTrue("The first pinned comment has no vote count", !Utils.isBlank(pinnedComment.getTextualLikeCount()));
+        }
+    }
+
+    /**
+     * Checks if the vote count works localized<br/>
+     * A pinned comment with >15K likes is used for the test
+     */
+    public static class LocalizedVoteCount {
+        private final static String url = "https://www.youtube.com/watch?v=QqsLTNkzvaY";
+        private static YoutubeCommentsExtractor extractor;
+
+        @BeforeClass
+        public static void setUp() throws Exception {
+            YoutubeParsingHelper.resetClientVersionAndKey();
+            YoutubeParsingHelper.setNumberGenerator(new Random(1));
+            NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "localized_vote_count"));
+            extractor = (YoutubeCommentsExtractor) YouTube
+                    .getCommentsExtractor(url);
+            // Force non english local here
+            extractor.forceLocalization(Localization.fromLocale(Locale.GERMANY));
+            extractor.fetchPage();
+        }
+
+        @Test
+        public void testGetCommentsFirst() throws IOException, ExtractionException {
+            final InfoItemsPage<CommentsInfoItem> comments = extractor.getInitialPage();
+
+            DefaultTests.defaultTestListOfItems(YouTube, comments.getItems(), comments.getErrors());
+
+            CommentsInfoItem pinnedComment = comments.getItems().get(0);
+
+            assertTrue("First comment isn't pinned", pinnedComment.isPinned());
+            assertTrue("The first pinned comment has no vote count", !Utils.isBlank(pinnedComment.getTextualLikeCount()));
+        }
+    }
 }