Merge pull request #589 from fynngodau/bandcamp-more-recent-pages

Load more featured pages and fix featured cover arts not loading

extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampFeaturedExtractor.java

@@ -25,6 +25,7 @@ public class BandcampFeaturedExtractor extends KioskExtractor<PlaylistInfoItem>
 
     public static final String KIOSK_FEATURED = "Featured";
     public static final String FEATURED_API_URL = BASE_API_URL + "/mobile/24/bootstrap_data";
+    public static final String MORE_FEATURED_API_URL = BASE_API_URL + "/mobile/24/feed_older_logged_out";
 
     private JsonObject json;
 
@@ -56,12 +57,18 @@ public class BandcampFeaturedExtractor extends KioskExtractor<PlaylistInfoItem>
     @Override
     public InfoItemsPage<PlaylistInfoItem> getInitialPage() throws IOException, ExtractionException {
 
-        final PlaylistInfoItemsCollector c = new PlaylistInfoItemsCollector(getServiceId());
 
         final JsonArray featuredStories = json.getObject("feed_content")
                 .getObject("stories")
                 .getArray("featured");
 
+        return extractItems(featuredStories);
+    }
+
+    private InfoItemsPage<PlaylistInfoItem> extractItems(JsonArray featuredStories) {
+
+        final PlaylistInfoItemsCollector c = new PlaylistInfoItemsCollector(getServiceId());
+
         for (int i = 0; i < featuredStories.size(); i++) {
             final JsonObject featuredStory = featuredStories.getObject(i);
 
@@ -73,12 +80,36 @@ public class BandcampFeaturedExtractor extends KioskExtractor<PlaylistInfoItem>
             c.commit(new BandcampPlaylistInfoItemFeaturedExtractor(featuredStory));
         }
 
-        return new InfoItemsPage<>(c, null);
+        final JsonObject lastFeaturedStory = featuredStories.getObject(featuredStories.size() - 1);
 
+        return new InfoItemsPage<>(c, getNextPageFrom(lastFeaturedStory));
+    }
+
+    /**
+     * Next Page can be generated from metadata of last featured story
+     */
+    private Page getNextPageFrom(JsonObject lastFeaturedStory) {
+        final long lastStoryDate = lastFeaturedStory.getLong("story_date");
+        final long lastStoryId = lastFeaturedStory.getLong("ntid");
+        final String lastStoryType = lastFeaturedStory.getString("story_type");
+        return new Page(
+                MORE_FEATURED_API_URL + "?story_groups=featured"
+                        + ':' + lastStoryDate + ':' + lastStoryType + ':' + lastStoryId
+        );
     }
 
     @Override
-    public InfoItemsPage<PlaylistInfoItem> getPage(Page page) {
-        return null;
+    public InfoItemsPage<PlaylistInfoItem> getPage(Page page) throws IOException, ExtractionException {
+
+        JsonObject response;
+        try {
+            response = JsonParser.object().from(
+                    getDownloader().get(page.getUrl()).responseBody()
+            );
+        } catch (final JsonParserException e) {
+            throw new ParsingException("Could not parse Bandcamp featured API response", e);
+        }
+
+        return extractItems(response.getObject("stories").getArray("featured"));
     }
 }

extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampPlaylistInfoItemFeaturedExtractor.java

@@ -35,6 +35,7 @@ public class BandcampPlaylistInfoItemFeaturedExtractor implements PlaylistInfoIt
 
     @Override
     public String getThumbnailUrl() {
-        return featuredStory.has("art_id") ? getImageUrl(featuredStory.getLong("art_id"), true) : "";
+        return featuredStory.has("art_id") ? getImageUrl(featuredStory.getLong("art_id"), true)
+                : getImageUrl(featuredStory.getLong("item_art_id"), true);
     }
 }

extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampFeaturedExtractorTest.java

@@ -6,6 +6,7 @@ import org.junit.BeforeClass;
 import org.junit.Test;
 import org.schabi.newpipe.downloader.DownloaderTestImpl;
 import org.schabi.newpipe.extractor.NewPipe;
+import org.schabi.newpipe.extractor.Page;
 import org.schabi.newpipe.extractor.exceptions.ExtractionException;
 import org.schabi.newpipe.extractor.playlist.PlaylistInfoItem;
 import org.schabi.newpipe.extractor.services.BaseListExtractorTest;
@@ -16,6 +17,7 @@ import java.io.IOException;
 import java.util.List;
 
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotEquals;
 import static org.junit.Assert.assertTrue;
 import static org.schabi.newpipe.extractor.ServiceList.Bandcamp;
 
@@ -37,7 +39,7 @@ public class BandcampFeaturedExtractorTest implements BaseListExtractorTest {
     @Test
     public void testFeaturedCount() throws ExtractionException, IOException {
         final List<PlaylistInfoItem> list = extractor.getInitialPage().getItems();
-        assertTrue(list.size() > 1);
+        assertTrue(list.size() > 5);
     }
 
     @Test
@@ -46,6 +48,21 @@ public class BandcampFeaturedExtractorTest implements BaseListExtractorTest {
         assertTrue(list.get(0).getUrl().contains("https://"));
     }
 
+    @Test
+    public void testMorePages() throws ExtractionException, IOException {
+
+        final Page page2 = extractor.getInitialPage().getNextPage();
+        final Page page3 = extractor.getPage(page2).getNextPage();
+
+        assertTrue(extractor.getPage(page2).getItems().size() > 5);
+
+        // Compare first item of second page with first item of third page
+        assertNotEquals(
+                extractor.getPage(page2).getItems().get(0),
+                extractor.getPage(page3).getItems().get(0)
+        );
+    }
+
     @Override
     public void testRelatedItems() throws Exception {
         DefaultTests.defaultTestRelatedItems(extractor);