From ce698b4d9c273ae849001e8f04d2867c2dd331f3 Mon Sep 17 00:00:00 2001
From: Christian Schabesberger <chris.schabesberger@mailbox.org>
Date: Sat, 24 Feb 2018 22:18:12 +0100
Subject: [PATCH] add more information about lits collector/extractor pattern

---
 assets/InfoItemsCollector_objectdiagram.dia   | Bin 0 -> 1263 bytes
 docs/01_Basic_concept_of_the_extractor.md     |  35 --------
 docs/01_Concept_of_the_extractor.md           |  79 ++++++++++++++++++
 docs/img/InfoItemsCollector_objectdiagram.svg |  39 +++++++++
 4 files changed, 118 insertions(+), 35 deletions(-)
 create mode 100644 assets/InfoItemsCollector_objectdiagram.dia
 delete mode 100644 docs/01_Basic_concept_of_the_extractor.md
 create mode 100644 docs/01_Concept_of_the_extractor.md
 create mode 100644 docs/img/InfoItemsCollector_objectdiagram.svg

diff --git a/assets/InfoItemsCollector_objectdiagram.dia b/assets/InfoItemsCollector_objectdiagram.dia
new file mode 100644
index 0000000000000000000000000000000000000000..f99ccafc4a963b5cd2c847d2a32532b818a84d84
GIT binary patch
literal 1263
zcmV<L1Q7cliwFP!000021MQnzZ{j!<fZzKoqV%;Ba)qQF7_C;j+L2~854-a+FUSxR
zJmbX3ak#wfZ(rv^AvYiip=IPqMU8uMj(y^D`9%Hk)7Hn*8qyGvU@U8jDoZe!6Bh;E
zSpN3!pC4@b)8yf!i@-Dg@hI>m{zMSQcVl_U*y?#O*lacm-i3e>s$di;AsqY%Fa`rI
zG>|6`l60)V1q{Sv*(hL)qFKbC6aXKN<r$d&^eBk}S5BoexjDgvN^5|}^5Y`8<Uyup
zkkfNQ_X>D$Mj`lFuUh4o<Frw|g0!gCUy%@T5w=^EL~8WJpGPv8RLF$_Z}Rw?`It^A
z8#w5ynrN*N<^$@XplpZx2B(hD6<gPJ?hMPZ9d%@mw7pN7o15L%9KNkNdRucSoURCE
z6d+bMIU@u^5TsTajbM8_;T&MT#ZE1&!zvbtF>+@93xLB*7LX3VMqAU!^(b;r2X0QH
zM#~1dY&qTb<gQfvt|#ql6rvf1RkKF{yPEplb<`K%PLjP3k9XV+skYkk;M&lFR`c@U
zMaYHWnQfMrsnN8|#0KZQUX;7kG3ijDSSKbh*g=}9|BJsmNIe?YX(@ZdS;~>XoT2sp
z?#L-*_5;pYw%Tuhe~~^&Uy~vE(1P#=1i54RtD5&!adli3PlF|At;k_VsZW|wTzxrr
z)G43Min=MEC<r_ZQ~i$OSWlYHKVP(wqG|lbRBWrAMISOCpV{Xw;*M&i9b)KD=Y$63
z_TxGZOAI7As_e9Vei|w2&~%P2tAnj2MBb9s8`Bh9&0V8*#&`vU=N6_aTHcnbwKK-x
zmQCYi6VdWs$ako!+F?EYnphx=FEEjB;hQ_^T(lHq>YALSjn2sxI_~Gs!Ge5d&<}qn
z81ocBXiW=#qHfi>iUh36c04XEfRFHwPZ0#6EQM?r=U1K`v-At`8ZtBo$916j3`%iS
zxxWjsq?+A~=JR%CDl!zy(36a+9_NWR*761Y!VTZX5O0^gz=zeZ6b6xB(dnG(b*jpB
zvp5NPg@z=KhhntscX#DhZj_C;Ap@*G{ptO!f?Y&*-;u38slQe@tG5ehPX1|UT?~g)
z)Z8dvEXih?4u*Z%@FUF73U^;aI+!9wOUBT;GKS7uX0=Dg&=SaqKVEc4QMI@vYIhA8
z9XxgkWY~%`)UE1-CLlvV#yf=!uF?Nv%P5HPLVJGwRrDi$_anNgn6{?JX;qMhAPqqp
zf;0qa2-3JVY3SD=4WoC`NWac_MGf=rQNs|ZAy7l0MjdMC0yW+MHN=-10yP9`2-Fa$
zAyDJ`s9{_KHHQ78hLuB&(cPoQP@slD4S^bUs9^}ycmvcJ3Dgj%Ay7l0hCmI08qHDT
zAKtgSfV!MFz9a8D@V@XS=!iMAj5*BChc%KD6y}p=C{CZ~5g$T`&G5*zUSbe&oKa%5
zC43OdmeXV%%~w|;XUG5cJiJ^^JtM~#XvPavTYEKLAE788#-~E~RJE>uu2D&gcn?V;
zmaj$Ujz6BVFegF4Hwxd<VX08={~|4UC}f3(k>r4qboeW0w9-N=p4GBvwc2O*X|6D#
zYX+-zZ^3H3&tT<p!%O5G#okZ+qF^;du)eEc)!#B$hxZ+<!+Wi)!wZ9TD1!A}1*_3D
ZSkwFDr1Rt<cNBW^@IS_hIGQd+004EoZn6LX

literal 0
HcmV?d00001

diff --git a/docs/01_Basic_concept_of_the_extractor.md b/docs/01_Basic_concept_of_the_extractor.md
deleted file mode 100644
index a7a096b..0000000
--- a/docs/01_Basic_concept_of_the_extractor.md
+++ /dev/null
@@ -1,35 +0,0 @@
-# Basic Concept of the Extractor
-
-## Collector/Extractor pattern
-
-Before we can start coding our own service we need to understand the basic concept of the extractor. There is a pattern
-you will find all over the code. It is called the __extractor/collector__ pattern. The idea behind this pattern is that
-the [extractor](https://teamnewpipe.github.io/NewPipeExtractor/javadoc/org/schabi/newpipe/extractor/Extractor.html)
-would produce single peaces of data, and the collector would take it and form usable data for the front end out of it.
-The collector also controls the parsing process, and takes care about error handling. So if the extractor fails at any
-point the collector will decide whether it should continue parsing or not. This requires the extractor to be made out of
-many small methods. One method for every data field the collector wants to have. The collectors are provided by NewPipe.
-You need to take care of the extractors.
-
-### Usage in the front end
-
-So typical call for retrieving data from a website would look like this:
-```java
-Info info;
-try {
-    // Create a new Extractor with a given context provided as parameter.
-    Extractor extractor = new Extractor(ome_meta_info);
-    // Retrieves the data form extractor and builds info package.
-    info = Info.getInfo(extractor);
-} catch(Exception e) {
-    // handle errors when collector decided to break up extraction
-}
-```
-
-
-
-
-
-
-
-
diff --git a/docs/01_Concept_of_the_extractor.md b/docs/01_Concept_of_the_extractor.md
new file mode 100644
index 0000000..3d0999b
--- /dev/null
+++ b/docs/01_Concept_of_the_extractor.md
@@ -0,0 +1,79 @@
+# Concept of the Extractor
+
+## Collector/Extractor pattern
+
+Before we can start coding our own service we need to understand the basic concept of the extractor. There is a pattern
+you will find all over the code. It is called the __extractor/collector__ pattern. The idea behind this pattern is that
+the [extractor](https://teamnewpipe.github.io/NewPipeExtractor/javadoc/org/schabi/newpipe/extractor/Extractor.html)
+would produce single peaces of data, and the collector would take it and form usable data for the front end out of it.
+The collector also controls the parsing process, and takes care about error handling. So if the extractor fails at any
+point the collector will decide whether it should continue parsing or not. This requires the extractor to be made out of
+many small methods. One method for every data field the collector wants to have. The collectors are provided by NewPipe.
+You need to take care of the extractors.
+
+### Usage in the front end
+
+So typical call for retrieving data from a website would look like this:
+```java
+Info info;
+try {
+    // Create a new Extractor with a given context provided as parameter.
+    Extractor extractor = new Extractor(some_meta_info);
+    // Retrieves the data form extractor and builds info package.
+    info = Info.getInfo(extractor);
+} catch(Exception e) {
+    // handle errors when collector decided to break up extraction
+}
+```
+
+### Typical implementation of a single data extractor
+
+The typical implementation of a single data extractor on the other hand would look like this:
+```java
+class MyExtractor extends FutureExtractor {
+
+    public MyExtractor(RequiredInfo requiredInfo, ForExtraction forExtraction) {
+        super(requiredInfo, forExtraction);
+
+        ...
+    }
+
+    @Override
+    public void fetch() {
+        // Actually fetch the page data here
+    }
+
+    @Override
+    public String someDataFiled() 
+        throws ExtractionException {    //The exception needs to be thrown if someting failed
+        // get piece of information and return it
+    }
+
+    ...                                 // More datafields
+}
+```
+
+## Collector/Extractor pattern for lists
+
+Sometimes information can not be represented as a structure, but as a list. In NewPipe an item of a list is called
+[InfoItem](https://teamnewpipe.github.io/NewPipeExtractor/javadoc/org/schabi/newpipe/extractor/InfoItem.html). In order
+to get such items a [InfoItemsCollector](https://teamnewpipe.github.io/NewPipeExtractor/javadoc/org/schabi/newpipe/extractor/InfoItemCollector.html)
+is used. For each item that should be extracted a new Extractor will be given to the InfoItemCollector via [commit()](https://teamnewpipe.github.io/NewPipeExtractor/javadoc/org/schabi/newpipe/extractor/InfoItemCollector.html#commit-E-).
+
+![InfoItemsCollector_objectdiagram.svg](img/InfoItemsCollector_objectdiagram.svg)
+
+When a streaming site shows a list it usually offers some additional information about that list, like it's title, a thumbnail
+or its creator. Such info can be called __list header__.
+
+Also if you open a list in a web browser the website usually does not load the whole list, but only a part
+of it. In order to get more you may have to click on a next page button, or scroll down. This is why a list in
+NewPipe is coped down into [InfoItemPage](https://teamnewpipe.github.io/NewPipeExtractor/javadoc/org/schabi/newpipe/extractor/ListExtractor.NextItemsResult.html)s. Each Page has its own URL, and needs to be extracted separately.
+
+List header information and extracting multiple pages of an InfoItem list can be handled by a
+[ListExtractor](https://teamnewpipe.github.io/NewPipeExtractor/javadoc/org/schabi/newpipe/extractor/ListExtractor.html)
+
+
+
+
+
+
diff --git a/docs/img/InfoItemsCollector_objectdiagram.svg b/docs/img/InfoItemsCollector_objectdiagram.svg
new file mode 100644
index 0000000..2f986a9
--- /dev/null
+++ b/docs/img/InfoItemsCollector_objectdiagram.svg
@@ -0,0 +1,39 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.0//EN" "http://www.w3.org/TR/2001/PR-SVG-20010719/DTD/svg10.dtd">
+<svg width="20cm" height="8cm" viewBox="199 199 382 159" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink">
+  <g>
+    <rect style="fill: #ffffff" x="200" y="260" width="141.3" height="36"/>
+    <rect style="fill: none; fill-opacity:0; stroke-width: 2; stroke: #000000" x="200" y="260" width="141.3" height="36"/>
+    <text font-size="12.7998" style="fill: #000000;text-anchor:middle;font-family:sans-serif;font-style:normal;font-weight:normal" x="270.65" y="281.9">
+      <tspan x="270.65" y="281.9">:InfoItemsCollector</tspan>
+    </text>
+    <line style="fill: none; fill-opacity:0; stroke-width: 1; stroke: #000000" x1="210" y1="284.9" x2="331.3" y2="284.9"/>
+  </g>
+  <g>
+    <rect style="fill: #ffffff" x="400" y="200" width="179.25" height="36"/>
+    <rect style="fill: none; fill-opacity:0; stroke-width: 2; stroke: #000000" x="400" y="200" width="179.25" height="36"/>
+    <text font-size="12.7998" style="fill: #000000;text-anchor:middle;font-family:sans-serif;font-style:normal;font-weight:normal" x="489.625" y="221.9">
+      <tspan x="489.625" y="221.9">itemExtractor1:Extractor</tspan>
+    </text>
+    <line style="fill: none; fill-opacity:0; stroke-width: 1; stroke: #000000" x1="410" y1="224.9" x2="569.25" y2="224.9"/>
+  </g>
+  <g>
+    <rect style="fill: #ffffff" x="400" y="260" width="179.25" height="36"/>
+    <rect style="fill: none; fill-opacity:0; stroke-width: 2; stroke: #000000" x="400" y="260" width="179.25" height="36"/>
+    <text font-size="12.7998" style="fill: #000000;text-anchor:middle;font-family:sans-serif;font-style:normal;font-weight:normal" x="489.625" y="281.9">
+      <tspan x="489.625" y="281.9">itemExtractor2:Extractor</tspan>
+    </text>
+    <line style="fill: none; fill-opacity:0; stroke-width: 1; stroke: #000000" x1="410" y1="284.9" x2="569.25" y2="284.9"/>
+  </g>
+  <g>
+    <rect style="fill: #ffffff" x="400" y="320" width="179.25" height="36"/>
+    <rect style="fill: none; fill-opacity:0; stroke-width: 2; stroke: #000000" x="400" y="320" width="179.25" height="36"/>
+    <text font-size="12.7998" style="fill: #000000;text-anchor:middle;font-family:sans-serif;font-style:normal;font-weight:normal" x="489.625" y="341.9">
+      <tspan x="489.625" y="341.9">itemExtractor3:Extractor</tspan>
+    </text>
+    <line style="fill: none; fill-opacity:0; stroke-width: 1; stroke: #000000" x1="410" y1="344.9" x2="569.25" y2="344.9"/>
+  </g>
+  <polyline style="fill: none; fill-opacity:0; stroke-width: 2; stroke: #000000" points="342.309,278 370.652,278 370.652,218 398.994,218 "/>
+  <polyline style="fill: none; fill-opacity:0; stroke-width: 2; stroke: #000000" points="342.309,278 343.309,278 397.994,278 398.994,278 "/>
+  <polyline style="fill: none; fill-opacity:0; stroke-width: 2; stroke: #000000" points="342.309,278 370.652,278 370.652,338 398.994,338 "/>
+</svg>