From 49cc643dccefc66585b883d6ce8b1db858f88ef0 Mon Sep 17 00:00:00 2001 From: kapodamy Date: Mon, 13 Jan 2020 21:25:26 -0300 Subject: [PATCH 1/9] decrease the size of samples per chunk --- .../newpipe/streams/Mp4FromDashWriter.java | 179 +++++++++++++----- 1 file changed, 129 insertions(+), 50 deletions(-) diff --git a/app/src/main/java/org/schabi/newpipe/streams/Mp4FromDashWriter.java b/app/src/main/java/org/schabi/newpipe/streams/Mp4FromDashWriter.java index 818f6148e..f94ebefb0 100644 --- a/app/src/main/java/org/schabi/newpipe/streams/Mp4FromDashWriter.java +++ b/app/src/main/java/org/schabi/newpipe/streams/Mp4FromDashWriter.java @@ -11,6 +11,7 @@ import org.schabi.newpipe.streams.io.SharpStream; import java.io.IOException; import java.nio.ByteBuffer; +import java.util.ArrayList; /** * @author kapodamy @@ -23,7 +24,6 @@ public class Mp4FromDashWriter { private final static byte SAMPLES_PER_CHUNK = 6;// ffmpeg uses 2, basic uses 1 (with 60fps uses 21 or 22). NewPipe will use 6 private final static long THRESHOLD_FOR_CO64 = 0xFFFEFFFFL;// near 3.999 GiB private final static int THRESHOLD_MOOV_LENGTH = (256 * 1024) + (2048 * 1024); // 2.2 MiB enough for: 1080p 60fps 00h35m00s - private final static short SINGLE_CHUNK_SAMPLE_BUFFER = 256; private final long time; @@ -46,6 +46,8 @@ public class Mp4FromDashWriter { private int overrideMainBrand = 0x00; + private ArrayList compatibleBrands = new ArrayList<>(5); + public Mp4FromDashWriter(SharpStream... sources) throws IOException { for (SharpStream src : sources) { if (!src.canRewind() && !src.canRead()) { @@ -57,6 +59,10 @@ public class Mp4FromDashWriter { readers = new Mp4DashReader[sourceTracks.length]; readersChunks = new Mp4DashChunk[readers.length]; time = (System.currentTimeMillis() / 1000L) + EPOCH_OFFSET; + + compatibleBrands.add(0x6D703431);// mp41 + compatibleBrands.add(0x69736F6D);// isom + compatibleBrands.add(0x69736F32);// iso2 } public Mp4Track[] getTracksFromSource(int sourceIndex) throws IllegalStateException { @@ -104,8 +110,8 @@ public class Mp4FromDashWriter { } } - public void setMainBrand(int brandId) { - overrideMainBrand = brandId; + public void setMainBrand(int brand) { + overrideMainBrand = brand; } public boolean isDone() { @@ -159,7 +165,13 @@ public class Mp4FromDashWriter { tablesInfo[i] = new TablesInfo(); } - boolean singleChunk = tracks.length == 1 && tracks[0].kind == TrackKind.Audio; + int single_sample_buffer; + if (tracks.length == 1 && tracks[0].kind == TrackKind.Audio) { + // near 1 second of audio data per chunk, avoid split the audio stream in large chunks + single_sample_buffer = tracks[0].trak.mdia.mdhd_timeScale / 1000; + } else { + single_sample_buffer = -1; + } for (int i = 0; i < readers.length; i++) { @@ -210,31 +222,10 @@ public class Mp4FromDashWriter { readers[i].rewind(); - int tmp = tablesInfo[i].stsz - SAMPLES_PER_CHUNK_INIT; - tablesInfo[i].stco = (tmp / SAMPLES_PER_CHUNK) + 1;// +1 for samples in first chunk - - tmp = tmp % SAMPLES_PER_CHUNK; - if (singleChunk) { - // avoid split audio streams in chunks - tablesInfo[i].stsc = 1; - tablesInfo[i].stsc_bEntries = new int[]{ - 1, tablesInfo[i].stsz, 1 - }; - tablesInfo[i].stco = 1; - } else if (tmp == 0) { - tablesInfo[i].stsc = 2;// first chunk (init) and succesive chunks - tablesInfo[i].stsc_bEntries = new int[]{ - 1, SAMPLES_PER_CHUNK_INIT, 1, - 2, SAMPLES_PER_CHUNK, 1 - }; + if (single_sample_buffer > 0) { + initChunkTables(tablesInfo[i], single_sample_buffer, single_sample_buffer); } else { - tablesInfo[i].stsc = 3;// first chunk (init) and successive chunks and remain chunk - tablesInfo[i].stsc_bEntries = new int[]{ - 1, SAMPLES_PER_CHUNK_INIT, 1, - 2, SAMPLES_PER_CHUNK, 1, - tablesInfo[i].stco + 1, tmp, 1 - }; - tablesInfo[i].stco++; + initChunkTables(tablesInfo[i], SAMPLES_PER_CHUNK_INIT, SAMPLES_PER_CHUNK); } sampleCount[i] = tablesInfo[i].stsz; @@ -274,6 +265,7 @@ public class Mp4FromDashWriter { // reserve moov space in the output stream /*if (outStream.canSetLength()) { long length = writeOffset + auxSize; + // warning: setLength() does not fill the unused space with zeros outStream.setLength(length); outSeek(length); } else {*/ @@ -292,10 +284,10 @@ public class Mp4FromDashWriter { } // tablesInfo contains row counts - // and after returning from make_moov() will contain table offsets + // and after returning from make_moov() will contain those table offsets make_moov(defaultMediaTime, tablesInfo, is64); - // write tables: stts stsc + // write tables: stts stsc sbgp // reset for ctts table: sampleCount sampleExtra for (int i = 0; i < readers.length; i++) { writeEntryArray(tablesInfo[i].stts, 2, sampleCount[i], defaultSampleDuration[i]); @@ -305,6 +297,7 @@ public class Mp4FromDashWriter { sampleCount[i] = 1;// the index is not base zero sampleExtra[i] = -1; } + writeEntryArray(tablesInfo[i].sbgp, 1, sampleCount[i]); } if (auxBuffer == null) { @@ -314,8 +307,8 @@ public class Mp4FromDashWriter { outWrite(make_mdat(totalSampleSize, is64)); int[] sampleIndex = new int[readers.length]; - int[] sizes = new int[singleChunk ? SINGLE_CHUNK_SAMPLE_BUFFER : SAMPLES_PER_CHUNK]; - int[] sync = new int[singleChunk ? SINGLE_CHUNK_SAMPLE_BUFFER : SAMPLES_PER_CHUNK]; + int[] sizes = new int[single_sample_buffer > 0 ? single_sample_buffer : SAMPLES_PER_CHUNK]; + int[] sync = new int[single_sample_buffer > 0 ? single_sample_buffer : SAMPLES_PER_CHUNK]; int written = readers.length; while (written > 0) { @@ -329,8 +322,8 @@ public class Mp4FromDashWriter { long chunkOffset = writeOffset; int syncCount = 0; int limit; - if (singleChunk) { - limit = SINGLE_CHUNK_SAMPLE_BUFFER; + if (single_sample_buffer > 0) { + limit = single_sample_buffer; } else { limit = sampleIndex[i] == 0 ? SAMPLES_PER_CHUNK_INIT : SAMPLES_PER_CHUNK; } @@ -390,10 +383,6 @@ public class Mp4FromDashWriter { } else { tablesInfo[i].stco = writeEntryArray(tablesInfo[i].stco, 1, (int) chunkOffset); } - - if (singleChunk) { - tablesInfo[i].stco = -1; - } } outRestore(); @@ -470,7 +459,42 @@ public class Mp4FromDashWriter { } } + private void initChunkTables(TablesInfo tables, int firstCount, int succesiveCount) { + // tables.stsz holds amount of samples of the track (total) + int totalSamples = (tables.stsz - firstCount); + float chunkAmount = totalSamples / (float) succesiveCount; + int remainChunkOffset = (int) Math.ceil(chunkAmount); + boolean remain = remainChunkOffset != (int) chunkAmount; + int index = 0; + tables.stsc = 1; + if (firstCount != succesiveCount) { + tables.stsc++; + } + if (remain) { + tables.stsc++; + } + + // stsc_table_entry = [first_chunk, samples_per_chunk, sample_description_index] + tables.stsc_bEntries = new int[tables.stsc * 3]; + tables.stco = remainChunkOffset + 1;// total entrys in chunk offset box + + tables.stsc_bEntries[index++] = 1; + tables.stsc_bEntries[index++] = firstCount; + tables.stsc_bEntries[index++] = 1; + + if (firstCount != succesiveCount) { + tables.stsc_bEntries[index++] = 2; + tables.stsc_bEntries[index++] = succesiveCount; + tables.stsc_bEntries[index++] = 1; + } + + if (remain) { + tables.stsc_bEntries[index++] = remainChunkOffset + 1; + tables.stsc_bEntries[index++] = totalSamples % succesiveCount; + tables.stsc_bEntries[index] = 1; + } + } private void outWrite(byte[] buffer) throws IOException { outWrite(buffer, buffer.length); @@ -585,19 +609,29 @@ public class Mp4FromDashWriter { private int make_ftyp() throws IOException { - byte[] buffer = new byte[]{ - 0x00, 0x00, 0x00, 0x1C, 0x66, 0x74, 0x79, 0x70,// ftyp - 0x6D, 0x70, 0x34, 0x32,// mayor brand (mp42) - 0x00, 0x00, 0x02, 0x00,// default minor version (512) - 0x6D, 0x70, 0x34, 0x31, 0x69, 0x73, 0x6F, 0x6D, 0x69, 0x73, 0x6F, 0x32// compatible brands: mp41 isom iso2 - }; + int size = 16 + (compatibleBrands.size() * 4); + if (overrideMainBrand != 0) size += 4; - if (overrideMainBrand != 0) - ByteBuffer.wrap(buffer).putInt(8, overrideMainBrand); + ByteBuffer buffer = ByteBuffer.allocate(size); + buffer.putInt(size); + buffer.putInt(0x66747970);// "ftyp" - outWrite(buffer); + if (overrideMainBrand == 0) { + buffer.putInt(0x6D703432);// mayor brand "mp42" + buffer.putInt(512);// default minor version + } else { + buffer.putInt(overrideMainBrand); + buffer.putInt(0); + buffer.putInt(0x6D703432);// "mp42" compatible brand + } - return buffer.length; + for (Integer brand : compatibleBrands) { + buffer.putInt(brand);// compatible brand + } + + outWrite(buffer.array()); + + return size; } private byte[] make_mdat(long refSize, boolean is64) { @@ -746,7 +780,6 @@ public class Mp4FromDashWriter { } private void make_mdia(Mdia mdia, TablesInfo tablesInfo, boolean is64) throws IOException { - int start_mdia = auxOffset(); auxWrite(new byte[]{0x00, 0x00, 0x00, 0x00, 0x6D, 0x64, 0x69, 0x61});// mdia auxWrite(mdia.mdhd); @@ -766,7 +799,7 @@ public class Mp4FromDashWriter { // And stsz can be empty if has a default sample size // if (moovSimulation) { - make(0x73747473, -1, 2, 1); + make(0x73747473, -1, 2, 1);// stts if (tablesInfo.stss > 0) { make(0x73747373, -1, 1, tablesInfo.stss); } @@ -789,6 +822,9 @@ public class Mp4FromDashWriter { tablesInfo.stco = make(is64 ? 0x636F3634 : 0x7374636F, -1, is64 ? 2 : 1, tablesInfo.stco); } + auxWrite(make_sgpd()); + tablesInfo.sbgp = make_sbgp();// during simulation the returned offset is ignored + lengthFor(start_stbl); lengthFor(start_minf); lengthFor(start_mdia); @@ -816,6 +852,48 @@ public class Mp4FromDashWriter { return buffer.array(); } + private int make_sbgp() throws IOException { + int offset = auxOffset(); + + auxWrite(new byte[] { + 0x00, 0x00, 0x00, 0x1C,// box size + 0x73, 0x62, 0x67, 0x70,// "sbpg" + 0x00, 0x00, 0x00, 0x00,// default box flags + 0x72, 0x6F, 0x6C, 0x6C,// group type "roll" + 0x00, 0x00, 0x00, 0x01,// group table size + 0x00, 0x00, 0x00, 0x00,// group[0] total samples (to be set later) + 0x00, 0x00, 0x00, 0x01// group[0] description index + }); + + return offset + 0x14; + } + + private byte[] make_sgpd() { + /* + * Sample Group Description Box + * + * ¿whats does? + * the table inside of this box gives information about the + * characteristics of sample groups. The descriptive information is any other + * information needed to define or characterize the sample group. + * + * ¿is replicabled this box? + * NO due lacks of documentation about this box but... + * most of m4a encoders and ffmpeg uses this box with dummy values (same values) + */ + + ByteBuffer buffer = ByteBuffer.wrap(new byte[] { + 0x00, 0x00, 0x00, 0x1A,// box size + 0x73, 0x67, 0x70, 0x64,// "sgpd" + 0x01, 0x00, 0x00, 0x00,// box flags (unknown flag sets) + 0x72, 0x6F, 0x6C, 0x6C, // ¿¿group type?? + 0x00, 0x00, 0x00, 0x02,// ¿¿?? + 0x00, 0x00, 0x00, 0x01,// ¿¿?? + (byte)0xFF, (byte)0xFF// ¿¿?? + }); + + return buffer.array(); + } class TablesInfo { @@ -827,5 +905,6 @@ public class Mp4FromDashWriter { int stsz_default; int stss; int stco; + int sbgp; } } From 9b71828b97a46227f214b76cf99e53bb1884a7ee Mon Sep 17 00:00:00 2001 From: kapodamy Date: Tue, 14 Jan 2020 01:08:46 -0300 Subject: [PATCH 2/9] implement sgpd and sbgp boxes in audio tracks --- .../org/schabi/newpipe/streams/Mp4FromDashWriter.java | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/app/src/main/java/org/schabi/newpipe/streams/Mp4FromDashWriter.java b/app/src/main/java/org/schabi/newpipe/streams/Mp4FromDashWriter.java index f94ebefb0..963d84b40 100644 --- a/app/src/main/java/org/schabi/newpipe/streams/Mp4FromDashWriter.java +++ b/app/src/main/java/org/schabi/newpipe/streams/Mp4FromDashWriter.java @@ -774,12 +774,12 @@ public class Mp4FromDashWriter { .array() ); - make_mdia(tracks[index].trak.mdia, tables, is64); + make_mdia(tracks[index].trak.mdia, tables, is64, tracks[index].kind == TrackKind.Audio); lengthFor(start); } - private void make_mdia(Mdia mdia, TablesInfo tablesInfo, boolean is64) throws IOException { + private void make_mdia(Mdia mdia, TablesInfo tablesInfo, boolean is64, boolean isAudio) throws IOException { int start_mdia = auxOffset(); auxWrite(new byte[]{0x00, 0x00, 0x00, 0x00, 0x6D, 0x64, 0x69, 0x61});// mdia auxWrite(mdia.mdhd); @@ -822,8 +822,10 @@ public class Mp4FromDashWriter { tablesInfo.stco = make(is64 ? 0x636F3634 : 0x7374636F, -1, is64 ? 2 : 1, tablesInfo.stco); } - auxWrite(make_sgpd()); - tablesInfo.sbgp = make_sbgp();// during simulation the returned offset is ignored + if (isAudio) { + auxWrite(make_sgpd()); + tablesInfo.sbgp = make_sbgp();// during simulation the returned offset is ignored + } lengthFor(start_stbl); lengthFor(start_minf); From 342377e69ae3157c0b11a03d32721b80844a0b7b Mon Sep 17 00:00:00 2001 From: kapodamy Date: Wed, 29 Jan 2020 16:04:24 -0300 Subject: [PATCH 3/9] restore offset after writting lastest CTTS entries --- .../org/schabi/newpipe/streams/Mp4FromDashWriter.java | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/app/src/main/java/org/schabi/newpipe/streams/Mp4FromDashWriter.java b/app/src/main/java/org/schabi/newpipe/streams/Mp4FromDashWriter.java index 963d84b40..3dfba8b2d 100644 --- a/app/src/main/java/org/schabi/newpipe/streams/Mp4FromDashWriter.java +++ b/app/src/main/java/org/schabi/newpipe/streams/Mp4FromDashWriter.java @@ -262,13 +262,6 @@ public class Mp4FromDashWriter { final int ftyp_size = make_ftyp(); - // reserve moov space in the output stream - /*if (outStream.canSetLength()) { - long length = writeOffset + auxSize; - // warning: setLength() does not fill the unused space with zeros - outStream.setLength(length); - outSeek(length); - } else {*/ if (auxSize > 0) { int length = auxSize; byte[] buffer = new byte[64 * 1024];// 64 KiB @@ -335,6 +328,7 @@ public class Mp4FromDashWriter { if (sample == null) { if (tablesInfo[i].ctts > 0 && sampleExtra[i] >= 0) { writeEntryArray(tablesInfo[i].ctts, 1, sampleCount[i], sampleExtra[i]);// flush last entries + outRestore(); } sampleIndex[i] = -1; break; From 0c5608506e9a2887b86cf6907e3c593efb442308 Mon Sep 17 00:00:00 2001 From: kapodamy Date: Wed, 29 Jan 2020 16:06:40 -0300 Subject: [PATCH 4/9] typo fixup --- .../java/org/schabi/newpipe/streams/Mp4FromDashWriter.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/app/src/main/java/org/schabi/newpipe/streams/Mp4FromDashWriter.java b/app/src/main/java/org/schabi/newpipe/streams/Mp4FromDashWriter.java index 3dfba8b2d..57a7aaa9c 100644 --- a/app/src/main/java/org/schabi/newpipe/streams/Mp4FromDashWriter.java +++ b/app/src/main/java/org/schabi/newpipe/streams/Mp4FromDashWriter.java @@ -250,7 +250,7 @@ public class Mp4FromDashWriter { boolean is64 = read > THRESHOLD_FOR_CO64; - // calculate the moov size; + // calculate the moov size int auxSize = make_moov(defaultMediaTime, tablesInfo, is64); if (auxSize < THRESHOLD_MOOV_LENGTH) { @@ -262,6 +262,7 @@ public class Mp4FromDashWriter { final int ftyp_size = make_ftyp(); + // reserve moov space in the output stream if (auxSize > 0) { int length = auxSize; byte[] buffer = new byte[64 * 1024];// 64 KiB @@ -328,7 +329,7 @@ public class Mp4FromDashWriter { if (sample == null) { if (tablesInfo[i].ctts > 0 && sampleExtra[i] >= 0) { writeEntryArray(tablesInfo[i].ctts, 1, sampleCount[i], sampleExtra[i]);// flush last entries - outRestore(); + outRestore(); } sampleIndex[i] = -1; break; From aaf5d7b89c5f60b32d08c343bf83a1807956d206 Mon Sep 17 00:00:00 2001 From: kapodamy Date: Thu, 6 Feb 2020 17:00:32 -0300 Subject: [PATCH 5/9] Update DataReader.java make rewind() method fully rewind the stream --- app/src/main/java/org/schabi/newpipe/streams/DataReader.java | 1 + 1 file changed, 1 insertion(+) diff --git a/app/src/main/java/org/schabi/newpipe/streams/DataReader.java b/app/src/main/java/org/schabi/newpipe/streams/DataReader.java index 0e62810c5..75b55cd73 100644 --- a/app/src/main/java/org/schabi/newpipe/streams/DataReader.java +++ b/app/src/main/java/org/schabi/newpipe/streams/DataReader.java @@ -137,6 +137,7 @@ public class DataReader { position = 0; readOffset = readBuffer.length; + readCount = 0; } public boolean canRewind() { From 7045f9711c04a4a0610553f605c8c69a7fd4f21f Mon Sep 17 00:00:00 2001 From: bopol Date: Sun, 2 Feb 2020 16:54:09 +0100 Subject: [PATCH 6/9] fix thumbnail for PeerTube, and description changes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit description: - PeerTube: it's now full description (it cut at 250 characters before), and it displays ok (newlines are ok, but markdown isn't) - MediaCCC: descriptions are now displayed well (newlines added) - YouTube: timestamps in descriptions are clickable and work more PeerTube fixes: thumbnail is now high quality age limit is now handled upload date in «recently added» feed is good now (it was one hour delayed) all fixes come from https://github.com/TeamNewPipe/NewPipeExtractor/pull/239, so it need to be merged before this PR --- app/build.gradle | 2 +- .../fragments/detail/VideoDetailFragment.java | 75 +++++++++++++------ 2 files changed, 53 insertions(+), 24 deletions(-) diff --git a/app/build.gradle b/app/build.gradle index f8fc1565f..0f7ad5f92 100644 --- a/app/build.gradle +++ b/app/build.gradle @@ -62,7 +62,7 @@ dependencies { exclude module: 'support-annotations' }) - implementation 'com.github.TeamNewPipe:NewPipeExtractor:ff61e284' + implementation 'com.github.B0pol:NewPipeExtractor:5756df8dc7e89b7383d1d1e07a91c30bdab6f868' testImplementation 'junit:junit:4.12' testImplementation 'org.mockito:mockito-core:2.23.0' diff --git a/app/src/main/java/org/schabi/newpipe/fragments/detail/VideoDetailFragment.java b/app/src/main/java/org/schabi/newpipe/fragments/detail/VideoDetailFragment.java index f59cfaef0..a297cddf3 100644 --- a/app/src/main/java/org/schabi/newpipe/fragments/detail/VideoDetailFragment.java +++ b/app/src/main/java/org/schabi/newpipe/fragments/detail/VideoDetailFragment.java @@ -2,7 +2,6 @@ package org.schabi.newpipe.fragments.detail; import android.app.Activity; import android.content.Context; -import android.content.DialogInterface; import android.content.Intent; import android.content.SharedPreferences; import android.net.Uri; @@ -18,7 +17,6 @@ import androidx.fragment.app.Fragment; import androidx.core.content.ContextCompat; import androidx.viewpager.widget.ViewPager; import androidx.appcompat.app.ActionBar; -import androidx.appcompat.app.AlertDialog; import androidx.appcompat.app.AppCompatActivity; import android.text.Html; import android.text.Spanned; @@ -193,6 +191,14 @@ public class VideoDetailFragment private TabLayout tabLayout; private FrameLayout relatedStreamsLayout; + private static final int DESCRIPTION_HTML = 1; + private static final int DESCRIPTION_MARKDOWN = 2; + private static final int DESCRIPTION_PLAIN_TEXT = 3; + + private static final int YOUTUBE_SERVICE_ID = ServiceList.YouTube.getServiceId(); + private static final int MEDIACCC_SERVICE_ID = ServiceList.MediaCCC.getServiceId(); + private static final int PEERTUBE_SERVICE_ID = ServiceList.PeerTube.getServiceId(); + /*////////////////////////////////////////////////////////////////////////*/ @@ -483,7 +489,6 @@ public class VideoDetailFragment videoUploadDateView = rootView.findViewById(R.id.detail_upload_date_view); videoDescriptionView = rootView.findViewById(R.id.detail_description_view); videoDescriptionView.setMovementMethod(LinkMovementMethod.getInstance()); - videoDescriptionView.setAutoLinkMask(Linkify.WEB_URLS); thumbsUpTextView = rootView.findViewById(R.id.detail_thumbs_up_count_view); thumbsUpImageView = rootView.findViewById(R.id.detail_thumbs_up_img_view); @@ -919,28 +924,39 @@ public class VideoDetailFragment return sortedVideoStreams != null ? sortedVideoStreams.get(selectedVideoStreamIndex) : null; } - private void prepareDescription(final String descriptionHtml) { - if (TextUtils.isEmpty(descriptionHtml)) { + private void prepareDescription(final String descriptionText, int descriptionTypeId) { + if (TextUtils.isEmpty(descriptionText)) { return; } - disposables.add(Single.just(descriptionHtml) - .map((@io.reactivex.annotations.NonNull String description) -> { - Spanned parsedDescription; - if (Build.VERSION.SDK_INT >= 24) { - parsedDescription = Html.fromHtml(description, 0); - } else { - //noinspection deprecation - parsedDescription = Html.fromHtml(description); - } - return parsedDescription; - }) - .subscribeOn(Schedulers.computation()) - .observeOn(AndroidSchedulers.mainThread()) - .subscribe((@io.reactivex.annotations.NonNull Spanned spanned) -> { - videoDescriptionView.setText(spanned); - videoDescriptionView.setVisibility(View.VISIBLE); - })); + if (descriptionTypeId == DESCRIPTION_PLAIN_TEXT) { + videoDescriptionView.setText(descriptionText, TextView.BufferType.SPANNABLE); + videoDescriptionView.setVisibility(View.VISIBLE); + } else if (descriptionTypeId == DESCRIPTION_MARKDOWN) { + //in the future we would use a library or a good method to show markdown. + //rn, we just remove **bold**, and let plain_text otherwise + videoDescriptionView.setText(descriptionText.replace("**", ""), TextView.BufferType.SPANNABLE); + videoDescriptionView.setVisibility(View.VISIBLE); + } else { + //== DESCRIPTION_HTML + disposables.add(Single.just(descriptionText) + .map((@io.reactivex.annotations.NonNull String description) -> { + Spanned parsedDescription; + if (Build.VERSION.SDK_INT >= 24) { + parsedDescription = Html.fromHtml(description, 0); + } else { + //noinspection deprecation + parsedDescription = Html.fromHtml(description); + } + return parsedDescription; + }) + .subscribeOn(Schedulers.computation()) + .observeOn(AndroidSchedulers.mainThread()) + .subscribe((@io.reactivex.annotations.NonNull Spanned spanned) -> { + videoDescriptionView.setText(spanned); + videoDescriptionView.setVisibility(View.VISIBLE); + })); + } } private void setHeightThumbnail() { @@ -1126,7 +1142,20 @@ public class VideoDetailFragment videoUploadDateView.setVisibility(View.GONE); } - prepareDescription(info.getDescription()); + int serviceId = info.getServiceId(); + + if (serviceId != YOUTUBE_SERVICE_ID) { + videoDescriptionView.setAutoLinkMask(Linkify.WEB_URLS); + } + + if (serviceId == PEERTUBE_SERVICE_ID) { + prepareDescription(info.getDescription(), DESCRIPTION_MARKDOWN); + } else if (serviceId == MEDIACCC_SERVICE_ID) { + prepareDescription(info.getDescription(), DESCRIPTION_PLAIN_TEXT); + } else { + prepareDescription(info.getDescription(), DESCRIPTION_HTML); + } + updateProgressInfo(info); animateView(spinnerToolbar, true, 500); From badaff8ebcd6f446a6a4b575adba43d627b4c7c2 Mon Sep 17 00:00:00 2001 From: bopol Date: Thu, 6 Feb 2020 23:54:36 +0100 Subject: [PATCH 7/9] refactor Description --- app/build.gradle | 2 +- .../fragments/detail/VideoDetailFragment.java | 60 +++++++------------ 2 files changed, 23 insertions(+), 39 deletions(-) diff --git a/app/build.gradle b/app/build.gradle index 0f7ad5f92..728d380c4 100644 --- a/app/build.gradle +++ b/app/build.gradle @@ -62,7 +62,7 @@ dependencies { exclude module: 'support-annotations' }) - implementation 'com.github.B0pol:NewPipeExtractor:5756df8dc7e89b7383d1d1e07a91c30bdab6f868' + implementation 'com.github.B0pol:NewPipeExtractor:11bcc78d9c8eb39e8d61a6f4bc4112025937f087' testImplementation 'junit:junit:4.12' testImplementation 'org.mockito:mockito-core:2.23.0' diff --git a/app/src/main/java/org/schabi/newpipe/fragments/detail/VideoDetailFragment.java b/app/src/main/java/org/schabi/newpipe/fragments/detail/VideoDetailFragment.java index a297cddf3..c6c8ca04c 100644 --- a/app/src/main/java/org/schabi/newpipe/fragments/detail/VideoDetailFragment.java +++ b/app/src/main/java/org/schabi/newpipe/fragments/detail/VideoDetailFragment.java @@ -56,6 +56,7 @@ import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeStreamExtractor; import org.schabi.newpipe.extractor.stream.AudioStream; +import org.schabi.newpipe.extractor.stream.Description; import org.schabi.newpipe.extractor.stream.Stream; import org.schabi.newpipe.extractor.stream.StreamInfo; import org.schabi.newpipe.extractor.stream.StreamType; @@ -191,14 +192,6 @@ public class VideoDetailFragment private TabLayout tabLayout; private FrameLayout relatedStreamsLayout; - private static final int DESCRIPTION_HTML = 1; - private static final int DESCRIPTION_MARKDOWN = 2; - private static final int DESCRIPTION_PLAIN_TEXT = 3; - - private static final int YOUTUBE_SERVICE_ID = ServiceList.YouTube.getServiceId(); - private static final int MEDIACCC_SERVICE_ID = ServiceList.MediaCCC.getServiceId(); - private static final int PEERTUBE_SERVICE_ID = ServiceList.PeerTube.getServiceId(); - /*////////////////////////////////////////////////////////////////////////*/ @@ -924,29 +917,24 @@ public class VideoDetailFragment return sortedVideoStreams != null ? sortedVideoStreams.get(selectedVideoStreamIndex) : null; } - private void prepareDescription(final String descriptionText, int descriptionTypeId) { - if (TextUtils.isEmpty(descriptionText)) { + private void prepareDescription(Description description) { + if (TextUtils.isEmpty(description.getContent()) || description == Description.emptyDescription) { return; } - if (descriptionTypeId == DESCRIPTION_PLAIN_TEXT) { - videoDescriptionView.setText(descriptionText, TextView.BufferType.SPANNABLE); - videoDescriptionView.setVisibility(View.VISIBLE); - } else if (descriptionTypeId == DESCRIPTION_MARKDOWN) { - //in the future we would use a library or a good method to show markdown. - //rn, we just remove **bold**, and let plain_text otherwise - videoDescriptionView.setText(descriptionText.replace("**", ""), TextView.BufferType.SPANNABLE); - videoDescriptionView.setVisibility(View.VISIBLE); - } else { - //== DESCRIPTION_HTML - disposables.add(Single.just(descriptionText) - .map((@io.reactivex.annotations.NonNull String description) -> { + if (description.getType() != Description.HTML) { + videoDescriptionView.setAutoLinkMask(Linkify.WEB_URLS); + } + + if (description.getType() == Description.HTML) { + disposables.add(Single.just(description.getContent()) + .map((@io.reactivex.annotations.NonNull String descriptionText) -> { Spanned parsedDescription; if (Build.VERSION.SDK_INT >= 24) { - parsedDescription = Html.fromHtml(description, 0); + parsedDescription = Html.fromHtml(descriptionText, 0); } else { //noinspection deprecation - parsedDescription = Html.fromHtml(description); + parsedDescription = Html.fromHtml(descriptionText); } return parsedDescription; }) @@ -956,6 +944,15 @@ public class VideoDetailFragment videoDescriptionView.setText(spanned); videoDescriptionView.setVisibility(View.VISIBLE); })); + } else if (description.getType() == Description.MARKDOWN) { + //in the future we would use a library or a good method to show markdown. + //rn, we just remove **bold**, and let PLAIN_TEXT otherwise + videoDescriptionView.setText(description.getContent().replace("**", ""), TextView.BufferType.SPANNABLE); + videoDescriptionView.setVisibility(View.VISIBLE); + } else { + //== Description.PLAIN_TEXT + videoDescriptionView.setText(description.getContent(), TextView.BufferType.SPANNABLE); + videoDescriptionView.setVisibility(View.VISIBLE); } } @@ -1142,20 +1139,7 @@ public class VideoDetailFragment videoUploadDateView.setVisibility(View.GONE); } - int serviceId = info.getServiceId(); - - if (serviceId != YOUTUBE_SERVICE_ID) { - videoDescriptionView.setAutoLinkMask(Linkify.WEB_URLS); - } - - if (serviceId == PEERTUBE_SERVICE_ID) { - prepareDescription(info.getDescription(), DESCRIPTION_MARKDOWN); - } else if (serviceId == MEDIACCC_SERVICE_ID) { - prepareDescription(info.getDescription(), DESCRIPTION_PLAIN_TEXT); - } else { - prepareDescription(info.getDescription(), DESCRIPTION_HTML); - } - + prepareDescription(info.getDescription()); updateProgressInfo(info); animateView(spinnerToolbar, true, 500); From 2d62fa401d63551785859b0d909f8414e94ba191 Mon Sep 17 00:00:00 2001 From: bopol Date: Sat, 8 Feb 2020 09:56:37 +0100 Subject: [PATCH 8/9] real markdown support for descriptions and update third-party licences in about page --- app/build.gradle | 4 ++++ .../org/schabi/newpipe/about/AboutActivity.java | 16 +++++++++------- .../fragments/detail/VideoDetailFragment.java | 14 +++++++------- 3 files changed, 20 insertions(+), 14 deletions(-) diff --git a/app/build.gradle b/app/build.gradle index 728d380c4..65ab78ffa 100644 --- a/app/build.gradle +++ b/app/build.gradle @@ -53,6 +53,7 @@ ext { okHttpLibVersion = '3.12.6' icepickLibVersion = '3.2.0' stethoLibVersion = '1.5.0' + markwonVersion = '4.2.1' } dependencies { @@ -108,4 +109,7 @@ dependencies { implementation "com.squareup.okhttp3:okhttp:${okHttpLibVersion}" debugImplementation "com.facebook.stetho:stetho-okhttp3:${stethoLibVersion}" + + implementation "io.noties.markwon:core:${markwonVersion}" + implementation "io.noties.markwon:linkify:${markwonVersion}" } diff --git a/app/src/main/java/org/schabi/newpipe/about/AboutActivity.java b/app/src/main/java/org/schabi/newpipe/about/AboutActivity.java index 9e23d9d3d..edfc54375 100644 --- a/app/src/main/java/org/schabi/newpipe/about/AboutActivity.java +++ b/app/src/main/java/org/schabi/newpipe/about/AboutActivity.java @@ -32,18 +32,20 @@ public class AboutActivity extends AppCompatActivity { * List of all software components */ private static final SoftwareComponent[] SOFTWARE_COMPONENTS = new SoftwareComponent[]{ - new SoftwareComponent("Giga Get", "2014", "Peter Cai", "https://github.com/PaperAirplane-Dev-Team/GigaGet", StandardLicenses.GPL2), - new SoftwareComponent("NewPipe Extractor", "2017", "Christian Schabesberger", "https://github.com/TeamNewPipe/NewPipeExtractor", StandardLicenses.GPL3), + new SoftwareComponent("Giga Get", "2014 - 2015", "Peter Cai", "https://github.com/PaperAirplane-Dev-Team/GigaGet", StandardLicenses.GPL2), + new SoftwareComponent("NewPipe Extractor", "2017 - 2020", "Christian Schabesberger", "https://github.com/TeamNewPipe/NewPipeExtractor", StandardLicenses.GPL3), new SoftwareComponent("Jsoup", "2017", "Jonathan Hedley", "https://github.com/jhy/jsoup", StandardLicenses.MIT), new SoftwareComponent("Rhino", "2015", "Mozilla", "https://www.mozilla.org/rhino/", StandardLicenses.MPL2), new SoftwareComponent("ACRA", "2013", "Kevin Gaudin", "http://www.acra.ch", StandardLicenses.APACHE2), new SoftwareComponent("Universal Image Loader", "2011 - 2015", "Sergey Tarasevich", "https://github.com/nostra13/Android-Universal-Image-Loader", StandardLicenses.APACHE2), - new SoftwareComponent("CircleImageView", "2014 - 2017", "Henning Dodenhof", "https://github.com/hdodenhof/CircleImageView", StandardLicenses.APACHE2), + new SoftwareComponent("CircleImageView", "2014 - 2020", "Henning Dodenhof", "https://github.com/hdodenhof/CircleImageView", StandardLicenses.APACHE2), new SoftwareComponent("NoNonsense-FilePicker", "2016", "Jonas Kalderstam", "https://github.com/spacecowboy/NoNonsense-FilePicker", StandardLicenses.MPL2), - new SoftwareComponent("ExoPlayer", "2014-2017", "Google Inc", "https://github.com/google/ExoPlayer", StandardLicenses.APACHE2), - new SoftwareComponent("RxAndroid", "2015", "The RxAndroid authors", "https://github.com/ReactiveX/RxAndroid", StandardLicenses.APACHE2), - new SoftwareComponent("RxJava", "2016-present", "RxJava Contributors", "https://github.com/ReactiveX/RxJava", StandardLicenses.APACHE2), - new SoftwareComponent("RxBinding", "2015", "Jake Wharton", "https://github.com/JakeWharton/RxBinding", StandardLicenses.APACHE2) + new SoftwareComponent("ExoPlayer", "2014 - 2020", "Google Inc", "https://github.com/google/ExoPlayer", StandardLicenses.APACHE2), + new SoftwareComponent("RxAndroid", "2015 - 2018", "The RxAndroid authors", "https://github.com/ReactiveX/RxAndroid", StandardLicenses.APACHE2), + new SoftwareComponent("RxJava", "2016 - 2020", "RxJava Contributors", "https://github.com/ReactiveX/RxJava", StandardLicenses.APACHE2), + new SoftwareComponent("RxBinding", "2015 - 2018", "Jake Wharton", "https://github.com/JakeWharton/RxBinding", StandardLicenses.APACHE2), + new SoftwareComponent("PrettyTime", "2012 - 2020", "Lincoln Baxter, III", "https://github.com/ocpsoft/prettytime", StandardLicenses.APACHE2), + new SoftwareComponent("Markwon", "2017 - 2020", "Noties", "https://github.com/noties/Markwon", StandardLicenses.APACHE2) }; /** diff --git a/app/src/main/java/org/schabi/newpipe/fragments/detail/VideoDetailFragment.java b/app/src/main/java/org/schabi/newpipe/fragments/detail/VideoDetailFragment.java index c6c8ca04c..3c594bdfa 100644 --- a/app/src/main/java/org/schabi/newpipe/fragments/detail/VideoDetailFragment.java +++ b/app/src/main/java/org/schabi/newpipe/fragments/detail/VideoDetailFragment.java @@ -95,6 +95,8 @@ import java.util.List; import java.util.concurrent.TimeUnit; import icepick.State; +import io.noties.markwon.Markwon; +import io.noties.markwon.linkify.LinkifyPlugin; import io.reactivex.Single; import io.reactivex.android.schedulers.AndroidSchedulers; import io.reactivex.disposables.CompositeDisposable; @@ -922,10 +924,6 @@ public class VideoDetailFragment return; } - if (description.getType() != Description.HTML) { - videoDescriptionView.setAutoLinkMask(Linkify.WEB_URLS); - } - if (description.getType() == Description.HTML) { disposables.add(Single.just(description.getContent()) .map((@io.reactivex.annotations.NonNull String descriptionText) -> { @@ -945,12 +943,14 @@ public class VideoDetailFragment videoDescriptionView.setVisibility(View.VISIBLE); })); } else if (description.getType() == Description.MARKDOWN) { - //in the future we would use a library or a good method to show markdown. - //rn, we just remove **bold**, and let PLAIN_TEXT otherwise - videoDescriptionView.setText(description.getContent().replace("**", ""), TextView.BufferType.SPANNABLE); + final Markwon markwon = Markwon.builder(getContext()) + .usePlugin(LinkifyPlugin.create()) + .build(); + markwon.setMarkdown(videoDescriptionView, description.getContent()); videoDescriptionView.setVisibility(View.VISIBLE); } else { //== Description.PLAIN_TEXT + videoDescriptionView.setAutoLinkMask(Linkify.WEB_URLS); videoDescriptionView.setText(description.getContent(), TextView.BufferType.SPANNABLE); videoDescriptionView.setVisibility(View.VISIBLE); } From 3f3d1bfccf9be4182bd2872358263d3c904e447a Mon Sep 17 00:00:00 2001 From: bopol Date: Sun, 9 Feb 2020 00:00:14 +0100 Subject: [PATCH 9/9] update extractor version --- app/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/build.gradle b/app/build.gradle index 65ab78ffa..5da8c9ff0 100644 --- a/app/build.gradle +++ b/app/build.gradle @@ -63,7 +63,7 @@ dependencies { exclude module: 'support-annotations' }) - implementation 'com.github.B0pol:NewPipeExtractor:11bcc78d9c8eb39e8d61a6f4bc4112025937f087' + implementation 'com.github.TeamNewPipe:NewPipeExtractor:9112a10' testImplementation 'junit:junit:4.12' testImplementation 'org.mockito:mockito-core:2.23.0'