From 8e152df46d6b4f22c05344e8b437c9451d0b167e Mon Sep 17 00:00:00 2001 From: Robin Date: Sun, 24 Nov 2019 11:10:50 +0100 Subject: [PATCH 01/28] Remember caption option in player, closes #2811 --- .../schabi/newpipe/player/VideoPlayer.java | 34 +++++++++++++++++-- app/src/main/res/values/settings_keys.xml | 1 + 2 files changed, 32 insertions(+), 3 deletions(-) diff --git a/app/src/main/java/org/schabi/newpipe/player/VideoPlayer.java b/app/src/main/java/org/schabi/newpipe/player/VideoPlayer.java index 360475ba2..a92ab410e 100644 --- a/app/src/main/java/org/schabi/newpipe/player/VideoPlayer.java +++ b/app/src/main/java/org/schabi/newpipe/player/VideoPlayer.java @@ -26,14 +26,13 @@ import android.animation.PropertyValuesHolder; import android.animation.ValueAnimator; import android.content.Context; import android.content.Intent; +import android.content.SharedPreferences; import android.graphics.Bitmap; import android.graphics.Color; import android.graphics.PorterDuff; import android.os.Build; import android.os.Handler; -import androidx.annotation.NonNull; -import androidx.annotation.Nullable; -import androidx.core.content.ContextCompat; +import android.preference.PreferenceManager; import android.util.Log; import android.view.Menu; import android.view.MenuItem; @@ -45,6 +44,10 @@ import android.widget.ProgressBar; import android.widget.SeekBar; import android.widget.TextView; +import androidx.annotation.NonNull; +import androidx.annotation.Nullable; +import androidx.core.content.ContextCompat; + import com.google.android.exoplayer2.C; import com.google.android.exoplayer2.PlaybackParameters; import com.google.android.exoplayer2.Player; @@ -285,6 +288,12 @@ public abstract class VideoPlayer extends BasePlayer if (captionPopupMenu == null) return; captionPopupMenu.getMenu().removeGroup(captionPopupMenuGroupId); + String userPreferredLanguage = PreferenceManager.getDefaultSharedPreferences(context) + .getString(context.getString(R.string.caption_user_set_key), null); + // english (auto-generated) + boolean searchForAutogenerated = userPreferredLanguage == null || + !userPreferredLanguage.contains("("); + // Add option for turning off caption MenuItem captionOffItem = captionPopupMenu.getMenu().add(captionPopupMenuGroupId, 0, Menu.NONE, R.string.caption_none); @@ -294,6 +303,8 @@ public abstract class VideoPlayer extends BasePlayer trackSelector.setParameters(trackSelector.buildUponParameters() .setRendererDisabled(textRendererIndex, true)); } + final SharedPreferences prefs = PreferenceManager.getDefaultSharedPreferences(context); + prefs.edit().remove(context.getString(R.string.caption_user_set_key)).commit(); return true; }); @@ -308,9 +319,26 @@ public abstract class VideoPlayer extends BasePlayer trackSelector.setPreferredTextLanguage(captionLanguage); trackSelector.setParameters(trackSelector.buildUponParameters() .setRendererDisabled(textRendererIndex, false)); + final SharedPreferences prefs = PreferenceManager.getDefaultSharedPreferences(context); + prefs.edit().putString(context.getString(R.string.caption_user_set_key), + captionLanguage).commit(); } return true; }); + // apply caption language from previous user preference + if(userPreferredLanguage != null && (captionLanguage.equals(userPreferredLanguage) || + searchForAutogenerated && captionLanguage.startsWith(userPreferredLanguage) || + userPreferredLanguage.contains("(") && + captionLanguage.startsWith(userPreferredLanguage.substring(0, + userPreferredLanguage.indexOf('('))))) { + final int textRendererIndex = getRendererIndex(C.TRACK_TYPE_TEXT); + if (textRendererIndex != RENDERER_UNAVAILABLE) { + trackSelector.setPreferredTextLanguage(captionLanguage); + trackSelector.setParameters(trackSelector.buildUponParameters() + .setRendererDisabled(textRendererIndex, false)); + } + searchForAutogenerated = false; + } } captionPopupMenu.setOnDismissListener(this); } diff --git a/app/src/main/res/values/settings_keys.xml b/app/src/main/res/values/settings_keys.xml index 80f2bb1f4..783ab7b5e 100644 --- a/app/src/main/res/values/settings_keys.xml +++ b/app/src/main/res/values/settings_keys.xml @@ -133,6 +133,7 @@ caption_settings_key + caption_user_set_key show_search_suggestions From 52a21e4a246600dcc4dfcc1fb9c558575be53c8c Mon Sep 17 00:00:00 2001 From: kapodamy Date: Mon, 23 Sep 2019 21:38:29 -0300 Subject: [PATCH 02/28] implement webm to ogg demuxer * used for opus audio stream * update WebMReader and WebMWriter * new post-processing algorithm --- .../newpipe/download/DownloadDialog.java | 4 +- .../newpipe/streams/OggFromWebMWriter.java | 488 ++++++++++++++++++ .../schabi/newpipe/streams/WebMReader.java | 55 +- .../schabi/newpipe/streams/WebMWriter.java | 27 +- .../postprocessing/OggFromWebmDemuxer.java | 44 ++ .../giga/postprocessing/Postprocessing.java | 6 +- 6 files changed, 595 insertions(+), 29 deletions(-) create mode 100644 app/src/main/java/org/schabi/newpipe/streams/OggFromWebMWriter.java create mode 100644 app/src/main/java/us/shandian/giga/postprocessing/OggFromWebmDemuxer.java diff --git a/app/src/main/java/org/schabi/newpipe/download/DownloadDialog.java b/app/src/main/java/org/schabi/newpipe/download/DownloadDialog.java index 59bffa933..90258a6dc 100644 --- a/app/src/main/java/org/schabi/newpipe/download/DownloadDialog.java +++ b/app/src/main/java/org/schabi/newpipe/download/DownloadDialog.java @@ -561,7 +561,7 @@ public class DownloadDialog extends DialogFragment implements RadioGroup.OnCheck mainStorage = mainStorageVideo; format = videoStreamsAdapter.getItem(selectedVideoIndex).getFormat(); mime = format.mimeType; - filename += format.suffix; + filename += format == MediaFormat.OPUS ? "ogg" : format.suffix; break; case R.id.subtitle_button: mainStorage = mainStorageVideo;// subtitle & video files go together @@ -778,6 +778,8 @@ public class DownloadDialog extends DialogFragment implements RadioGroup.OnCheck if (selectedStream.getFormat() == MediaFormat.M4A) { psName = Postprocessing.ALGORITHM_M4A_NO_DASH; + } else if (selectedStream.getFormat() == MediaFormat.OPUS) { + psName = Postprocessing.ALGORITHM_OGG_FROM_WEBM_DEMUXER; } break; case R.id.video_button: diff --git a/app/src/main/java/org/schabi/newpipe/streams/OggFromWebMWriter.java b/app/src/main/java/org/schabi/newpipe/streams/OggFromWebMWriter.java new file mode 100644 index 000000000..2b3d778c6 --- /dev/null +++ b/app/src/main/java/org/schabi/newpipe/streams/OggFromWebMWriter.java @@ -0,0 +1,488 @@ +package org.schabi.newpipe.streams; + +import android.support.annotation.NonNull; + +import org.schabi.newpipe.streams.WebMReader.Cluster; +import org.schabi.newpipe.streams.WebMReader.Segment; +import org.schabi.newpipe.streams.WebMReader.SimpleBlock; +import org.schabi.newpipe.streams.WebMReader.WebMTrack; +import org.schabi.newpipe.streams.io.SharpStream; + +import java.io.Closeable; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.util.ArrayList; +import java.util.Random; + +import javax.annotation.Nullable; + +/** + * @author kapodamy + */ +public class OggFromWebMWriter implements Closeable { + + private static final byte FLAG_UNSET = 0x00; + //private static final byte FLAG_CONTINUED = 0x01; + private static final byte FLAG_FIRST = 0x02; + private static final byte FLAG_LAST = 0x04; + + private final static byte SEGMENTS_PER_PACKET = 50;// used in ffmpeg, which is near 1 second at 48kHz + private final static byte HEADER_CHECKSUM_OFFSET = 22; + + private boolean done = false; + private boolean parsed = false; + + private SharpStream source; + private SharpStream output; + + private int sequence_count = 0; + private final int STREAM_ID; + + private WebMReader webm = null; + private WebMTrack webm_track = null; + private int track_index = 0; + + public OggFromWebMWriter(@NonNull SharpStream source, @NonNull SharpStream target) { + if (!source.canRead() || !source.canRewind()) { + throw new IllegalArgumentException("source stream must be readable and allows seeking"); + } + if (!target.canWrite() || !target.canRewind()) { + throw new IllegalArgumentException("output stream must be writable and allows seeking"); + } + + this.source = source; + this.output = target; + + this.STREAM_ID = (new Random(System.currentTimeMillis())).nextInt(); + + populate_crc32_table(); + } + + public boolean isDone() { + return done; + } + + public boolean isParsed() { + return parsed; + } + + public WebMTrack[] getTracksFromSource() throws IllegalStateException { + if (!parsed) { + throw new IllegalStateException("source must be parsed first"); + } + + return webm.getAvailableTracks(); + } + + public void parseSource() throws IOException, IllegalStateException { + if (done) { + throw new IllegalStateException("already done"); + } + if (parsed) { + throw new IllegalStateException("already parsed"); + } + + try { + webm = new WebMReader(source); + webm.parse(); + webm_segment = webm.getNextSegment(); + } finally { + parsed = true; + } + } + + public void selectTrack(int trackIndex) throws IOException { + if (!parsed) { + throw new IllegalStateException("source must be parsed first"); + } + if (done) { + throw new IOException("already done"); + } + if (webm_track != null) { + throw new IOException("tracks already selected"); + } + + switch (webm.getAvailableTracks()[trackIndex].kind) { + case Audio: + case Video: + break; + default: + throw new UnsupportedOperationException("the track must an audio or video stream"); + } + + try { + webm_track = webm.selectTrack(trackIndex); + track_index = trackIndex; + } finally { + parsed = true; + } + } + + @Override + public void close() throws IOException { + done = true; + parsed = true; + + webm_track = null; + webm = null; + + if (!output.isClosed()) { + output.flush(); + } + + source.close(); + output.close(); + } + + public void build() throws IOException { + float resolution; + int read; + byte[] buffer; + int checksum; + byte flag = FLAG_FIRST;// obligatory + + switch (webm_track.kind) { + case Audio: + resolution = getSampleFrequencyFromTrack(webm_track.bMetadata); + if (resolution == 0f) { + throw new RuntimeException("cannot get the audio sample rate"); + } + break; + case Video: + // WARNING: untested + if (webm_track.defaultDuration == 0) { + throw new RuntimeException("missing default frame time"); + } + resolution = 1000f / ((float) webm_track.defaultDuration / webm_segment.info.timecodeScale); + break; + default: + throw new RuntimeException("not implemented"); + } + + /* step 1.1: write codec init data, in most cases must be present */ + if (webm_track.codecPrivate != null) { + addPacketSegment(webm_track.codecPrivate.length); + dump_packetHeader(flag, 0x00, webm_track.codecPrivate); + flag = FLAG_UNSET; + } + + /* step 1.2: write metadata */ + buffer = make_metadata(); + if (buffer != null) { + addPacketSegment(buffer.length); + dump_packetHeader(flag, 0x00, buffer); + flag = FLAG_UNSET; + } + + buffer = new byte[8 * 1024]; + + /* step 1.3: write headers */ + long approx_packets = webm_segment.info.duration / webm_segment.info.timecodeScale; + approx_packets = approx_packets / (approx_packets / SEGMENTS_PER_PACKET); + + ArrayList pending_offsets = new ArrayList<>((int) approx_packets); + ArrayList pending_checksums = new ArrayList<>((int) approx_packets); + ArrayList data_offsets = new ArrayList<>((int) approx_packets); + + int page_size = 0; + SimpleBlock bloq; + + while (webm_segment != null) { + bloq = getNextBlock(); + + if (bloq != null && addPacketSegment(bloq.dataSize)) { + page_size += bloq.dataSize; + + if (segment_table_size < SEGMENTS_PER_PACKET) { + continue; + } + + // calculate the current packet duration using the next block + bloq = getNextBlock(); + } + + double elapsed_ns = webm_track.codecDelay; + + if (bloq == null) { + flag = FLAG_LAST; + elapsed_ns += webm_block_last_timecode; + + if (webm_track.defaultDuration > 0) { + elapsed_ns += webm_track.defaultDuration; + } else { + // hardcoded way, guess the sample duration + elapsed_ns += webm_block_near_duration; + } + } else { + elapsed_ns += bloq.absoluteTimeCodeNs; + } + + // get the sample count in the page + elapsed_ns = (elapsed_ns / 1000000000d) * resolution; + elapsed_ns = Math.ceil(elapsed_ns); + + long offset = output_offset + HEADER_CHECKSUM_OFFSET; + pending_offsets.add(offset); + + checksum = dump_packetHeader(flag, (long) elapsed_ns, null); + pending_checksums.add(checksum); + + data_offsets.add((short) (output_offset - offset)); + + // reserve space in the page + while (page_size > 0) { + int write = Math.min(page_size, buffer.length); + out_write(buffer, write); + page_size -= write; + } + + webm_block = bloq; + } + + /* step 2.1: write stream data */ + output.rewind(); + output_offset = 0; + + source.rewind(); + + webm = new WebMReader(source); + webm.parse(); + webm_track = webm.selectTrack(track_index); + + for (int i = 0; i < pending_offsets.size(); i++) { + checksum = pending_checksums.get(i); + segment_table_size = 0; + + out_seek(pending_offsets.get(i) + data_offsets.get(i)); + + while (segment_table_size < SEGMENTS_PER_PACKET) { + bloq = getNextBlock(); + + if (bloq == null || !addPacketSegment(bloq.dataSize)) { + webm_block = bloq;// use this block later (if not null) + break; + } + + // NOTE: calling bloq.data.close() is unnecessary + while ((read = bloq.data.read(buffer)) != -1) { + out_write(buffer, read); + checksum = calc_crc32(checksum, buffer, read); + } + } + + pending_checksums.set(i, checksum); + } + + /* step 2.2: write every checksum */ + output.rewind(); + output_offset = 0; + buffer = new byte[4]; + + ByteBuffer buff = ByteBuffer.wrap(buffer); + buff.order(ByteOrder.LITTLE_ENDIAN); + + for (int i = 0; i < pending_checksums.size(); i++) { + out_seek(pending_offsets.get(i)); + buff.putInt(0, pending_checksums.get(i)); + out_write(buffer); + } + } + + private int dump_packetHeader(byte flag, long gran_pos, byte[] immediate_page) throws IOException { + ByteBuffer buffer = ByteBuffer.allocate(27 + segment_table_size); + + buffer.putInt(0x4F676753);// "OggS" binary string + buffer.put((byte) 0x00);// version + buffer.put(flag);// type + + buffer.order(ByteOrder.LITTLE_ENDIAN); + + buffer.putLong(gran_pos);// granulate position + + buffer.putInt(STREAM_ID);// bitstream serial number + buffer.putInt(sequence_count++);// page sequence number + + buffer.putInt(0x00);// page checksum + + buffer.order(ByteOrder.BIG_ENDIAN); + + buffer.put((byte) segment_table_size);// segment table + buffer.put(segment_table, 0, segment_table_size);// segment size + + segment_table_size = 0;// clear segment table for next header + + byte[] buff = buffer.array(); + int checksum_crc32 = calc_crc32(0x00, buff, buff.length); + + if (immediate_page != null) { + checksum_crc32 = calc_crc32(checksum_crc32, immediate_page, immediate_page.length); + buffer.order(ByteOrder.LITTLE_ENDIAN); + buffer.putInt(HEADER_CHECKSUM_OFFSET, checksum_crc32); + + out_write(buff); + out_write(immediate_page); + return 0; + } + + out_write(buff); + return checksum_crc32; + } + + @Nullable + private byte[] make_metadata() { + if ("A_OPUS".equals(webm_track.codecId)) { + return new byte[]{ + 0x4F, 0x70, 0x75, 0x73, 0x54, 0x61, 0x67, 0x73,// "OpusTags" binary string + 0x07, 0x00, 0x00, 0x00,// writting application string size + 0x4E, 0x65, 0x77, 0x50, 0x69, 0x70, 0x65,// "NewPipe" binary string + 0x00, 0x00, 0x00, 0x00// additional tags count (zero means no tags) + }; + } else if ("A_VORBIS".equals(webm_track.codecId)) { + return new byte[]{ + 0x03,// ???????? + 0x76, 0x6f, 0x72, 0x62, 0x69, 0x73,// "vorbis" binary string + 0x07, 0x00, 0x00, 0x00,// writting application string size + 0x4E, 0x65, 0x77, 0x50, 0x69, 0x70, 0x65,// "NewPipe" binary string + 0x01, 0x00, 0x00, 0x00,// additional tags count (zero means no tags) + + /* + // whole file duration (not implemented) + 0x44,// tag string size + 0x55, 0x52, 0x41, 0x54, 0x49, 0x4F, 0x4E, 0x3D, 0x30, 0x30, 0x3A, 0x30, 0x30, 0x3A, 0x30, + 0x30, 0x2E, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30 + */ + 0x0F,// tag string size + 0x00, 0x00, 0x00, 0x45, 0x4E, 0x43, 0x4F, 0x44, 0x45, 0x52, 0x3D,// "ENCODER=" binary string + 0x4E, 0x65, 0x77, 0x50, 0x69, 0x70, 0x65,// "NewPipe" binary string + 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00// ???????? + }; + } + + // not implemented for the desired codec + return null; + } + + // + private Segment webm_segment = null; + private Cluster webm_cluter = null; + private SimpleBlock webm_block = null; + private long webm_block_last_timecode = 0; + private long webm_block_near_duration = 0; + + private SimpleBlock getNextBlock() throws IOException { + SimpleBlock res; + + if (webm_block != null) { + res = webm_block; + webm_block = null; + return res; + } + + if (webm_segment == null) { + webm_segment = webm.getNextSegment(); + if (webm_segment == null) { + return null;// no more blocks in the selected track + } + } + + if (webm_cluter == null) { + webm_cluter = webm_segment.getNextCluster(); + if (webm_cluter == null) { + webm_segment = null; + return getNextBlock(); + } + } + + res = webm_cluter.getNextSimpleBlock(); + if (res == null) { + webm_cluter = null; + return getNextBlock(); + } + + webm_block_near_duration = res.absoluteTimeCodeNs - webm_block_last_timecode; + webm_block_last_timecode = res.absoluteTimeCodeNs; + + return res; + } + + private float getSampleFrequencyFromTrack(byte[] bMetadata) { + // hardcoded way + ByteBuffer buffer = ByteBuffer.wrap(bMetadata); + + while (buffer.remaining() >= 6) { + int id = buffer.getShort() & 0xFFFF; + if (id == 0x0000B584) { + return buffer.getFloat(); + } + } + + return 0f; + } + // + + // + private int segment_table_size = 0; + private final byte[] segment_table = new byte[255]; + + private boolean addPacketSegment(long size) { + // check if possible add the segment, without overflow the table + int available = (segment_table.length - segment_table_size) * 255; + if (available < size) { + return false;// not enough space on the page + } + + while (size > 0) { + segment_table[segment_table_size++] = (byte) Math.min(size, 255); + size -= 255; + } + + return true; + } + // + + // + private long output_offset = 0; + + private void out_write(byte[] buffer) throws IOException { + output.write(buffer); + output_offset += buffer.length; + } + + private void out_write(byte[] buffer, int size) throws IOException { + output.write(buffer, 0, size); + output_offset += size; + } + + private void out_seek(long offset) throws IOException { + //if (output.canSeek()) { output.seek(offset); } + output.skip(offset - output_offset); + output_offset = offset; + } + // + + // + private final int[] crc32_table = new int[256]; + + private void populate_crc32_table() { + for (int i = 0; i < 0x100; i++) { + int crc = i << 24; + for (int j = 0; j < 8; j++) { + long b = crc >>> 31; + crc <<= 1; + crc ^= (int) (0x100000000L - b) & 0x04c11db7; + } + crc32_table[i] = crc; + } + } + + private int calc_crc32(int initial_crc, byte[] buffer, int size) { + for (int i = 0; i < size; i++) { + int reg = (initial_crc >>> 24) & 0xff; + initial_crc = (initial_crc << 8) ^ crc32_table[reg ^ (buffer[i] & 0xff)]; + } + + return initial_crc; + } + // +} diff --git a/app/src/main/java/org/schabi/newpipe/streams/WebMReader.java b/app/src/main/java/org/schabi/newpipe/streams/WebMReader.java index 0c635ebe3..13c15370d 100644 --- a/app/src/main/java/org/schabi/newpipe/streams/WebMReader.java +++ b/app/src/main/java/org/schabi/newpipe/streams/WebMReader.java @@ -15,7 +15,7 @@ import java.util.NoSuchElementException; */ public class WebMReader { - // + // private final static int ID_EMBL = 0x0A45DFA3; private final static int ID_EMBLReadVersion = 0x02F7; private final static int ID_EMBLDocType = 0x0282; @@ -37,10 +37,13 @@ public class WebMReader { private final static int ID_Audio = 0x61; private final static int ID_DefaultDuration = 0x3E383; private final static int ID_FlagLacing = 0x1C; + private final static int ID_CodecDelay = 0x16AA; private final static int ID_Cluster = 0x0F43B675; private final static int ID_Timecode = 0x67; private final static int ID_SimpleBlock = 0x23; + private final static int ID_Block = 0x21; + private final static int ID_GroupBlock = 0x20; // public enum TrackKind { @@ -96,7 +99,7 @@ public class WebMReader { } ensure(segment.ref); - + // WARNING: track cannot be the same or have different index in new segments Element elem = untilElement(null, ID_Segment); if (elem == null) { done = true; @@ -189,6 +192,9 @@ public class WebMReader { Element elem; while (ref == null ? stream.available() : (stream.position() < (ref.offset + ref.size))) { elem = readElement(); + if (expected.length < 1) { + return elem; + } for (int type : expected) { if (elem.type == type) { return elem; @@ -300,9 +306,7 @@ public class WebMReader { WebMTrack entry = new WebMTrack(); boolean drop = false; Element elem; - while ((elem = untilElement(elem_trackEntry, - ID_TrackNumber, ID_TrackType, ID_CodecID, ID_CodecPrivate, ID_FlagLacing, ID_DefaultDuration, ID_Audio, ID_Video - )) != null) { + while ((elem = untilElement(elem_trackEntry)) != null) { switch (elem.type) { case ID_TrackNumber: entry.trackNumber = readNumber(elem); @@ -326,8 +330,9 @@ public class WebMReader { case ID_FlagLacing: drop = readNumber(elem) != lacingExpected; break; + case ID_CodecDelay: + entry.codecDelay = readNumber(elem); default: - System.out.println(); break; } ensure(elem); @@ -360,12 +365,13 @@ public class WebMReader { private SimpleBlock readSimpleBlock(Element ref) throws IOException { SimpleBlock obj = new SimpleBlock(ref); - obj.dataSize = stream.position(); obj.trackNumber = readEncodedNumber(); obj.relativeTimeCode = stream.readShort(); obj.flags = (byte) stream.read(); obj.dataSize = (ref.offset + ref.size) - stream.position(); + obj.createdFromBlock = ref.type == ID_Block; + // NOTE: lacing is not implemented, and will be mixed with the stream data if (obj.dataSize < 0) { throw new IOException(String.format("Unexpected SimpleBlock element size, missing %s bytes", -obj.dataSize)); } @@ -409,6 +415,7 @@ public class WebMReader { public byte[] bMetadata; public TrackKind kind; public long defaultDuration; + public long codecDelay; } public class Segment { @@ -448,6 +455,7 @@ public class WebMReader { public class SimpleBlock { public InputStream data; + public boolean createdFromBlock; SimpleBlock(Element ref) { this.ref = ref; @@ -455,6 +463,7 @@ public class WebMReader { public long trackNumber; public short relativeTimeCode; + public long absoluteTimeCodeNs; public byte flags; public long dataSize; private final Element ref; @@ -468,33 +477,55 @@ public class WebMReader { Element ref; SimpleBlock currentSimpleBlock = null; + Element currentBlockGroup = null; public long timecode; Cluster(Element ref) { this.ref = ref; } - boolean check() { + boolean insideClusterBounds() { return stream.position() >= (ref.offset + ref.size); } public SimpleBlock getNextSimpleBlock() throws IOException { - if (check()) { + if (insideClusterBounds()) { return null; } - if (currentSimpleBlock != null) { + + if (currentBlockGroup != null) { + ensure(currentBlockGroup); + currentBlockGroup = null; + currentSimpleBlock = null; + } else if (currentSimpleBlock != null) { ensure(currentSimpleBlock.ref); } - while (!check()) { - Element elem = untilElement(ref, ID_SimpleBlock); + while (!insideClusterBounds()) { + Element elem = untilElement(ref, ID_SimpleBlock, ID_GroupBlock); if (elem == null) { return null; } + if (elem.type == ID_GroupBlock) { + currentBlockGroup = elem; + elem = untilElement(currentBlockGroup, ID_Block); + + if (elem == null) { + ensure(currentBlockGroup); + currentBlockGroup = null; + continue; + } + } + currentSimpleBlock = readSimpleBlock(elem); if (currentSimpleBlock.trackNumber == tracks[selectedTrack].trackNumber) { currentSimpleBlock.data = stream.getView((int) currentSimpleBlock.dataSize); + + // calculate the timestamp in nanoseconds + currentSimpleBlock.absoluteTimeCodeNs = currentSimpleBlock.relativeTimeCode + this.timecode; + currentSimpleBlock.absoluteTimeCodeNs *= segment.info.timecodeScale; + return currentSimpleBlock; } diff --git a/app/src/main/java/org/schabi/newpipe/streams/WebMWriter.java b/app/src/main/java/org/schabi/newpipe/streams/WebMWriter.java index e5881fd0b..1bf994b1e 100644 --- a/app/src/main/java/org/schabi/newpipe/streams/WebMWriter.java +++ b/app/src/main/java/org/schabi/newpipe/streams/WebMWriter.java @@ -8,6 +8,7 @@ import org.schabi.newpipe.streams.WebMReader.SimpleBlock; import org.schabi.newpipe.streams.WebMReader.WebMTrack; import org.schabi.newpipe.streams.io.SharpStream; +import java.io.Closeable; import java.io.IOException; import java.io.InputStream; import java.nio.ByteBuffer; @@ -17,7 +18,7 @@ import java.util.ArrayList; /** * @author kapodamy */ -public class WebMWriter { +public class WebMWriter implements Closeable { private final static int BUFFER_SIZE = 8 * 1024; private final static int DEFAULT_TIMECODE_SCALE = 1000000; @@ -35,7 +36,7 @@ public class WebMWriter { private long written = 0; private Segment[] readersSegment; - private Cluster[] readersCluter; + private Cluster[] readersCluster; private int[] predefinedDurations; @@ -81,7 +82,7 @@ public class WebMWriter { public void selectTracks(int... trackIndex) throws IOException { try { readersSegment = new Segment[readers.length]; - readersCluter = new Cluster[readers.length]; + readersCluster = new Cluster[readers.length]; predefinedDurations = new int[readers.length]; for (int i = 0; i < readers.length; i++) { @@ -102,6 +103,7 @@ public class WebMWriter { return parsed; } + @Override public void close() { done = true; parsed = true; @@ -114,7 +116,7 @@ public class WebMWriter { readers = null; infoTracks = null; readersSegment = null; - readersCluter = null; + readersCluster = null; outBuffer = null; } @@ -334,17 +336,17 @@ public class WebMWriter { } } - if (readersCluter[internalTrackId] == null) { - readersCluter[internalTrackId] = readersSegment[internalTrackId].getNextCluster(); - if (readersCluter[internalTrackId] == null) { + if (readersCluster[internalTrackId] == null) { + readersCluster[internalTrackId] = readersSegment[internalTrackId].getNextCluster(); + if (readersCluster[internalTrackId] == null) { readersSegment[internalTrackId] = null; return getNextBlockFrom(internalTrackId); } } - SimpleBlock res = readersCluter[internalTrackId].getNextSimpleBlock(); + SimpleBlock res = readersCluster[internalTrackId].getNextSimpleBlock(); if (res == null) { - readersCluter[internalTrackId] = null; + readersCluster[internalTrackId] = null; return new Block();// fake block to indicate the end of the cluster } @@ -353,16 +355,11 @@ public class WebMWriter { bloq.dataSize = (int) res.dataSize; bloq.trackNumber = internalTrackId; bloq.flags = res.flags; - bloq.absoluteTimecode = convertTimecode(res.relativeTimeCode, readersSegment[internalTrackId].info.timecodeScale); - bloq.absoluteTimecode += readersCluter[internalTrackId].timecode; + bloq.absoluteTimecode = res.absoluteTimeCodeNs / DEFAULT_TIMECODE_SCALE; return bloq; } - private short convertTimecode(int time, long oldTimeScale) { - return (short) (time * (DEFAULT_TIMECODE_SCALE / oldTimeScale)); - } - private void seekTo(SharpStream stream, long offset) throws IOException { if (stream.canSeek()) { stream.seek(offset); diff --git a/app/src/main/java/us/shandian/giga/postprocessing/OggFromWebmDemuxer.java b/app/src/main/java/us/shandian/giga/postprocessing/OggFromWebmDemuxer.java new file mode 100644 index 000000000..65aa30fa3 --- /dev/null +++ b/app/src/main/java/us/shandian/giga/postprocessing/OggFromWebmDemuxer.java @@ -0,0 +1,44 @@ +package us.shandian.giga.postprocessing; + +import android.support.annotation.NonNull; + +import org.schabi.newpipe.streams.OggFromWebMWriter; +import org.schabi.newpipe.streams.io.SharpStream; + +import java.io.IOException; +import java.nio.ByteBuffer; + +class OggFromWebmDemuxer extends Postprocessing { + + OggFromWebmDemuxer() { + super(false, true, ALGORITHM_OGG_FROM_WEBM_DEMUXER); + } + + @Override + boolean test(SharpStream... sources) throws IOException { + ByteBuffer buffer = ByteBuffer.allocate(4); + sources[0].read(buffer.array()); + + // youtube uses WebM as container, but the file extension (format suffix) is "*.opus" + // check if the file is a webm/mkv file before proceed + + switch (buffer.getInt()) { + case 0x1a45dfa3: + return true;// webm + case 0x4F676753: + return false;// ogg + } + + throw new UnsupportedOperationException("file not recognized, failed to demux the audio stream"); + } + + @Override + int process(SharpStream out, @NonNull SharpStream... sources) throws IOException { + OggFromWebMWriter demuxer = new OggFromWebMWriter(sources[0], out); + demuxer.parseSource(); + demuxer.selectTrack(0); + demuxer.build(); + + return OK_RESULT; + } +} diff --git a/app/src/main/java/us/shandian/giga/postprocessing/Postprocessing.java b/app/src/main/java/us/shandian/giga/postprocessing/Postprocessing.java index 22cc325d5..92510c3df 100644 --- a/app/src/main/java/us/shandian/giga/postprocessing/Postprocessing.java +++ b/app/src/main/java/us/shandian/giga/postprocessing/Postprocessing.java @@ -28,6 +28,7 @@ public abstract class Postprocessing implements Serializable { public transient static final String ALGORITHM_WEBM_MUXER = "webm"; public transient static final String ALGORITHM_MP4_FROM_DASH_MUXER = "mp4D-mp4"; public transient static final String ALGORITHM_M4A_NO_DASH = "mp4D-m4a"; + public transient static final String ALGORITHM_OGG_FROM_WEBM_DEMUXER = "webm-ogg-d"; public static Postprocessing getAlgorithm(@NonNull String algorithmName, String[] args) { Postprocessing instance; @@ -45,6 +46,9 @@ public abstract class Postprocessing implements Serializable { case ALGORITHM_M4A_NO_DASH: instance = new M4aNoDash(); break; + case ALGORITHM_OGG_FROM_WEBM_DEMUXER: + instance = new OggFromWebmDemuxer(); + break; /*case "example-algorithm": instance = new ExampleAlgorithm();*/ default: @@ -212,7 +216,7 @@ public abstract class Postprocessing implements Serializable { * * @param out output stream * @param sources files to be processed - * @return a error code, 0 means the operation was successful + * @return an error code, {@code OK_RESULT} means the operation was successful * @throws IOException if an I/O error occurs. */ abstract int process(SharpStream out, SharpStream... sources) throws IOException; From 0cdfa6e377e48002247ec515b7eb2a3353f5c007 Mon Sep 17 00:00:00 2001 From: kapodamy Date: Wed, 25 Sep 2019 16:24:52 -0300 Subject: [PATCH 03/28] rewrite OggFromWebMWriter * reduce the number of iterations over the output file (less seeking) * fix audio samples with size of 255 do not handled correctly in the segment table (allows writing audio streams with 70kbps and 160kbps bitrate) * add support for VORBIS codec metadata * write packets based on the timestamp --- .../newpipe/streams/OggFromWebMWriter.java | 348 ++++++++++-------- .../postprocessing/OggFromWebmDemuxer.java | 4 +- 2 files changed, 203 insertions(+), 149 deletions(-) diff --git a/app/src/main/java/org/schabi/newpipe/streams/OggFromWebMWriter.java b/app/src/main/java/org/schabi/newpipe/streams/OggFromWebMWriter.java index 2b3d778c6..091ae6d2a 100644 --- a/app/src/main/java/org/schabi/newpipe/streams/OggFromWebMWriter.java +++ b/app/src/main/java/org/schabi/newpipe/streams/OggFromWebMWriter.java @@ -23,12 +23,16 @@ import javax.annotation.Nullable; public class OggFromWebMWriter implements Closeable { private static final byte FLAG_UNSET = 0x00; - //private static final byte FLAG_CONTINUED = 0x01; + private static final byte FLAG_CONTINUED = 0x01; private static final byte FLAG_FIRST = 0x02; private static final byte FLAG_LAST = 0x04; - private final static byte SEGMENTS_PER_PACKET = 50;// used in ffmpeg, which is near 1 second at 48kHz private final static byte HEADER_CHECKSUM_OFFSET = 22; + private final static byte HEADER_SIZE = 27; + + private final static short BUFFER_SIZE = 8 * 1024;// 8KiB + + private final static int TIME_SCALE_NS = 1000000000; private boolean done = false; private boolean parsed = false; @@ -38,10 +42,23 @@ public class OggFromWebMWriter implements Closeable { private int sequence_count = 0; private final int STREAM_ID; + private byte packet_flag = FLAG_FIRST; + private int track_index = 0; private WebMReader webm = null; private WebMTrack webm_track = null; - private int track_index = 0; + private Segment webm_segment = null; + private Cluster webm_cluster = null; + private SimpleBlock webm_block = null; + + private long webm_block_last_timecode = 0; + private long webm_block_near_duration = 0; + + private short segment_table_size = 0; + private final byte[] segment_table = new byte[255]; + private long segment_table_next_timestamp = TIME_SCALE_NS; + + private final int[] crc32_table = new int[256]; public OggFromWebMWriter(@NonNull SharpStream source, @NonNull SharpStream target) { if (!source.canRead() || !source.canRewind()) { @@ -139,9 +156,8 @@ public class OggFromWebMWriter implements Closeable { float resolution; int read; byte[] buffer; - int checksum; - byte flag = FLAG_FIRST;// obligatory + /* step 1: get the amount of frames per seconds */ switch (webm_track.kind) { case Audio: resolution = getSampleFrequencyFromTrack(webm_track.bMetadata); @@ -160,52 +176,65 @@ public class OggFromWebMWriter implements Closeable { throw new RuntimeException("not implemented"); } - /* step 1.1: write codec init data, in most cases must be present */ + /* step 2a: create packet with code init data */ + ArrayList data_extra = new ArrayList<>(4); + if (webm_track.codecPrivate != null) { addPacketSegment(webm_track.codecPrivate.length); - dump_packetHeader(flag, 0x00, webm_track.codecPrivate); - flag = FLAG_UNSET; + ByteBuffer buff = byte_buffer(HEADER_SIZE + segment_table_size + webm_track.codecPrivate.length); + + make_packetHeader(0x00, buff, webm_track.codecPrivate); + data_extra.add(buff.array()); } - /* step 1.2: write metadata */ + /* step 2b: create packet with metadata */ buffer = make_metadata(); if (buffer != null) { addPacketSegment(buffer.length); - dump_packetHeader(flag, 0x00, buffer); - flag = FLAG_UNSET; + ByteBuffer buff = byte_buffer(HEADER_SIZE + segment_table_size + buffer.length); + + make_packetHeader(0x00, buff, buffer); + data_extra.add(buff.array()); } - buffer = new byte[8 * 1024]; - /* step 1.3: write headers */ - long approx_packets = webm_segment.info.duration / webm_segment.info.timecodeScale; - approx_packets = approx_packets / (approx_packets / SEGMENTS_PER_PACKET); - - ArrayList pending_offsets = new ArrayList<>((int) approx_packets); - ArrayList pending_checksums = new ArrayList<>((int) approx_packets); - ArrayList data_offsets = new ArrayList<>((int) approx_packets); - - int page_size = 0; + /* step 3: calculate amount of packets */ SimpleBlock bloq; + int reserve_header = 0; + int headers_amount = 0; while (webm_segment != null) { bloq = getNextBlock(); - if (bloq != null && addPacketSegment(bloq.dataSize)) { - page_size += bloq.dataSize; - - if (segment_table_size < SEGMENTS_PER_PACKET) { - continue; - } - - // calculate the current packet duration using the next block - bloq = getNextBlock(); + if (addPacketSegment(bloq)) { + continue; } + reserve_header += HEADER_SIZE + segment_table_size;// header size + clearSegmentTable(); + webm_block = bloq; + headers_amount++; + } + + /* step 4: create packet headers */ + rewind_source(); + + ByteBuffer headers = byte_buffer(reserve_header); + short[] headers_size = new short[headers_amount]; + int header_index = 0; + + while (webm_segment != null) { + bloq = getNextBlock(); + + if (addPacketSegment(bloq)) { + continue; + } + + // calculate the current packet duration using the next block double elapsed_ns = webm_track.codecDelay; if (bloq == null) { - flag = FLAG_LAST; + packet_flag = FLAG_LAST;// note: if the flag is FLAG_CONTINUED, is changed elapsed_ns += webm_block_last_timecode; if (webm_track.defaultDuration > 0) { @@ -219,84 +248,83 @@ public class OggFromWebMWriter implements Closeable { } // get the sample count in the page - elapsed_ns = (elapsed_ns / 1000000000d) * resolution; - elapsed_ns = Math.ceil(elapsed_ns); - - long offset = output_offset + HEADER_CHECKSUM_OFFSET; - pending_offsets.add(offset); - - checksum = dump_packetHeader(flag, (long) elapsed_ns, null); - pending_checksums.add(checksum); - - data_offsets.add((short) (output_offset - offset)); - - // reserve space in the page - while (page_size > 0) { - int write = Math.min(page_size, buffer.length); - out_write(buffer, write); - page_size -= write; - } + elapsed_ns = elapsed_ns / TIME_SCALE_NS; + elapsed_ns = Math.ceil(elapsed_ns * resolution); + // create header + headers_size[header_index++] = make_packetHeader((long) elapsed_ns, headers, null); webm_block = bloq; } - /* step 2.1: write stream data */ - output.rewind(); - output_offset = 0; - source.rewind(); + /* step 5: calculate checksums */ + rewind_source(); - webm = new WebMReader(source); - webm.parse(); - webm_track = webm.selectTrack(track_index); + int offset = 0; + buffer = new byte[BUFFER_SIZE]; - for (int i = 0; i < pending_offsets.size(); i++) { - checksum = pending_checksums.get(i); - segment_table_size = 0; + for (header_index = 0; header_index < headers_size.length; header_index++) { + int checksum_offset = offset + HEADER_CHECKSUM_OFFSET; + int checksum = headers.getInt(checksum_offset); - out_seek(pending_offsets.get(i) + data_offsets.get(i)); - - while (segment_table_size < SEGMENTS_PER_PACKET) { + while (webm_segment != null) { bloq = getNextBlock(); - if (bloq == null || !addPacketSegment(bloq.dataSize)) { - webm_block = bloq;// use this block later (if not null) + if (!addPacketSegment(bloq)) { + clearSegmentTable(); + webm_block = bloq; break; } - // NOTE: calling bloq.data.close() is unnecessary - while ((read = bloq.data.read(buffer)) != -1) { - out_write(buffer, read); - checksum = calc_crc32(checksum, buffer, read); + // calculate page checksum + while ((read = bloq.data.read(buffer)) > 0) { + checksum = calc_crc32(checksum, buffer, 0, read); } } - pending_checksums.set(i, checksum); + headers.putInt(checksum_offset, checksum); + offset += headers_size[header_index]; } - /* step 2.2: write every checksum */ - output.rewind(); - output_offset = 0; - buffer = new byte[4]; + /* step 6: write extra headers */ + rewind_source(); - ByteBuffer buff = ByteBuffer.wrap(buffer); - buff.order(ByteOrder.LITTLE_ENDIAN); + for (byte[] buff : data_extra) { + output.write(buff); + } - for (int i = 0; i < pending_checksums.size(); i++) { - out_seek(pending_offsets.get(i)); - buff.putInt(0, pending_checksums.get(i)); - out_write(buffer); + /* step 7: write stream packets */ + byte[] headers_buffers = headers.array(); + offset = 0; + buffer = new byte[BUFFER_SIZE]; + + for (header_index = 0; header_index < headers_size.length; header_index++) { + output.write(headers_buffers, offset, headers_size[header_index]); + offset += headers_size[header_index]; + + while (webm_segment != null) { + bloq = getNextBlock(); + + if (addPacketSegment(bloq)) { + while ((read = bloq.data.read(buffer)) > 0) { + output.write(buffer, 0, read); + } + } else { + clearSegmentTable(); + webm_block = bloq; + break; + } + } } } - private int dump_packetHeader(byte flag, long gran_pos, byte[] immediate_page) throws IOException { - ByteBuffer buffer = ByteBuffer.allocate(27 + segment_table_size); + private short make_packetHeader(long gran_pos, ByteBuffer buffer, byte[] immediate_page) { + int offset = buffer.position(); + short length = HEADER_SIZE; - buffer.putInt(0x4F676753);// "OggS" binary string + buffer.putInt(0x5367674f);// "OggS" binary string in little-endian buffer.put((byte) 0x00);// version - buffer.put(flag);// type - - buffer.order(ByteOrder.LITTLE_ENDIAN); + buffer.put(packet_flag);// type buffer.putLong(gran_pos);// granulate position @@ -305,28 +333,24 @@ public class OggFromWebMWriter implements Closeable { buffer.putInt(0x00);// page checksum - buffer.order(ByteOrder.BIG_ENDIAN); - buffer.put((byte) segment_table_size);// segment table buffer.put(segment_table, 0, segment_table_size);// segment size - segment_table_size = 0;// clear segment table for next header + length += segment_table_size; - byte[] buff = buffer.array(); - int checksum_crc32 = calc_crc32(0x00, buff, buff.length); + clearSegmentTable();// clear segment table for next header + + int checksum_crc32 = calc_crc32(0x00, buffer.array(), offset, length); if (immediate_page != null) { - checksum_crc32 = calc_crc32(checksum_crc32, immediate_page, immediate_page.length); - buffer.order(ByteOrder.LITTLE_ENDIAN); - buffer.putInt(HEADER_CHECKSUM_OFFSET, checksum_crc32); - - out_write(buff); - out_write(immediate_page); - return 0; + checksum_crc32 = calc_crc32(checksum_crc32, immediate_page, 0, immediate_page.length); + System.arraycopy(immediate_page, 0, buffer.array(), length, immediate_page.length); + segment_table_next_timestamp -= TIME_SCALE_NS; } - out_write(buff); - return checksum_crc32; + buffer.putInt(offset + HEADER_CHECKSUM_OFFSET, checksum_crc32); + + return length; } @Nullable @@ -334,7 +358,7 @@ public class OggFromWebMWriter implements Closeable { if ("A_OPUS".equals(webm_track.codecId)) { return new byte[]{ 0x4F, 0x70, 0x75, 0x73, 0x54, 0x61, 0x67, 0x73,// "OpusTags" binary string - 0x07, 0x00, 0x00, 0x00,// writting application string size + 0x07, 0x00, 0x00, 0x00,// writing application string size 0x4E, 0x65, 0x77, 0x50, 0x69, 0x70, 0x65,// "NewPipe" binary string 0x00, 0x00, 0x00, 0x00// additional tags count (zero means no tags) }; @@ -342,15 +366,15 @@ public class OggFromWebMWriter implements Closeable { return new byte[]{ 0x03,// ???????? 0x76, 0x6f, 0x72, 0x62, 0x69, 0x73,// "vorbis" binary string - 0x07, 0x00, 0x00, 0x00,// writting application string size + 0x07, 0x00, 0x00, 0x00,// writing application string size 0x4E, 0x65, 0x77, 0x50, 0x69, 0x70, 0x65,// "NewPipe" binary string 0x01, 0x00, 0x00, 0x00,// additional tags count (zero means no tags) /* - // whole file duration (not implemented) - 0x44,// tag string size - 0x55, 0x52, 0x41, 0x54, 0x49, 0x4F, 0x4E, 0x3D, 0x30, 0x30, 0x3A, 0x30, 0x30, 0x3A, 0x30, - 0x30, 0x2E, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30 + // whole file duration (not implemented) + 0x44,// tag string size + 0x55, 0x52, 0x41, 0x54, 0x49, 0x4F, 0x4E, 0x3D, 0x30, 0x30, 0x3A, 0x30, 0x30, 0x3A, 0x30, + 0x30, 0x2E, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30 */ 0x0F,// tag string size 0x00, 0x00, 0x00, 0x45, 0x4E, 0x43, 0x4F, 0x44, 0x45, 0x52, 0x3D,// "ENCODER=" binary string @@ -363,13 +387,26 @@ public class OggFromWebMWriter implements Closeable { return null; } - // - private Segment webm_segment = null; - private Cluster webm_cluter = null; - private SimpleBlock webm_block = null; - private long webm_block_last_timecode = 0; - private long webm_block_near_duration = 0; + private void rewind_source() throws IOException { + source.rewind(); + webm = new WebMReader(source); + webm.parse(); + webm_track = webm.selectTrack(track_index); + webm_segment = webm.getNextSegment(); + webm_cluster = null; + webm_block = null; + webm_block_last_timecode = 0L; + + segment_table_next_timestamp = TIME_SCALE_NS; + } + + private ByteBuffer byte_buffer(int size) { + return ByteBuffer.allocate(size).order(ByteOrder.LITTLE_ENDIAN); + } + + // + @Nullable private SimpleBlock getNextBlock() throws IOException { SimpleBlock res; @@ -386,17 +423,17 @@ public class OggFromWebMWriter implements Closeable { } } - if (webm_cluter == null) { - webm_cluter = webm_segment.getNextCluster(); - if (webm_cluter == null) { + if (webm_cluster == null) { + webm_cluster = webm_segment.getNextCluster(); + if (webm_cluster == null) { webm_segment = null; return getNextBlock(); } } - res = webm_cluter.getNextSimpleBlock(); + res = webm_cluster.getNextSimpleBlock(); if (res == null) { - webm_cluter = null; + webm_cluster = null; return getNextBlock(); } @@ -421,49 +458,64 @@ public class OggFromWebMWriter implements Closeable { } // - // - private int segment_table_size = 0; - private final byte[] segment_table = new byte[255]; + // + private void clearSegmentTable() { + if (packet_flag != FLAG_CONTINUED) { + segment_table_next_timestamp += TIME_SCALE_NS; + packet_flag = FLAG_UNSET; + } + segment_table_size = 0; + } - private boolean addPacketSegment(long size) { - // check if possible add the segment, without overflow the table + private boolean addPacketSegment(SimpleBlock block) { + if (block == null) { + return false; + } + + long timestamp = block.absoluteTimeCodeNs + webm_track.codecDelay; + + if (timestamp >= segment_table_next_timestamp) { + return false; + } + + boolean result = addPacketSegment((int) block.dataSize); + + if (!result && segment_table_next_timestamp < timestamp) { + // WARNING: ¡¡¡¡ not implemented (lack of documentation) !!!! + packet_flag = FLAG_CONTINUED; + } + + return result; + } + + private boolean addPacketSegment(int size) { int available = (segment_table.length - segment_table_size) * 255; + boolean extra = size == 255; + + if (extra) { + // add a zero byte entry in the table + // required to indicate the sample size is exactly 255 + available -= 255; + } + + // check if possible add the segment, without overflow the table if (available < size) { return false;// not enough space on the page } - while (size > 0) { + for (; size > 0; size -= 255) { segment_table[segment_table_size++] = (byte) Math.min(size, 255); - size -= 255; + } + + if (extra) { + segment_table[segment_table_size++] = 0x00; } return true; } // - // - private long output_offset = 0; - - private void out_write(byte[] buffer) throws IOException { - output.write(buffer); - output_offset += buffer.length; - } - - private void out_write(byte[] buffer, int size) throws IOException { - output.write(buffer, 0, size); - output_offset += size; - } - - private void out_seek(long offset) throws IOException { - //if (output.canSeek()) { output.seek(offset); } - output.skip(offset - output_offset); - output_offset = offset; - } - // - // - private final int[] crc32_table = new int[256]; - private void populate_crc32_table() { for (int i = 0; i < 0x100; i++) { int crc = i << 24; @@ -476,10 +528,12 @@ public class OggFromWebMWriter implements Closeable { } } - private int calc_crc32(int initial_crc, byte[] buffer, int size) { - for (int i = 0; i < size; i++) { + private int calc_crc32(int initial_crc, byte[] buffer, int offset, int size) { + size += offset; + + for (; offset < size; offset++) { int reg = (initial_crc >>> 24) & 0xff; - initial_crc = (initial_crc << 8) ^ crc32_table[reg ^ (buffer[i] & 0xff)]; + initial_crc = (initial_crc << 8) ^ crc32_table[reg ^ (buffer[offset] & 0xff)]; } return initial_crc; diff --git a/app/src/main/java/us/shandian/giga/postprocessing/OggFromWebmDemuxer.java b/app/src/main/java/us/shandian/giga/postprocessing/OggFromWebmDemuxer.java index 65aa30fa3..605c0a88b 100644 --- a/app/src/main/java/us/shandian/giga/postprocessing/OggFromWebmDemuxer.java +++ b/app/src/main/java/us/shandian/giga/postprocessing/OggFromWebmDemuxer.java @@ -11,7 +11,7 @@ import java.nio.ByteBuffer; class OggFromWebmDemuxer extends Postprocessing { OggFromWebmDemuxer() { - super(false, true, ALGORITHM_OGG_FROM_WEBM_DEMUXER); + super(true, true, ALGORITHM_OGG_FROM_WEBM_DEMUXER); } @Override @@ -24,7 +24,7 @@ class OggFromWebmDemuxer extends Postprocessing { switch (buffer.getInt()) { case 0x1a45dfa3: - return true;// webm + return true;// webm/mkv case 0x4F676753: return false;// ogg } From 3108c903dd043de3a9d7fc4ac17bb8591d05b361 Mon Sep 17 00:00:00 2001 From: Robin Date: Sun, 24 Nov 2019 16:24:39 +0100 Subject: [PATCH 04/28] squashed commit --- .../java/org/schabi/newpipe/player/VideoPlayer.java | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/app/src/main/java/org/schabi/newpipe/player/VideoPlayer.java b/app/src/main/java/org/schabi/newpipe/player/VideoPlayer.java index a92ab410e..0734139e1 100644 --- a/app/src/main/java/org/schabi/newpipe/player/VideoPlayer.java +++ b/app/src/main/java/org/schabi/newpipe/player/VideoPlayer.java @@ -290,8 +290,13 @@ public abstract class VideoPlayer extends BasePlayer String userPreferredLanguage = PreferenceManager.getDefaultSharedPreferences(context) .getString(context.getString(R.string.caption_user_set_key), null); - // english (auto-generated) - boolean searchForAutogenerated = userPreferredLanguage == null || + /* + * only search for autogenerated cc as fallback + * if "(auto-generated)" was not already selected + * we are only looking for "(" instead of "(auto-generated)" to hopefully get all + * internationalized variants such as "(automatisch-erzeugt)" and so on + */ + boolean searchForAutogenerated = userPreferredLanguage != null && !userPreferredLanguage.contains("("); // Add option for turning off caption @@ -326,7 +331,7 @@ public abstract class VideoPlayer extends BasePlayer return true; }); // apply caption language from previous user preference - if(userPreferredLanguage != null && (captionLanguage.equals(userPreferredLanguage) || + if (userPreferredLanguage != null && (captionLanguage.equals(userPreferredLanguage) || searchForAutogenerated && captionLanguage.startsWith(userPreferredLanguage) || userPreferredLanguage.contains("(") && captionLanguage.startsWith(userPreferredLanguage.substring(0, From c891f2f1eda4ade08bda3e74d9454b98743980b4 Mon Sep 17 00:00:00 2001 From: kapodamy Date: Sat, 28 Sep 2019 18:11:05 -0300 Subject: [PATCH 05/28] long-term downloads resume * recovery infrastructure * bump serialVersionUID of DownloadMission * misc cleanup in DownloadMission.java * remove unused/redundant from strings.xml --- .../newpipe/download/DownloadDialog.java | 34 ++- .../giga/get/DownloadInitializer.java | 15 ++ .../us/shandian/giga/get/DownloadMission.java | 96 ++++++-- .../giga/get/DownloadMissionRecover.java | 222 ++++++++++++++++++ .../shandian/giga/get/DownloadRunnable.java | 27 ++- .../giga/get/DownloadRunnableFallback.java | 11 +- .../giga/get/MissionRecoveryInfo.java | 79 +++++++ .../giga/service/DownloadManager.java | 1 - .../giga/service/DownloadManagerService.java | 36 ++- .../giga/ui/adapter/MissionAdapter.java | 12 +- app/src/main/res/values-ar/strings.xml | 1 - app/src/main/res/values-be/strings.xml | 1 - app/src/main/res/values-cmn/strings.xml | 1 - app/src/main/res/values-cs/strings.xml | 1 - app/src/main/res/values-da/strings.xml | 1 - app/src/main/res/values-de/strings.xml | 1 - app/src/main/res/values-el/strings.xml | 1 - app/src/main/res/values-es/strings.xml | 9 +- app/src/main/res/values-et/strings.xml | 1 - app/src/main/res/values-eu/strings.xml | 1 - app/src/main/res/values-fr/strings.xml | 1 - app/src/main/res/values-he/strings.xml | 1 - app/src/main/res/values-hr/strings.xml | 1 - app/src/main/res/values-id/strings.xml | 1 - app/src/main/res/values-it/strings.xml | 1 - app/src/main/res/values-ja/strings.xml | 1 - app/src/main/res/values-ko/strings.xml | 1 - app/src/main/res/values-ms/strings.xml | 1 - app/src/main/res/values-nb-rNO/strings.xml | 2 +- app/src/main/res/values-nl-rBE/strings.xml | 1 - app/src/main/res/values-nl/strings.xml | 1 - app/src/main/res/values-pa/strings.xml | 1 - app/src/main/res/values-pl/strings.xml | 1 - app/src/main/res/values-pt-rBR/strings.xml | 1 - app/src/main/res/values-pt/strings.xml | 1 - app/src/main/res/values-ru/strings.xml | 1 - app/src/main/res/values-sk/strings.xml | 1 - app/src/main/res/values-tr/strings.xml | 1 - app/src/main/res/values-uk/strings.xml | 1 - app/src/main/res/values-vi/strings.xml | 1 - app/src/main/res/values-zh-rTW/strings.xml | 1 - app/src/main/res/values/strings.xml | 2 +- 42 files changed, 478 insertions(+), 97 deletions(-) create mode 100644 app/src/main/java/us/shandian/giga/get/DownloadMissionRecover.java create mode 100644 app/src/main/java/us/shandian/giga/get/MissionRecoveryInfo.java diff --git a/app/src/main/java/org/schabi/newpipe/download/DownloadDialog.java b/app/src/main/java/org/schabi/newpipe/download/DownloadDialog.java index 90258a6dc..0006b3c12 100644 --- a/app/src/main/java/org/schabi/newpipe/download/DownloadDialog.java +++ b/app/src/main/java/org/schabi/newpipe/download/DownloadDialog.java @@ -68,6 +68,7 @@ import java.util.Locale; import icepick.Icepick; import icepick.State; import io.reactivex.disposables.CompositeDisposable; +import us.shandian.giga.get.MissionRecoveryInfo; import us.shandian.giga.io.StoredDirectoryHelper; import us.shandian.giga.io.StoredFileHelper; import us.shandian.giga.postprocessing.Postprocessing; @@ -762,12 +763,13 @@ public class DownloadDialog extends DialogFragment implements RadioGroup.OnCheck } Stream selectedStream; + Stream secondaryStream = null; char kind; int threads = threadsSeekBar.getProgress() + 1; String[] urls; + MissionRecoveryInfo[] recoveryInfo; String psName = null; String[] psArgs = null; - String secondaryStreamUrl = null; long nearLength = 0; // more download logic: select muxer, subtitle converter, etc. @@ -786,12 +788,12 @@ public class DownloadDialog extends DialogFragment implements RadioGroup.OnCheck kind = 'v'; selectedStream = videoStreamsAdapter.getItem(selectedVideoIndex); - SecondaryStreamHelper secondaryStream = videoStreamsAdapter + SecondaryStreamHelper secondary = videoStreamsAdapter .getAllSecondary() .get(wrappedVideoStreams.getStreamsList().indexOf(selectedStream)); - if (secondaryStream != null) { - secondaryStreamUrl = secondaryStream.getStream().getUrl(); + if (secondary != null) { + secondaryStream = secondary.getStream(); if (selectedStream.getFormat() == MediaFormat.MPEG_4) psName = Postprocessing.ALGORITHM_MP4_FROM_DASH_MUXER; @@ -803,8 +805,8 @@ public class DownloadDialog extends DialogFragment implements RadioGroup.OnCheck // set nearLength, only, if both sizes are fetched or known. This probably // does not work on slow networks but is later updated in the downloader - if (secondaryStream.getSizeInBytes() > 0 && videoSize > 0) { - nearLength = secondaryStream.getSizeInBytes() + videoSize; + if (secondary.getSizeInBytes() > 0 && videoSize > 0) { + nearLength = secondary.getSizeInBytes() + videoSize; } } break; @@ -826,13 +828,25 @@ public class DownloadDialog extends DialogFragment implements RadioGroup.OnCheck return; } - if (secondaryStreamUrl == null) { - urls = new String[]{selectedStream.getUrl()}; + if (secondaryStream == null) { + urls = new String[]{ + selectedStream.getUrl() + }; + recoveryInfo = new MissionRecoveryInfo[]{ + new MissionRecoveryInfo(selectedStream) + }; } else { - urls = new String[]{selectedStream.getUrl(), secondaryStreamUrl}; + urls = new String[]{ + selectedStream.getUrl(), secondaryStream.getUrl() + }; + recoveryInfo = new MissionRecoveryInfo[]{ + new MissionRecoveryInfo(selectedStream), new MissionRecoveryInfo(secondaryStream) + }; } - DownloadManagerService.startMission(context, urls, storage, kind, threads, currentInfo.getUrl(), psName, psArgs, nearLength); + DownloadManagerService.startMission( + context, urls, storage, kind, threads, currentInfo.getUrl(), psName, psArgs, nearLength, recoveryInfo + ); dismiss(); } diff --git a/app/src/main/java/us/shandian/giga/get/DownloadInitializer.java b/app/src/main/java/us/shandian/giga/get/DownloadInitializer.java index 247faeb6d..593feafa7 100644 --- a/app/src/main/java/us/shandian/giga/get/DownloadInitializer.java +++ b/app/src/main/java/us/shandian/giga/get/DownloadInitializer.java @@ -1,6 +1,7 @@ package us.shandian.giga.get; import androidx.annotation.NonNull; +import android.text.TextUtils; import android.util.Log; import org.schabi.newpipe.streams.io.SharpStream; @@ -151,6 +152,20 @@ public class DownloadInitializer extends Thread { if (!mMission.running || Thread.interrupted()) return; + if (!mMission.unknownLength && mMission.recoveryInfo != null) { + String entityTag = mConn.getHeaderField("ETAG"); + String lastModified = mConn.getHeaderField("Last-Modified"); + MissionRecoveryInfo recovery = mMission.recoveryInfo[mMission.current]; + + if (!TextUtils.isEmpty(entityTag)) { + recovery.validateCondition = entityTag; + } else if (!TextUtils.isEmpty(lastModified)) { + recovery.validateCondition = lastModified;// Note: this is less precise + } else { + recovery.validateCondition = null; + } + } + mMission.running = false; break; } catch (InterruptedIOException | ClosedByInterruptException e) { diff --git a/app/src/main/java/us/shandian/giga/get/DownloadMission.java b/app/src/main/java/us/shandian/giga/get/DownloadMission.java index d78f8e32b..77b417118 100644 --- a/app/src/main/java/us/shandian/giga/get/DownloadMission.java +++ b/app/src/main/java/us/shandian/giga/get/DownloadMission.java @@ -27,7 +27,7 @@ import us.shandian.giga.util.Utility; import static org.schabi.newpipe.BuildConfig.DEBUG; public class DownloadMission extends Mission { - private static final long serialVersionUID = 5L;// last bump: 30 june 2019 + private static final long serialVersionUID = 6L;// last bump: 28 september 2019 static final int BUFFER_SIZE = 64 * 1024; static final int BLOCK_SIZE = 512 * 1024; @@ -51,8 +51,9 @@ public class DownloadMission extends Mission { public static final int ERROR_INSUFFICIENT_STORAGE = 1010; public static final int ERROR_PROGRESS_LOST = 1011; public static final int ERROR_TIMEOUT = 1012; + public static final int ERROR_RESOURCE_GONE = 1013; public static final int ERROR_HTTP_NO_CONTENT = 204; - public static final int ERROR_HTTP_UNSUPPORTED_RANGE = 206; + static final int ERROR_HTTP_FORBIDDEN = 403; /** * The urls of the file to download @@ -125,6 +126,11 @@ public class DownloadMission extends Mission { */ public int threadCount = 3; + /** + * information required to recover a download + */ + public MissionRecoveryInfo[] recoveryInfo; + private transient int finishCount; public transient boolean running; public boolean enqueued; @@ -132,7 +138,6 @@ public class DownloadMission extends Mission { public int errCode = ERROR_NOTHING; public Exception errObject = null; - public transient boolean recovered; public transient Handler mHandler; private transient boolean mWritingToFile; private transient boolean[] blockAcquired; @@ -197,9 +202,9 @@ public class DownloadMission extends Mission { } /** - * Open connection + * Opens a connection * - * @param threadId id of the calling thread, used only for debug + * @param threadId id of the calling thread, used only for debugging * @param rangeStart range start * @param rangeEnd range end * @return a {@link java.net.URLConnection URLConnection} linking to the URL. @@ -251,7 +256,7 @@ public class DownloadMission extends Mission { case 204: case 205: case 207: - throw new HttpError(conn.getResponseCode()); + throw new HttpError(statusCode); case 416: return;// let the download thread handle this error default: @@ -270,10 +275,6 @@ public class DownloadMission extends Mission { synchronized void notifyProgress(long deltaLen) { if (!running) return; - if (recovered) { - recovered = false; - } - if (unknownLength) { length += deltaLen;// Update length before proceeding } @@ -344,7 +345,6 @@ public class DownloadMission extends Mission { if (running) { running = false; - recovered = true; if (threads != null) selfPause(); } } @@ -409,12 +409,13 @@ public class DownloadMission extends Mission { * Start downloading with multiple threads. */ public void start() { - if (running || isFinished()) return; + if (running || isFinished() || urls.length < 1) return; // ensure that the previous state is completely paused. - joinForThread(init); + int maxWait = 10000;// 10 seconds + joinForThread(init, maxWait); if (threads != null) { - for (Thread thread : threads) joinForThread(thread); + for (Thread thread : threads) joinForThread(thread, maxWait); threads = null; } @@ -431,6 +432,11 @@ public class DownloadMission extends Mission { return; } + if (urls[current] == null) { + doRecover(null); + return; + } + if (blocks == null) { initializer(); return; @@ -478,7 +484,6 @@ public class DownloadMission extends Mission { } running = false; - recovered = true; if (init != null && init.isAlive()) { // NOTE: if start() method is running ¡will no have effect! @@ -563,7 +568,7 @@ public class DownloadMission extends Mission { * Write this {@link DownloadMission} to the meta file asynchronously * if no thread is already running. */ - private void writeThisToFile() { + void writeThisToFile() { synchronized (LOCK) { if (deleted) return; Utility.writeToFile(metadata, DownloadMission.this); @@ -667,6 +672,7 @@ public class DownloadMission extends Mission { * @return {@code true} is this mission its "healthy", otherwise, {@code false} */ public boolean isCorrupt() { + if (urls.length < 1) return false; return (isPsFailed() || errCode == ERROR_POSTPROCESSING_HOLD) || isFinished(); } @@ -710,6 +716,48 @@ public class DownloadMission extends Mission { return true; } + /** + * Attempts to recover the download + * + * @param fromError exception which require update the url from the source + */ + void doRecover(Exception fromError) { + Log.i(TAG, "Attempting to recover the mission: " + storage.getName()); + + if (recoveryInfo == null) { + if (fromError == null) + notifyError(ERROR_RESOURCE_GONE, null); + else + notifyError(fromError); + + urls = new String[0];// mark this mission as dead + return; + } + + if (threads != null) { + for (Thread thread : threads) { + if (thread == Thread.currentThread()) continue; + thread.interrupt(); + joinForThread(thread, 0); + } + } + + // set the current download url to null in case if the recovery + // process is canceled. Next time start() method is called the + // recovery will be executed, saving time + urls[current] = null; + + if (recoveryInfo[current].attempts >= maxRetry) { + recoveryInfo[current].attempts = 0; + notifyError(fromError); + return; + } + + threads = new Thread[]{ + runAsync(DownloadMissionRecover.mID, new DownloadMissionRecover(this, fromError)) + }; + } + private boolean deleteThisFromFile() { synchronized (LOCK) { return metadata.delete(); @@ -749,7 +797,13 @@ public class DownloadMission extends Mission { return who; } - private void joinForThread(Thread thread) { + /** + * Waits at most {@code millis} milliseconds for the thread to die + * + * @param thread the desired thread + * @param millis the time to wait in milliseconds + */ + private void joinForThread(Thread thread, int millis) { if (thread == null || !thread.isAlive()) return; if (thread == Thread.currentThread()) return; @@ -764,7 +818,7 @@ public class DownloadMission extends Mission { // start() method called quickly after pause() try { - thread.join(10000); + thread.join(millis); } catch (InterruptedException e) { Log.d(TAG, "timeout on join : " + thread.getName()); throw new RuntimeException("A thread is still running:\n" + thread.getName()); @@ -785,9 +839,9 @@ public class DownloadMission extends Mission { } } - static class Block { - int position; - int done; + public static class Block { + public int position; + public int done; } private static class Lock implements Serializable { diff --git a/app/src/main/java/us/shandian/giga/get/DownloadMissionRecover.java b/app/src/main/java/us/shandian/giga/get/DownloadMissionRecover.java new file mode 100644 index 000000000..9abd93717 --- /dev/null +++ b/app/src/main/java/us/shandian/giga/get/DownloadMissionRecover.java @@ -0,0 +1,222 @@ +package us.shandian.giga.get; + +import android.util.Log; + +import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.StreamingService; +import org.schabi.newpipe.extractor.stream.AudioStream; +import org.schabi.newpipe.extractor.stream.StreamExtractor; +import org.schabi.newpipe.extractor.stream.SubtitlesStream; +import org.schabi.newpipe.extractor.stream.VideoStream; + +import java.io.IOException; +import java.net.HttpURLConnection; +import java.nio.channels.ClosedByInterruptException; +import java.util.List; + +import static us.shandian.giga.get.DownloadMission.ERROR_RESOURCE_GONE; + +public class DownloadMissionRecover extends Thread { + private static final String TAG = "DownloadMissionRecover"; + static final int mID = -3; + + private final DownloadMission mMission; + private final MissionRecoveryInfo mRecovery; + private final Exception mFromError; + private HttpURLConnection mConn; + + DownloadMissionRecover(DownloadMission mission, Exception originError) { + mMission = mission; + mFromError = originError; + mRecovery = mission.recoveryInfo[mission.current]; + } + + @Override + public void run() { + if (mMission.source == null) { + mMission.notifyError(mFromError); + return; + } + + try { + /*if (mMission.source.startsWith(MissionRecoveryInfo.DIRECT_SOURCE)) { + resolve(mMission.source.substring(MissionRecoveryInfo.DIRECT_SOURCE.length())); + return; + }*/ + + StreamingService svr = NewPipe.getServiceByUrl(mMission.source); + + if (svr == null) { + throw new RuntimeException("Unknown source service"); + } + + StreamExtractor extractor = svr.getStreamExtractor(mMission.source); + extractor.fetchPage(); + + if (!mMission.running || super.isInterrupted()) return; + + String url = null; + + switch (mMission.kind) { + case 'a': + for (AudioStream audio : extractor.getAudioStreams()) { + if (audio.average_bitrate == mRecovery.desiredBitrate && audio.getFormat() == mRecovery.format) { + url = audio.getUrl(); + break; + } + } + break; + case 'v': + List videoStreams; + if (mRecovery.desired2) + videoStreams = extractor.getVideoOnlyStreams(); + else + videoStreams = extractor.getVideoStreams(); + for (VideoStream video : videoStreams) { + if (video.resolution.equals(mRecovery.desired) && video.getFormat() == mRecovery.format) { + url = video.getUrl(); + break; + } + } + break; + case 's': + for (SubtitlesStream subtitles : extractor.getSubtitles(mRecovery.format)) { + String tag = subtitles.getLanguageTag(); + if (tag.equals(mRecovery.desired) && subtitles.isAutoGenerated() == mRecovery.desired2) { + url = subtitles.getURL(); + break; + } + } + break; + default: + throw new RuntimeException("Unknown stream type"); + } + + resolve(url); + } catch (Exception e) { + if (!mMission.running || e instanceof ClosedByInterruptException) return; + mRecovery.attempts++; + mMission.notifyError(e); + } + } + + private void resolve(String url) throws IOException, DownloadMission.HttpError { + if (mRecovery.validateCondition == null) { + Log.w(TAG, "validation condition not defined, the resource can be stale"); + } + + if (mMission.unknownLength || mRecovery.validateCondition == null) { + recover(url, false); + return; + } + + /////////////////////////////////////////////////////////////////////// + ////// Validate the http resource doing a range request + ///////////////////// + try { + mConn = mMission.openConnection(url, mID, mMission.length - 10, mMission.length); + mConn.setRequestProperty("If-Range", mRecovery.validateCondition); + mMission.establishConnection(mID, mConn); + + int code = mConn.getResponseCode(); + + switch (code) { + case 200: + case 413: + // stale + recover(url, true); + return; + case 206: + // in case of validation using the Last-Modified date, check the resource length + long[] contentRange = parseContentRange(mConn.getHeaderField("Content-Range")); + boolean lengthMismatch = contentRange[2] != -1 && contentRange[2] != mMission.length; + + recover(url, lengthMismatch); + return; + } + + throw new DownloadMission.HttpError(code); + } catch (Exception e) { + if (!mMission.running || e instanceof ClosedByInterruptException) return; + throw e; + } finally { + this.interrupt(); + } + } + + private void recover(String url, boolean stale) { + Log.i(TAG, + String.format("download recovered name=%s isStale=%s url=%s", mMission.storage.getName(), stale, url) + ); + + if (url == null) { + mMission.notifyError(ERROR_RESOURCE_GONE, null); + return; + } + + mMission.urls[mMission.current] = url; + mRecovery.attempts = 0; + + if (stale) { + mMission.resetState(false, false, DownloadMission.ERROR_NOTHING); + } + + mMission.writeThisToFile(); + + if (!mMission.running || super.isInterrupted()) return; + + mMission.running = false; + mMission.start(); + } + + private long[] parseContentRange(String value) { + long[] range = new long[3]; + + if (value == null) { + // this never should happen + return range; + } + + try { + value = value.trim(); + + if (!value.startsWith("bytes")) { + return range;// unknown range type + } + + int space = value.lastIndexOf(' ') + 1; + int dash = value.indexOf('-', space) + 1; + int bar = value.indexOf('/', dash); + + // start + range[0] = Long.parseLong(value.substring(space, dash - 1)); + + // end + range[1] = Long.parseLong(value.substring(dash, bar)); + + // resource length + value = value.substring(bar + 1); + if (value.equals("*")) { + range[2] = -1;// unknown length received from the server but should be valid + } else { + range[2] = Long.parseLong(value); + } + } catch (Exception e) { + // nothing to do + } + + return range; + } + + @Override + public void interrupt() { + super.interrupt(); + if (mConn != null) { + try { + mConn.disconnect(); + } catch (Exception e) { + // nothing to do + } + } + } +} diff --git a/app/src/main/java/us/shandian/giga/get/DownloadRunnable.java b/app/src/main/java/us/shandian/giga/get/DownloadRunnable.java index f5b9b06d4..1d2a4eee7 100644 --- a/app/src/main/java/us/shandian/giga/get/DownloadRunnable.java +++ b/app/src/main/java/us/shandian/giga/get/DownloadRunnable.java @@ -10,8 +10,10 @@ import java.net.HttpURLConnection; import java.nio.channels.ClosedByInterruptException; import us.shandian.giga.get.DownloadMission.Block; +import us.shandian.giga.get.DownloadMission.HttpError; import static org.schabi.newpipe.BuildConfig.DEBUG; +import static us.shandian.giga.get.DownloadMission.ERROR_HTTP_FORBIDDEN; /** @@ -19,7 +21,7 @@ import static org.schabi.newpipe.BuildConfig.DEBUG; * an error occurs or the process is stopped. */ public class DownloadRunnable extends Thread { - private static final String TAG = DownloadRunnable.class.getSimpleName(); + private static final String TAG = "DownloadRunnable"; private final DownloadMission mMission; private final int mId; @@ -41,13 +43,7 @@ public class DownloadRunnable extends Thread { public void run() { boolean retry = false; Block block = null; - int retryCount = 0; - - if (DEBUG) { - Log.d(TAG, mId + ":recovered: " + mMission.recovered); - } - SharpStream f; try { @@ -133,6 +129,17 @@ public class DownloadRunnable extends Thread { } catch (Exception e) { if (!mMission.running || e instanceof ClosedByInterruptException) break; + if (e instanceof HttpError && ((HttpError) e).statusCode == ERROR_HTTP_FORBIDDEN) { + // for youtube streams. The url has expired, recover + f.close(); + + if (mId == 1) { + // only the first thread will execute the recovery procedure + mMission.doRecover(e); + } + return; + } + if (retryCount++ >= mMission.maxRetry) { mMission.notifyError(e); break; @@ -144,11 +151,7 @@ public class DownloadRunnable extends Thread { } } - try { - f.close(); - } catch (Exception err) { - // ¿ejected media storage? ¿file deleted? ¿storage ran out of space? - } + f.close(); if (DEBUG) { Log.d(TAG, "thread " + mId + " exited from main download loop"); diff --git a/app/src/main/java/us/shandian/giga/get/DownloadRunnableFallback.java b/app/src/main/java/us/shandian/giga/get/DownloadRunnableFallback.java index 7fb1f0c77..b5937c577 100644 --- a/app/src/main/java/us/shandian/giga/get/DownloadRunnableFallback.java +++ b/app/src/main/java/us/shandian/giga/get/DownloadRunnableFallback.java @@ -10,9 +10,11 @@ import java.io.InputStream; import java.net.HttpURLConnection; import java.nio.channels.ClosedByInterruptException; +import us.shandian.giga.get.DownloadMission.HttpError; import us.shandian.giga.util.Utility; import static org.schabi.newpipe.BuildConfig.DEBUG; +import static us.shandian.giga.get.DownloadMission.ERROR_HTTP_FORBIDDEN; /** * Single-threaded fallback mode @@ -85,7 +87,7 @@ public class DownloadRunnableFallback extends Thread { mIs = mConn.getInputStream(); - byte[] buf = new byte[64 * 1024]; + byte[] buf = new byte[DownloadMission.BUFFER_SIZE]; int len = 0; while (mMission.running && (len = mIs.read(buf, 0, buf.length)) != -1) { @@ -103,6 +105,13 @@ public class DownloadRunnableFallback extends Thread { if (!mMission.running || e instanceof ClosedByInterruptException) return; + if (e instanceof HttpError && ((HttpError) e).statusCode == ERROR_HTTP_FORBIDDEN) { + // for youtube streams. The url has expired, recover + mMission.doRecover(e); + dispose(); + return; + } + if (mRetryCount++ >= mMission.maxRetry) { mMission.notifyError(e); return; diff --git a/app/src/main/java/us/shandian/giga/get/MissionRecoveryInfo.java b/app/src/main/java/us/shandian/giga/get/MissionRecoveryInfo.java new file mode 100644 index 000000000..553ba6d89 --- /dev/null +++ b/app/src/main/java/us/shandian/giga/get/MissionRecoveryInfo.java @@ -0,0 +1,79 @@ +package us.shandian.giga.get; + +import android.os.Parcel; +import android.os.Parcelable; +import android.support.annotation.NonNull; + +import org.schabi.newpipe.extractor.MediaFormat; +import org.schabi.newpipe.extractor.stream.AudioStream; +import org.schabi.newpipe.extractor.stream.Stream; +import org.schabi.newpipe.extractor.stream.SubtitlesStream; +import org.schabi.newpipe.extractor.stream.VideoStream; + +import java.io.Serializable; + +public class MissionRecoveryInfo implements Serializable, Parcelable { + private static final long serialVersionUID = 0L; + //public static final String DIRECT_SOURCE = "direct-source://"; + + public MediaFormat format; + String desired; + boolean desired2; + int desiredBitrate; + + transient int attempts = 0; + + String validateCondition = null; + + public MissionRecoveryInfo(@NonNull Stream stream) { + if (stream instanceof AudioStream) { + desiredBitrate = ((AudioStream) stream).average_bitrate; + desired2 = false; + } else if (stream instanceof VideoStream) { + desired = ((VideoStream) stream).getResolution(); + desired2 = ((VideoStream) stream).isVideoOnly(); + } else if (stream instanceof SubtitlesStream) { + desired = ((SubtitlesStream) stream).getLanguageTag(); + desired2 = ((SubtitlesStream) stream).isAutoGenerated(); + } else { + throw new RuntimeException("Unknown stream kind"); + } + + format = stream.getFormat(); + if (format == null) throw new NullPointerException("Stream format cannot be null"); + } + + @Override + public int describeContents() { + return 0; + } + + @Override + public void writeToParcel(Parcel parcel, int flags) { + parcel.writeInt(this.format.ordinal()); + parcel.writeString(this.desired); + parcel.writeInt(this.desired2 ? 0x01 : 0x00); + parcel.writeInt(this.desiredBitrate); + parcel.writeString(this.validateCondition); + } + + private MissionRecoveryInfo(Parcel parcel) { + this.format = MediaFormat.values()[parcel.readInt()]; + this.desired = parcel.readString(); + this.desired2 = parcel.readInt() != 0x00; + this.desiredBitrate = parcel.readInt(); + this.validateCondition = parcel.readString(); + } + + public static final Parcelable.Creator CREATOR = new Parcelable.Creator() { + @Override + public MissionRecoveryInfo createFromParcel(Parcel source) { + return new MissionRecoveryInfo(source); + } + + @Override + public MissionRecoveryInfo[] newArray(int size) { + return new MissionRecoveryInfo[size]; + } + }; +} diff --git a/app/src/main/java/us/shandian/giga/service/DownloadManager.java b/app/src/main/java/us/shandian/giga/service/DownloadManager.java index 3d34411b9..a859a87ca 100644 --- a/app/src/main/java/us/shandian/giga/service/DownloadManager.java +++ b/app/src/main/java/us/shandian/giga/service/DownloadManager.java @@ -177,7 +177,6 @@ public class DownloadManager { mis.psAlgorithm.setTemporalDir(pickAvailableTemporalDir(ctx)); } - mis.recovered = exists; mis.metadata = sub; mis.maxRetry = mPrefMaxRetry; mis.mHandler = mHandler; diff --git a/app/src/main/java/us/shandian/giga/service/DownloadManagerService.java b/app/src/main/java/us/shandian/giga/service/DownloadManagerService.java index 461787b62..ea9029c0b 100755 --- a/app/src/main/java/us/shandian/giga/service/DownloadManagerService.java +++ b/app/src/main/java/us/shandian/giga/service/DownloadManagerService.java @@ -23,6 +23,7 @@ import android.os.Handler; import android.os.Handler.Callback; import android.os.IBinder; import android.os.Message; +import android.os.Parcelable; import android.preference.PreferenceManager; import androidx.annotation.NonNull; import androidx.annotation.Nullable; @@ -40,8 +41,11 @@ import org.schabi.newpipe.player.helper.LockManager; import java.io.File; import java.io.IOException; import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; import us.shandian.giga.get.DownloadMission; +import us.shandian.giga.get.MissionRecoveryInfo; import us.shandian.giga.io.StoredDirectoryHelper; import us.shandian.giga.io.StoredFileHelper; import us.shandian.giga.postprocessing.Postprocessing; @@ -73,6 +77,7 @@ public class DownloadManagerService extends Service { private static final String EXTRA_PATH = "DownloadManagerService.extra.storagePath"; private static final String EXTRA_PARENT_PATH = "DownloadManagerService.extra.storageParentPath"; private static final String EXTRA_STORAGE_TAG = "DownloadManagerService.extra.storageTag"; + private static final String EXTRA_RECOVERY_INFO = "DownloadManagerService.extra.recoveryInfo"; private static final String ACTION_RESET_DOWNLOAD_FINISHED = APPLICATION_ID + ".reset_download_finished"; private static final String ACTION_OPEN_DOWNLOADS_FINISHED = APPLICATION_ID + ".open_downloads_finished"; @@ -364,18 +369,20 @@ public class DownloadManagerService extends Service { /** * Start a new download mission * - * @param context the activity context - * @param urls the list of urls to download - * @param storage where the file is saved - * @param kind type of file (a: audio v: video s: subtitle ?: file-extension defined) - * @param threads the number of threads maximal used to download chunks of the file. - * @param psName the name of the required post-processing algorithm, or {@code null} to ignore. - * @param source source url of the resource - * @param psArgs the arguments for the post-processing algorithm. - * @param nearLength the approximated final length of the file + * @param context the activity context + * @param urls array of urls to download + * @param storage where the file is saved + * @param kind type of file (a: audio v: video s: subtitle ?: file-extension defined) + * @param threads the number of threads maximal used to download chunks of the file. + * @param psName the name of the required post-processing algorithm, or {@code null} to ignore. + * @param source source url of the resource + * @param psArgs the arguments for the post-processing algorithm. + * @param nearLength the approximated final length of the file + * @param recoveryInfo array of MissionRecoveryInfo, in case is required recover the download */ - public static void startMission(Context context, String[] urls, StoredFileHelper storage, char kind, - int threads, String source, String psName, String[] psArgs, long nearLength) { + public static void startMission(Context context, String[] urls, StoredFileHelper storage, + char kind, int threads, String source, String psName, + String[] psArgs, long nearLength, MissionRecoveryInfo[] recoveryInfo) { Intent intent = new Intent(context, DownloadManagerService.class); intent.setAction(Intent.ACTION_RUN); intent.putExtra(EXTRA_URLS, urls); @@ -385,6 +392,7 @@ public class DownloadManagerService extends Service { intent.putExtra(EXTRA_POSTPROCESSING_NAME, psName); intent.putExtra(EXTRA_POSTPROCESSING_ARGS, psArgs); intent.putExtra(EXTRA_NEAR_LENGTH, nearLength); + intent.putExtra(EXTRA_RECOVERY_INFO, recoveryInfo); intent.putExtra(EXTRA_PARENT_PATH, storage.getParentUri()); intent.putExtra(EXTRA_PATH, storage.getUri()); @@ -404,6 +412,7 @@ public class DownloadManagerService extends Service { String source = intent.getStringExtra(EXTRA_SOURCE); long nearLength = intent.getLongExtra(EXTRA_NEAR_LENGTH, 0); String tag = intent.getStringExtra(EXTRA_STORAGE_TAG); + Parcelable[] parcelRecovery = intent.getParcelableArrayExtra(EXTRA_RECOVERY_INFO); StoredFileHelper storage; try { @@ -418,10 +427,15 @@ public class DownloadManagerService extends Service { else ps = Postprocessing.getAlgorithm(psName, psArgs); + MissionRecoveryInfo[] recovery = new MissionRecoveryInfo[parcelRecovery.length]; + for (int i = 0; i < parcelRecovery.length; i++) + recovery[i] = (MissionRecoveryInfo) parcelRecovery[i]; + final DownloadMission mission = new DownloadMission(urls, storage, kind, ps); mission.threadCount = threads; mission.source = source; mission.nearLength = nearLength; + mission.recoveryInfo = recovery; if (ps != null) ps.setTemporalDir(DownloadManager.pickAvailableTemporalDir(this)); diff --git a/app/src/main/java/us/shandian/giga/ui/adapter/MissionAdapter.java b/app/src/main/java/us/shandian/giga/ui/adapter/MissionAdapter.java index 6d1169031..6c6198750 100644 --- a/app/src/main/java/us/shandian/giga/ui/adapter/MissionAdapter.java +++ b/app/src/main/java/us/shandian/giga/ui/adapter/MissionAdapter.java @@ -62,7 +62,6 @@ import static android.content.Intent.FLAG_GRANT_READ_URI_PERMISSION; import static us.shandian.giga.get.DownloadMission.ERROR_CONNECT_HOST; import static us.shandian.giga.get.DownloadMission.ERROR_FILE_CREATION; import static us.shandian.giga.get.DownloadMission.ERROR_HTTP_NO_CONTENT; -import static us.shandian.giga.get.DownloadMission.ERROR_HTTP_UNSUPPORTED_RANGE; import static us.shandian.giga.get.DownloadMission.ERROR_INSUFFICIENT_STORAGE; import static us.shandian.giga.get.DownloadMission.ERROR_NOTHING; import static us.shandian.giga.get.DownloadMission.ERROR_PATH_CREATION; @@ -71,6 +70,7 @@ import static us.shandian.giga.get.DownloadMission.ERROR_POSTPROCESSING; import static us.shandian.giga.get.DownloadMission.ERROR_POSTPROCESSING_HOLD; import static us.shandian.giga.get.DownloadMission.ERROR_POSTPROCESSING_STOPPED; import static us.shandian.giga.get.DownloadMission.ERROR_PROGRESS_LOST; +import static us.shandian.giga.get.DownloadMission.ERROR_RESOURCE_GONE; import static us.shandian.giga.get.DownloadMission.ERROR_SSL_EXCEPTION; import static us.shandian.giga.get.DownloadMission.ERROR_TIMEOUT; import static us.shandian.giga.get.DownloadMission.ERROR_UNKNOWN_EXCEPTION; @@ -430,7 +430,7 @@ public class MissionAdapter extends Adapter implements Handler.Callb switch (mission.errCode) { case 416: - msg = R.string.error_http_requested_range_not_satisfiable; + msg = R.string.error_http_unsupported_range; break; case 404: msg = R.string.error_http_not_found; @@ -443,9 +443,6 @@ public class MissionAdapter extends Adapter implements Handler.Callb case ERROR_HTTP_NO_CONTENT: msg = R.string.error_http_no_content; break; - case ERROR_HTTP_UNSUPPORTED_RANGE: - msg = R.string.error_http_unsupported_range; - break; case ERROR_PATH_CREATION: msg = R.string.error_path_creation; break; @@ -480,6 +477,9 @@ public class MissionAdapter extends Adapter implements Handler.Callb case ERROR_TIMEOUT: msg = R.string.error_timeout; break; + case ERROR_RESOURCE_GONE: + msg = R.string.error_download_resource_gone; + break; default: if (mission.errCode >= 100 && mission.errCode < 600) { msgEx = "HTTP " + mission.errCode; @@ -859,7 +859,7 @@ public class MissionAdapter extends Adapter implements Handler.Callb delete.setVisible(true); - boolean flag = !mission.isPsFailed(); + boolean flag = !mission.isPsFailed() && mission.urls.length > 0; start.setVisible(flag); queue.setVisible(flag); } diff --git a/app/src/main/res/values-ar/strings.xml b/app/src/main/res/values-ar/strings.xml index 7156d08ba..43b45d15e 100644 --- a/app/src/main/res/values-ar/strings.xml +++ b/app/src/main/res/values-ar/strings.xml @@ -468,7 +468,6 @@ لا يمكن الاتصال بالخادم الخادم لايقوم بإرسال البيانات الخادم لا يقبل التنزيل المتعدد، إعادة المحاولة مع @string/msg_threads = 1 - عدم استيفاء النطاق المطلوب غير موجود فشلت المعالجة الاولية حذف التنزيلات المنتهية diff --git a/app/src/main/res/values-be/strings.xml b/app/src/main/res/values-be/strings.xml index 93307cbcf..3c79a96d3 100644 --- a/app/src/main/res/values-be/strings.xml +++ b/app/src/main/res/values-be/strings.xml @@ -455,7 +455,6 @@ Немагчыма злучыцца з серверам Не атрымалася атрымаць дадзеныя з сервера Сервер не падтрымлівае шматструменную загрузку, паспрабуйце з @string/msg_threads = 1 - Запытаны дыяпазон недапушчальны Не знойдзена Пасляапрацоўка не ўдалася Ачысціць завершаныя diff --git a/app/src/main/res/values-cmn/strings.xml b/app/src/main/res/values-cmn/strings.xml index 49801a190..bcb145c16 100644 --- a/app/src/main/res/values-cmn/strings.xml +++ b/app/src/main/res/values-cmn/strings.xml @@ -460,7 +460,6 @@ NewPipe 更新可用! 无法创建目标文件夹 服务器不接受多线程下载, 请使用 @string/msg_threads = 1重试 - 请求范围无法满足 继续进行%s个待下载转移 切换至移动数据时有用，尽管一些下载无法被暂停 显示评论 diff --git a/app/src/main/res/values-cs/strings.xml b/app/src/main/res/values-cs/strings.xml index d539923fe..b741e0d16 100644 --- a/app/src/main/res/values-cs/strings.xml +++ b/app/src/main/res/values-cs/strings.xml @@ -463,7 +463,6 @@ otevření ve vyskakovacím okně Nelze se připojit k serveru Server neposílá data Server neakceptuje vícevláknové stahování, opakujte akci s @string/msg_threads = 1 - Požadovaný rozsah nelze splnit Nenalezeno Post-processing selhal Vyčistit dokončená stahování diff --git a/app/src/main/res/values-da/strings.xml b/app/src/main/res/values-da/strings.xml index 42ffd474b..199c2f85d 100644 --- a/app/src/main/res/values-da/strings.xml +++ b/app/src/main/res/values-da/strings.xml @@ -380,7 +380,6 @@ Kan ikke forbinde til serveren Serveren sender ikke data Serveren accepterer ikke multitrådede downloads; prøv igen med @string/msg_threads = 1 - Det anmodede interval er ikke gyldigt Ikke fundet Efterbehandling fejlede Stop diff --git a/app/src/main/res/values-de/strings.xml b/app/src/main/res/values-de/strings.xml index 2d6b5b6d2..3279e919c 100644 --- a/app/src/main/res/values-de/strings.xml +++ b/app/src/main/res/values-de/strings.xml @@ -454,7 +454,6 @@ Kann nicht mit dem Server verbinden Der Server sendet keine Daten Der Server erlaubt kein mehrfädiges Herunterladen – wiederhole mit @string/msg_threads = 1 - Gewünschter Bereich ist nicht verfügbar Nicht gefunden Nachbearbeitung fehlgeschlagen Um fertige Downloads bereinigen diff --git a/app/src/main/res/values-el/strings.xml b/app/src/main/res/values-el/strings.xml index 4f3499cfd..372cbb1a2 100644 --- a/app/src/main/res/values-el/strings.xml +++ b/app/src/main/res/values-el/strings.xml @@ -456,7 +456,6 @@ Αδυναμία σύνδεσης με τον εξυπηρετητή Ο εξυπηρετητής δεν μπορεί να στείλει τα δεδομένα Ο εξυπηρετητής δέν υποστηρίζει πολυνηματικές λήψεις, ξαναπροσπαθήστε με @string/msg_threads = 1 - Το ζητούμενο εύρος δεν μπορεί να εξυπηρετηθεί Δεν βρέθηκε Μετεπεξεργασία απέτυχε Εκκαθάριση ολοκληρωμένων λήψεων diff --git a/app/src/main/res/values-es/strings.xml b/app/src/main/res/values-es/strings.xml index 3aa0bac66..2f69e62cb 100644 --- a/app/src/main/res/values-es/strings.xml +++ b/app/src/main/res/values-es/strings.xml @@ -351,8 +351,8 @@ \n3. Inicie sesión cuando se le pida \n4. Copie la URL del perfil a la que fue redireccionado. suID, soundcloud.com/suID - Observe que esta operación puede causar un uso intensivo de la red. -\n + Observe que esta operación puede causar un uso intensivo de la red. +\n \n¿Quiere continuar\? Cargar miniaturas Desactívela para evitar la carga de miniaturas y ahorrar datos y memoria. Se vaciará la antememoria de imágenes en la memoria volátil y en el disco. @@ -444,8 +444,8 @@ Fallo la conexión segura No se pudo encontrar el servidor No se puede conectar con el servidor - El servidor no está enviando datos - El servidor no acepta descargas multiproceso; intente de nuevo con @string/msg_threads = 1 + El servidor no devolvio datos + El servidor no acepta descargas multi-hilos, intente de nuevo con @string/msg_threads = 1 No se puede satisfacer el intervalo seleccionado No encontrado Falló el posprocesamiento @@ -453,6 +453,7 @@ No hay suficiente espacio disponible en el dispositivo Se perdió el progreso porque el archivo fue eliminado Tiempo de espera excedido + El recurso solicitado ya no esta disponible Preguntar dónde descargar Se preguntará dónde guardar cada descarga Se le preguntará dónde guardar cada descarga. diff --git a/app/src/main/res/values-et/strings.xml b/app/src/main/res/values-et/strings.xml index baad94b5d..4dfcc3d0e 100644 --- a/app/src/main/res/values-et/strings.xml +++ b/app/src/main/res/values-et/strings.xml @@ -457,7 +457,6 @@ Serveriga ei saadud ühendust Server ei saada andmeid Server ei toeta mitmelõimelisi allalaadimisi. Proovi uuesti kasutades @string/msg_threads = 1 - Taotletud vahemik ei ole rahuldatav Ei leitud Järeltöötlemine nurjus Eemalda lõpetatud allalaadimised diff --git a/app/src/main/res/values-eu/strings.xml b/app/src/main/res/values-eu/strings.xml index 7da39393e..7b636d383 100644 --- a/app/src/main/res/values-eu/strings.xml +++ b/app/src/main/res/values-eu/strings.xml @@ -456,7 +456,6 @@ Ezin da zerbitzariarekin konektatu Zerbitzariak ez du daturik bidaltzen Zerbitzariak ez ditu hainbat hariko deskargak onartzen, saiatu @string/msg_threads = 1 erabilita - Eskatutako barrutia ezin da bete Ez aurkitua Post-prozesuak huts egin du Garbitu amaitutako deskargak diff --git a/app/src/main/res/values-fr/strings.xml b/app/src/main/res/values-fr/strings.xml index 147502088..b4388e39f 100644 --- a/app/src/main/res/values-fr/strings.xml +++ b/app/src/main/res/values-fr/strings.xml @@ -467,7 +467,6 @@ Utilisation des onglets par défaut, erreur lors de la lecture des onglets enregistrés Le serveur n’accepte pas les téléchargements multi-fils, veuillez réessayer avec @string/msg_threads = 1 Continuer vos %s transferts en attente depuis Téléchargement - Le domaine désiré n\'est pas disponible Afficher les commentaires Désactiver pour ne pas afficher les commentaires Lecture automatique diff --git a/app/src/main/res/values-he/strings.xml b/app/src/main/res/values-he/strings.xml index b5a0778d4..5e340d8b3 100644 --- a/app/src/main/res/values-he/strings.xml +++ b/app/src/main/res/values-he/strings.xml @@ -461,7 +461,6 @@ לא ניתן להתחבר לשרת השרת לא שולח נתונים "השרת לא מקבל הורדות רב ערוציות, מוטב לנסות שוב עם ‎@string/msg_threads = 1 " - הטווח המבוקש לא מתאים לא נמצא העיבוד המאוחר נכשל פינוי ההורדות שהסתיימו diff --git a/app/src/main/res/values-hr/strings.xml b/app/src/main/res/values-hr/strings.xml index e85d5810e..aa4ff9113 100644 --- a/app/src/main/res/values-hr/strings.xml +++ b/app/src/main/res/values-hr/strings.xml @@ -454,7 +454,6 @@ Nije moguće povezati se s serverom Server ne šalje podatke Poslužitelj ne prihvaća preuzimanja s više niti, pokušaj ponovo s @string/msg_threads = 1 - Traženi raspon nije zadovoljavajući Nije pronađeno Naknadna obrada nije uspjela Obriši završena preuzimanja diff --git a/app/src/main/res/values-id/strings.xml b/app/src/main/res/values-id/strings.xml index db738d749..d52f5fafa 100644 --- a/app/src/main/res/values-id/strings.xml +++ b/app/src/main/res/values-id/strings.xml @@ -450,7 +450,6 @@ Tidak dapat terhubung ke server Server tidak mengirim data Server tidak menerima unduhan multi-utas, coba lagi dengan @string/msg_threads = 1 - Rentang yang diminta tidak memuaskan Tidak ditemukan Pengolahan-pasca gagal Hapus unduhan yang sudah selesai diff --git a/app/src/main/res/values-it/strings.xml b/app/src/main/res/values-it/strings.xml index 35fdebeda..c92292f99 100644 --- a/app/src/main/res/values-it/strings.xml +++ b/app/src/main/res/values-it/strings.xml @@ -454,7 +454,6 @@ Impossibile connettersi al server Il server non invia dati Il server non accetta download multipli, riprovare con @string/msg_threads = 1 - Intervallo richiesto non soddisfatto Non trovato Post-processing fallito Pulisci i download completati diff --git a/app/src/main/res/values-ja/strings.xml b/app/src/main/res/values-ja/strings.xml index b67da798c..58ca2ebff 100644 --- a/app/src/main/res/values-ja/strings.xml +++ b/app/src/main/res/values-ja/strings.xml @@ -440,7 +440,6 @@ サーバに接続できません サーバがデータを送信していません サーバが同時接続ダウンロードを受け付けません。再試行してください @string/msg_threads = 1 - 必要な範囲が満たされていません 見つかりません 保存処理に失敗しました 完了済みを一覧から削除します diff --git a/app/src/main/res/values-ko/strings.xml b/app/src/main/res/values-ko/strings.xml index 333891910..fdc76b04e 100644 --- a/app/src/main/res/values-ko/strings.xml +++ b/app/src/main/res/values-ko/strings.xml @@ -451,7 +451,6 @@ 서버에 접속할 수 없습니다 서버가 데이터를 전송하지 않고 있습니다 서버가 다중 스레드 다운로드를 받아들이지 않습니다, @string/msg_threads = 1 를 사용해 다시 시도해보세요 - 요청된 HTTP 범위가 충분하지 않습니다 HTTP 찾을 수 없습니다 후처리 작업이 실패하였습니다 완료된 다운로드 비우기 diff --git a/app/src/main/res/values-ms/strings.xml b/app/src/main/res/values-ms/strings.xml index c7fa5de92..daa120ea2 100644 --- a/app/src/main/res/values-ms/strings.xml +++ b/app/src/main/res/values-ms/strings.xml @@ -450,7 +450,6 @@ Tidak dapat menyambung ke server Server tidak menghantar data Server tidak menerima muat turun berbilang thread, cuba lagi dengan @string/msg_threads = 1 - Julat yang diminta tidak memuaskan Tidak ditemui Pemprosesan-pasca gagal Hapuskan senarai muat turun yang selesai diff --git a/app/src/main/res/values-nb-rNO/strings.xml b/app/src/main/res/values-nb-rNO/strings.xml index d26886844..6262480b0 100644 --- a/app/src/main/res/values-nb-rNO/strings.xml +++ b/app/src/main/res/values-nb-rNO/strings.xml @@ -496,7 +496,7 @@ Sett nedlastinger på pause Spør om hvor ting skal lastes ned til Du vil bli spurt om hvor hver nedlasting skal plasseres - Du vil bli spurt om hvor hver nedlasting skal plasseres. + Du vil bli spurt om hvor hver nedlasting skal plasseres. \nSkru på SAF hvis du vil laste ned til eksternt SD-kort Bruk SAF Lagringstilgangsrammeverk (SAF) tillater nedlastinger til eksternt SD-kort. diff --git a/app/src/main/res/values-nl-rBE/strings.xml b/app/src/main/res/values-nl-rBE/strings.xml index 94feb4915..f64ff6bf9 100644 --- a/app/src/main/res/values-nl-rBE/strings.xml +++ b/app/src/main/res/values-nl-rBE/strings.xml @@ -454,7 +454,6 @@ Kan geen verbinding maken met de server De server verzendt geen gegevens De server aanvaardt geen meerdradige downloads, probeert het opnieuw met @string/msg_threads = 1 - Gevraagd bereik niet beschikbaar Niet gevonden Nabewerking mislukt Voltooide downloads wissen diff --git a/app/src/main/res/values-nl/strings.xml b/app/src/main/res/values-nl/strings.xml index f7acba6ae..6aecc2cd1 100644 --- a/app/src/main/res/values-nl/strings.xml +++ b/app/src/main/res/values-nl/strings.xml @@ -454,7 +454,6 @@ Kan niet met de server verbinden De server verzendt geen gegevens De server accepteert geen multi-threaded downloads, probeer het opnieuw met @string/msg_threads = 1 - Gevraagde bereik niet beschikbaar Niet gevonden Nabewerking mislukt Voltooide downloads wissen diff --git a/app/src/main/res/values-pa/strings.xml b/app/src/main/res/values-pa/strings.xml index c31eb805d..b57564eba 100644 --- a/app/src/main/res/values-pa/strings.xml +++ b/app/src/main/res/values-pa/strings.xml @@ -450,7 +450,6 @@ ਸਰਵਰ ਨਾਲ ਜੁੜ ਨਹੀਂ ਸਕਦਾ ਸਰਵਰ ਨੇ ਡਾਟਾ ਨਹੀਂ ਭੇਜਿਆ ਸਰਵਰ ਮਲਟੀ-Threaded ਡਾਊਨਲੋਡਸ ਨੂੰ ਸਵੀਕਾਰ ਨਹੀਂ ਕਰਦਾ, ਇਸ ਨਾਲ ਦੁਬਾਰਾ ਕੋਸ਼ਿਸ਼ ਕਰੋ @string/msg_threads = 1 - ਬੇਨਤੀ ਕੀਤੀ ਸੀਮਾ ਤਸੱਲੀਬਖਸ਼ ਨਹੀਂ ਹੈ ਨਹੀਂ ਲਭਿਆ Post-processing ਫੇਲ੍ਹ ਮੁਕੰਮਲ ਹੋਈਆਂ ਡਾਊਨਲੋਡ ਸਾਫ਼ ਕਰੋ diff --git a/app/src/main/res/values-pl/strings.xml b/app/src/main/res/values-pl/strings.xml index d3c84aa22..ca1e52ff2 100644 --- a/app/src/main/res/values-pl/strings.xml +++ b/app/src/main/res/values-pl/strings.xml @@ -456,7 +456,6 @@ Nie można połączyć się z serwerem Serwer nie wysyła danych Serwer nie akceptuje pobierania wielowątkowego, spróbuj ponownie za pomocą @string/msg_threads = 1 - Niewłaściwy zakres Nie znaleziono Przetwarzanie końcowe nie powiodło się Wyczyść ukończone pobieranie diff --git a/app/src/main/res/values-pt-rBR/strings.xml b/app/src/main/res/values-pt-rBR/strings.xml index aaac4fd4c..0bdf4d006 100644 --- a/app/src/main/res/values-pt-rBR/strings.xml +++ b/app/src/main/res/values-pt-rBR/strings.xml @@ -463,7 +463,6 @@ abrir em modo popup Não foi possível conectar ao servidor O servidor não envia dados O servidor não aceita downloads em multi-thread, tente com @string/msg_threads = 1 - Intervalo solicitado não aceito Não encontrado Falha no pós processamento Limpar downloads finalizados diff --git a/app/src/main/res/values-pt/strings.xml b/app/src/main/res/values-pt/strings.xml index 5d7cd8146..6d55023d1 100644 --- a/app/src/main/res/values-pt/strings.xml +++ b/app/src/main/res/values-pt/strings.xml @@ -452,7 +452,6 @@ Não é possível ligar ao servidor O servidor não envia dados O servidor não aceita transferências de vários processos, tente novamente com @string/msg_threads = 1 - Intervalo solicitado não satisfatório Não encontrado Pós-processamento falhado Limpar transferências concluídas diff --git a/app/src/main/res/values-ru/strings.xml b/app/src/main/res/values-ru/strings.xml index 6f079a221..51771e1b1 100644 --- a/app/src/main/res/values-ru/strings.xml +++ b/app/src/main/res/values-ru/strings.xml @@ -454,7 +454,6 @@ Доступ запрещён системой Сервер не найден Сервер не принимает многопоточные загрузки, повторная попытка с @string/msg_threads = 1 - Запрашиваемый диапазон недопустим Не найдено Очистить завершённые Остановить diff --git a/app/src/main/res/values-sk/strings.xml b/app/src/main/res/values-sk/strings.xml index 09502f60a..36c0afd84 100644 --- a/app/src/main/res/values-sk/strings.xml +++ b/app/src/main/res/values-sk/strings.xml @@ -462,7 +462,6 @@ Nepodarilo sa pripojiť k serveru Server neposiela údaje Server neakceptuje preberanie viacerých vlákien, zopakujte s @string/msg_threads = 1 - Požadovaný rozsah nie je uspokojivý Nenájdené Post-spracovanie zlyhalo Vyčistiť dokončené sťahovania diff --git a/app/src/main/res/values-tr/strings.xml b/app/src/main/res/values-tr/strings.xml index c17b58f50..6c9c66f69 100644 --- a/app/src/main/res/values-tr/strings.xml +++ b/app/src/main/res/values-tr/strings.xml @@ -449,7 +449,6 @@ Sunucuya bağlanılamıyor Sunucu veri göndermiyor Sunucu, çok iş parçacıklı indirmeleri kabul etmez, @string/msg_threads = 1 ile yeniden deneyin - İstenen aralık karşılanamıyor Bulunamadı İşlem sonrası başarısız Tamamlanan indirmeleri temizle diff --git a/app/src/main/res/values-uk/strings.xml b/app/src/main/res/values-uk/strings.xml index 375557b04..fcce99e89 100644 --- a/app/src/main/res/values-uk/strings.xml +++ b/app/src/main/res/values-uk/strings.xml @@ -471,7 +471,6 @@ Помилка зчитування збережених вкладок. Використовую типові вкладки. Вкладки, що відображаються на головній сторінці Показувати сповіщення з пропозицією оновити застосунок за наявності нової версії - Запитуваний діапазон неприпустимий Продовжити ваші %s відкладених переміщень із Завантажень Корисно під час переходу на мобільні дані, хоча деякі завантаження не можуть бути призупинені Показувати коментарі diff --git a/app/src/main/res/values-vi/strings.xml b/app/src/main/res/values-vi/strings.xml index 74b8b395c..f8860acfd 100644 --- a/app/src/main/res/values-vi/strings.xml +++ b/app/src/main/res/values-vi/strings.xml @@ -449,7 +449,6 @@ Không thế kết nối với máy chủ Máy chủ không gửi dữ liệu về Máy chủ không chấp nhận tải đa luồng, thử lại với số luồng = 1 - (HTTP) Không thể đáp ứng khoảng dữ liệu đã yêu cầu Không tìm thấy Xử lý thất bại Dọn các tải về đã hoàn thành diff --git a/app/src/main/res/values-zh-rTW/strings.xml b/app/src/main/res/values-zh-rTW/strings.xml index fe4c1b00a..310bae3a3 100644 --- a/app/src/main/res/values-zh-rTW/strings.xml +++ b/app/src/main/res/values-zh-rTW/strings.xml @@ -447,7 +447,6 @@ 無法連線到伺服器 伺服器沒有傳送資料 伺服器不接受多執行緒下載，請以 @string/msg_threads = 1 重試 - 請求範圍無法滿足 找不到 後處理失敗 清除已結束的下載 diff --git a/app/src/main/res/values/strings.xml b/app/src/main/res/values/strings.xml index a34b00ea9..2917fb9fd 100644 --- a/app/src/main/res/values/strings.xml +++ b/app/src/main/res/values/strings.xml @@ -551,13 +551,13 @@ Can not connect to the server The server does not send data The server does not accept multi-threaded downloads, retry with @string/msg_threads = 1 - Requested range not satisfiable Not found Post-processing failed NewPipe was closed while working on the file No space left on device Progress lost, because the file was deleted Connection timeout + The solicited resource is not available anymore Clear finished downloads Are you sure? Continue your %s pending transfers from Downloads From 429ee7eb9381034a6cbbd55427b312c3363ba3c0 Mon Sep 17 00:00:00 2001 From: kapodamy Date: Sun, 29 Sep 2019 01:44:13 -0300 Subject: [PATCH 06/28] Mp4FromDashWriter fixes * correct calculation of "co64" box and usage of 64bits offsets * generate one chunk for audio streams like ffmpeg does, attempt to fix cut-off audio * misc. cleanup --- .../newpipe/streams/Mp4FromDashWriter.java | 79 ++++++++++++------- 1 file changed, 50 insertions(+), 29 deletions(-) diff --git a/app/src/main/java/org/schabi/newpipe/streams/Mp4FromDashWriter.java b/app/src/main/java/org/schabi/newpipe/streams/Mp4FromDashWriter.java index 03aab447c..420f77955 100644 --- a/app/src/main/java/org/schabi/newpipe/streams/Mp4FromDashWriter.java +++ b/app/src/main/java/org/schabi/newpipe/streams/Mp4FromDashWriter.java @@ -6,6 +6,7 @@ import org.schabi.newpipe.streams.Mp4DashReader.Mp4DashChunk; import org.schabi.newpipe.streams.Mp4DashReader.Mp4DashSample; import org.schabi.newpipe.streams.Mp4DashReader.Mp4Track; import org.schabi.newpipe.streams.Mp4DashReader.TrunEntry; +import org.schabi.newpipe.streams.Mp4DashReader.TrackKind; import org.schabi.newpipe.streams.io.SharpStream; import java.io.IOException; @@ -22,6 +23,7 @@ public class Mp4FromDashWriter { private final static byte SAMPLES_PER_CHUNK = 6;// ffmpeg uses 2, basic uses 1 (with 60fps uses 21 or 22). NewPipe will use 6 private final static long THRESHOLD_FOR_CO64 = 0xFFFEFFFFL;// near 3.999 GiB private final static int THRESHOLD_MOOV_LENGTH = (256 * 1024) + (2048 * 1024); // 2.2 MiB enough for: 1080p 60fps 00h35m00s + private final static short SINGLE_CHUNK_SAMPLE_BUFFER = 256; private final long time; @@ -145,7 +147,7 @@ public class Mp4FromDashWriter { // not allowed for very short tracks (less than 0.5 seconds) // outStream = output; - int read = 8;// mdat box header size + long read = 8;// mdat box header size long totalSampleSize = 0; int[] sampleExtra = new int[readers.length]; int[] defaultMediaTime = new int[readers.length]; @@ -157,6 +159,8 @@ public class Mp4FromDashWriter { tablesInfo[i] = new TablesInfo(); } + boolean singleChunk = tracks.length == 1 && tracks[0].kind == TrackKind.Audio; + // for (int i = 0; i < readers.length; i++) { int samplesSize = 0; @@ -210,14 +214,21 @@ public class Mp4FromDashWriter { tablesInfo[i].stco = (tmp / SAMPLES_PER_CHUNK) + 1;// +1 for samples in first chunk tmp = tmp % SAMPLES_PER_CHUNK; - if (tmp == 0) { + if (singleChunk) { + // avoid split audio streams in chunks + tablesInfo[i].stsc = 1; + tablesInfo[i].stsc_bEntries = new int[]{ + 1, tablesInfo[i].stsz, 1 + }; + tablesInfo[i].stco = 1; + } else if (tmp == 0) { tablesInfo[i].stsc = 2;// first chunk (init) and succesive chunks tablesInfo[i].stsc_bEntries = new int[]{ 1, SAMPLES_PER_CHUNK_INIT, 1, 2, SAMPLES_PER_CHUNK, 1 }; } else { - tablesInfo[i].stsc = 3;// first chunk (init) and succesive chunks and remain chunk + tablesInfo[i].stsc = 3;// first chunk (init) and successive chunks and remain chunk tablesInfo[i].stsc_bEntries = new int[]{ 1, SAMPLES_PER_CHUNK_INIT, 1, 2, SAMPLES_PER_CHUNK, 1, @@ -268,10 +279,10 @@ public class Mp4FromDashWriter { } else {*/ if (auxSize > 0) { int length = auxSize; - byte[] buffer = new byte[8 * 1024];// 8 KiB + byte[] buffer = new byte[64 * 1024];// 64 KiB while (length > 0) { int count = Math.min(length, buffer.length); - outWrite(buffer, 0, count); + outWrite(buffer, count); length -= count; } } @@ -280,7 +291,7 @@ public class Mp4FromDashWriter { outSeek(ftyp_size); } - // tablesInfo contais row counts + // tablesInfo contains row counts // and after returning from make_moov() will contain table offsets make_moov(defaultMediaTime, tablesInfo, is64); @@ -291,7 +302,7 @@ public class Mp4FromDashWriter { writeEntryArray(tablesInfo[i].stsc, tablesInfo[i].stsc_bEntries.length, tablesInfo[i].stsc_bEntries); tablesInfo[i].stsc_bEntries = null; if (tablesInfo[i].ctts > 0) { - sampleCount[i] = 1;// index is not base zero + sampleCount[i] = 1;// the index is not base zero sampleExtra[i] = -1; } } @@ -303,8 +314,8 @@ public class Mp4FromDashWriter { outWrite(make_mdat(totalSampleSize, is64)); int[] sampleIndex = new int[readers.length]; - int[] sizes = new int[SAMPLES_PER_CHUNK]; - int[] sync = new int[SAMPLES_PER_CHUNK]; + int[] sizes = new int[singleChunk ? SINGLE_CHUNK_SAMPLE_BUFFER : SAMPLES_PER_CHUNK]; + int[] sync = new int[singleChunk ? SINGLE_CHUNK_SAMPLE_BUFFER : SAMPLES_PER_CHUNK]; int written = readers.length; while (written > 0) { @@ -317,7 +328,12 @@ public class Mp4FromDashWriter { long chunkOffset = writeOffset; int syncCount = 0; - int limit = sampleIndex[i] == 0 ? SAMPLES_PER_CHUNK_INIT : SAMPLES_PER_CHUNK; + int limit; + if (singleChunk) { + limit = SINGLE_CHUNK_SAMPLE_BUFFER; + } else { + limit = sampleIndex[i] == 0 ? SAMPLES_PER_CHUNK_INIT : SAMPLES_PER_CHUNK; + } int j = 0; for (; j < limit; j++) { @@ -354,7 +370,7 @@ public class Mp4FromDashWriter { sizes[j] = sample.data.length; } - outWrite(sample.data, 0, sample.data.length); + outWrite(sample.data, sample.data.length); } if (j > 0) { @@ -368,10 +384,16 @@ public class Mp4FromDashWriter { tablesInfo[i].stss = writeEntryArray(tablesInfo[i].stss, syncCount, sync); } - if (is64) { - tablesInfo[i].stco = writeEntry64(tablesInfo[i].stco, chunkOffset); - } else { - tablesInfo[i].stco = writeEntryArray(tablesInfo[i].stco, 1, (int) chunkOffset); + if (tablesInfo[i].stco > 0) { + if (is64) { + tablesInfo[i].stco = writeEntry64(tablesInfo[i].stco, chunkOffset); + } else { + tablesInfo[i].stco = writeEntryArray(tablesInfo[i].stco, 1, (int) chunkOffset); + } + + if (singleChunk) { + tablesInfo[i].stco = -1; + } } outRestore(); @@ -451,12 +473,12 @@ public class Mp4FromDashWriter { // private void outWrite(byte[] buffer) throws IOException { - outWrite(buffer, 0, buffer.length); + outWrite(buffer, buffer.length); } - private void outWrite(byte[] buffer, int offset, int count) throws IOException { + private void outWrite(byte[] buffer, int count) throws IOException { writeOffset += count; - outStream.write(buffer, offset, count); + outStream.write(buffer, 0, count); } private void outSeek(long offset) throws IOException { @@ -509,7 +531,6 @@ public class Mp4FromDashWriter { ); if (extra >= 0) { - //size += 4;// commented for auxiliar buffer !!! offset += 4; auxWrite(extra); } @@ -531,7 +552,7 @@ public class Mp4FromDashWriter { if (moovSimulation) { writeOffset += buffer.length; } else if (auxBuffer == null) { - outWrite(buffer, 0, buffer.length); + outWrite(buffer, buffer.length); } else { auxBuffer.put(buffer); } @@ -703,7 +724,7 @@ public class Mp4FromDashWriter { int mediaTime; if (tracks[index].trak.edst_elst == null) { - // is a audio track ¿is edst/elst opcional for audio tracks? + // is a audio track ¿is edst/elst optional for audio tracks? mediaTime = 0x00;// ffmpeg set this value as zero, instead of defaultMediaTime bMediaRate = 0x00010000; } else { @@ -798,13 +819,13 @@ public class Mp4FromDashWriter { class TablesInfo { - public int stts; - public int stsc; - public int[] stsc_bEntries; - public int ctts; - public int stsz; - public int stsz_default; - public int stss; - public int stco; + int stts; + int stsc; + int[] stsc_bEntries; + int ctts; + int stsz; + int stsz_default; + int stss; + int stco; } } From 160a33e8c844a7aa7534fe798a3472a015315480 Mon Sep 17 00:00:00 2001 From: kapodamy Date: Mon, 30 Sep 2019 23:52:49 -0300 Subject: [PATCH 07/28] misc changes * OggFromWebMWriter: rewrite (again), reduce iterations over the input. Works as-is (video streams are not supported) * WebMReader: use int for SimpleBlock.dataSize instead of long * Download Recovery: allow recovering uninitialized downloads * check range-requests using HEAD method instead of GET * DownloadRunnableFallback: add workaround for 32kB/s issue, unknown issue origin, wont fix * reporting downloads errors now include the source url with the selected quality and format --- .../newpipe/streams/OggFromWebMWriter.java | 216 +++++------------- .../schabi/newpipe/streams/WebMReader.java | 4 +- .../giga/get/DownloadInitializer.java | 35 +-- .../us/shandian/giga/get/DownloadMission.java | 36 +-- .../giga/get/DownloadMissionRecover.java | 146 +++++++++--- .../shandian/giga/get/DownloadRunnable.java | 2 +- .../giga/get/DownloadRunnableFallback.java | 20 +- .../giga/get/MissionRecoveryInfo.java | 43 +++- .../giga/ui/adapter/MissionAdapter.java | 36 ++- app/src/main/res/values-es/strings.xml | 2 +- app/src/main/res/values/strings.xml | 2 +- 11 files changed, 294 insertions(+), 248 deletions(-) diff --git a/app/src/main/java/org/schabi/newpipe/streams/OggFromWebMWriter.java b/app/src/main/java/org/schabi/newpipe/streams/OggFromWebMWriter.java index 091ae6d2a..e6363e423 100644 --- a/app/src/main/java/org/schabi/newpipe/streams/OggFromWebMWriter.java +++ b/app/src/main/java/org/schabi/newpipe/streams/OggFromWebMWriter.java @@ -12,8 +12,6 @@ import java.io.Closeable; import java.io.IOException; import java.nio.ByteBuffer; import java.nio.ByteOrder; -import java.util.ArrayList; -import java.util.Random; import javax.annotation.Nullable; @@ -23,15 +21,13 @@ import javax.annotation.Nullable; public class OggFromWebMWriter implements Closeable { private static final byte FLAG_UNSET = 0x00; - private static final byte FLAG_CONTINUED = 0x01; + //private static final byte FLAG_CONTINUED = 0x01; private static final byte FLAG_FIRST = 0x02; private static final byte FLAG_LAST = 0x04; private final static byte HEADER_CHECKSUM_OFFSET = 22; private final static byte HEADER_SIZE = 27; - private final static short BUFFER_SIZE = 8 * 1024;// 8KiB - private final static int TIME_SCALE_NS = 1000000000; private boolean done = false; @@ -43,7 +39,6 @@ public class OggFromWebMWriter implements Closeable { private int sequence_count = 0; private final int STREAM_ID; private byte packet_flag = FLAG_FIRST; - private int track_index = 0; private WebMReader webm = null; private WebMTrack webm_track = null; @@ -71,7 +66,7 @@ public class OggFromWebMWriter implements Closeable { this.source = source; this.output = target; - this.STREAM_ID = (new Random(System.currentTimeMillis())).nextInt(); + this.STREAM_ID = (int) System.currentTimeMillis(); populate_crc32_table(); } @@ -130,7 +125,6 @@ public class OggFromWebMWriter implements Closeable { try { webm_track = webm.selectTrack(trackIndex); - track_index = trackIndex; } finally { parsed = true; } @@ -154,8 +148,11 @@ public class OggFromWebMWriter implements Closeable { public void build() throws IOException { float resolution; - int read; - byte[] buffer; + SimpleBlock bloq; + ByteBuffer header = ByteBuffer.allocate(27 + (255 * 255)); + ByteBuffer page = ByteBuffer.allocate(64 * 1024); + + header.order(ByteOrder.LITTLE_ENDIAN); /* step 1: get the amount of frames per seconds */ switch (webm_track.kind) { @@ -176,57 +173,32 @@ public class OggFromWebMWriter implements Closeable { throw new RuntimeException("not implemented"); } - /* step 2a: create packet with code init data */ - ArrayList data_extra = new ArrayList<>(4); - + /* step 2: create packet with code init data */ if (webm_track.codecPrivate != null) { addPacketSegment(webm_track.codecPrivate.length); - ByteBuffer buff = byte_buffer(HEADER_SIZE + segment_table_size + webm_track.codecPrivate.length); - - make_packetHeader(0x00, buff, webm_track.codecPrivate); - data_extra.add(buff.array()); + make_packetHeader(0x00, header, webm_track.codecPrivate); + write(header); + output.write(webm_track.codecPrivate); } - /* step 2b: create packet with metadata */ - buffer = make_metadata(); + /* step 3: create packet with metadata */ + byte[] buffer = make_metadata(); if (buffer != null) { addPacketSegment(buffer.length); - ByteBuffer buff = byte_buffer(HEADER_SIZE + segment_table_size + buffer.length); - - make_packetHeader(0x00, buff, buffer); - data_extra.add(buff.array()); + make_packetHeader(0x00, header, buffer); + write(header); + output.write(buffer); } - - /* step 3: calculate amount of packets */ - SimpleBlock bloq; - int reserve_header = 0; - int headers_amount = 0; - + /* step 4: calculate amount of packets */ while (webm_segment != null) { bloq = getNextBlock(); - if (addPacketSegment(bloq)) { - continue; - } - - reserve_header += HEADER_SIZE + segment_table_size;// header size - clearSegmentTable(); - webm_block = bloq; - headers_amount++; - } - - /* step 4: create packet headers */ - rewind_source(); - - ByteBuffer headers = byte_buffer(reserve_header); - short[] headers_size = new short[headers_amount]; - int header_index = 0; - - while (webm_segment != null) { - bloq = getNextBlock(); - - if (addPacketSegment(bloq)) { + if (bloq != null && addPacketSegment(bloq)) { + int pos = page.position(); + //noinspection ResultOfMethodCallIgnored + bloq.data.read(page.array(), pos, bloq.dataSize); + page.position(pos + bloq.dataSize); continue; } @@ -251,75 +223,21 @@ public class OggFromWebMWriter implements Closeable { elapsed_ns = elapsed_ns / TIME_SCALE_NS; elapsed_ns = Math.ceil(elapsed_ns * resolution); - // create header - headers_size[header_index++] = make_packetHeader((long) elapsed_ns, headers, null); + // create header and calculate page checksum + int checksum = make_packetHeader((long) elapsed_ns, header, null); + checksum = calc_crc32(checksum, page.array(), page.position()); + + header.putInt(HEADER_CHECKSUM_OFFSET, checksum); + + // dump data + write(header); + write(page); + webm_block = bloq; } - - - /* step 5: calculate checksums */ - rewind_source(); - - int offset = 0; - buffer = new byte[BUFFER_SIZE]; - - for (header_index = 0; header_index < headers_size.length; header_index++) { - int checksum_offset = offset + HEADER_CHECKSUM_OFFSET; - int checksum = headers.getInt(checksum_offset); - - while (webm_segment != null) { - bloq = getNextBlock(); - - if (!addPacketSegment(bloq)) { - clearSegmentTable(); - webm_block = bloq; - break; - } - - // calculate page checksum - while ((read = bloq.data.read(buffer)) > 0) { - checksum = calc_crc32(checksum, buffer, 0, read); - } - } - - headers.putInt(checksum_offset, checksum); - offset += headers_size[header_index]; - } - - /* step 6: write extra headers */ - rewind_source(); - - for (byte[] buff : data_extra) { - output.write(buff); - } - - /* step 7: write stream packets */ - byte[] headers_buffers = headers.array(); - offset = 0; - buffer = new byte[BUFFER_SIZE]; - - for (header_index = 0; header_index < headers_size.length; header_index++) { - output.write(headers_buffers, offset, headers_size[header_index]); - offset += headers_size[header_index]; - - while (webm_segment != null) { - bloq = getNextBlock(); - - if (addPacketSegment(bloq)) { - while ((read = bloq.data.read(buffer)) > 0) { - output.write(buffer, 0, read); - } - } else { - clearSegmentTable(); - webm_block = bloq; - break; - } - } - } } - private short make_packetHeader(long gran_pos, ByteBuffer buffer, byte[] immediate_page) { - int offset = buffer.position(); + private int make_packetHeader(long gran_pos, @NonNull ByteBuffer buffer, byte[] immediate_page) { short length = HEADER_SIZE; buffer.putInt(0x5367674f);// "OggS" binary string in little-endian @@ -340,17 +258,15 @@ public class OggFromWebMWriter implements Closeable { clearSegmentTable();// clear segment table for next header - int checksum_crc32 = calc_crc32(0x00, buffer.array(), offset, length); + int checksum_crc32 = calc_crc32(0x00, buffer.array(), length); if (immediate_page != null) { - checksum_crc32 = calc_crc32(checksum_crc32, immediate_page, 0, immediate_page.length); - System.arraycopy(immediate_page, 0, buffer.array(), length, immediate_page.length); + checksum_crc32 = calc_crc32(checksum_crc32, immediate_page, immediate_page.length); + buffer.putInt(HEADER_CHECKSUM_OFFSET, checksum_crc32); segment_table_next_timestamp -= TIME_SCALE_NS; } - buffer.putInt(offset + HEADER_CHECKSUM_OFFSET, checksum_crc32); - - return length; + return checksum_crc32; } @Nullable @@ -358,7 +274,7 @@ public class OggFromWebMWriter implements Closeable { if ("A_OPUS".equals(webm_track.codecId)) { return new byte[]{ 0x4F, 0x70, 0x75, 0x73, 0x54, 0x61, 0x67, 0x73,// "OpusTags" binary string - 0x07, 0x00, 0x00, 0x00,// writing application string size + 0x07, 0x00, 0x00, 0x00,// writting application string size 0x4E, 0x65, 0x77, 0x50, 0x69, 0x70, 0x65,// "NewPipe" binary string 0x00, 0x00, 0x00, 0x00// additional tags count (zero means no tags) }; @@ -366,7 +282,7 @@ public class OggFromWebMWriter implements Closeable { return new byte[]{ 0x03,// ???????? 0x76, 0x6f, 0x72, 0x62, 0x69, 0x73,// "vorbis" binary string - 0x07, 0x00, 0x00, 0x00,// writing application string size + 0x07, 0x00, 0x00, 0x00,// writting application string size 0x4E, 0x65, 0x77, 0x50, 0x69, 0x70, 0x65,// "NewPipe" binary string 0x01, 0x00, 0x00, 0x00,// additional tags count (zero means no tags) @@ -387,22 +303,9 @@ public class OggFromWebMWriter implements Closeable { return null; } - private void rewind_source() throws IOException { - source.rewind(); - - webm = new WebMReader(source); - webm.parse(); - webm_track = webm.selectTrack(track_index); - webm_segment = webm.getNextSegment(); - webm_cluster = null; - webm_block = null; - webm_block_last_timecode = 0L; - - segment_table_next_timestamp = TIME_SCALE_NS; - } - - private ByteBuffer byte_buffer(int size) { - return ByteBuffer.allocate(size).order(ByteOrder.LITTLE_ENDIAN); + private void write(ByteBuffer buffer) throws IOException { + output.write(buffer.array(), 0, buffer.position()); + buffer.position(0); } // @@ -460,41 +363,32 @@ public class OggFromWebMWriter implements Closeable { // private void clearSegmentTable() { - if (packet_flag != FLAG_CONTINUED) { - segment_table_next_timestamp += TIME_SCALE_NS; - packet_flag = FLAG_UNSET; - } + segment_table_next_timestamp += TIME_SCALE_NS; + packet_flag = FLAG_UNSET; segment_table_size = 0; } private boolean addPacketSegment(SimpleBlock block) { - if (block == null) { - return false; - } - long timestamp = block.absoluteTimeCodeNs + webm_track.codecDelay; if (timestamp >= segment_table_next_timestamp) { return false; } - boolean result = addPacketSegment((int) block.dataSize); - - if (!result && segment_table_next_timestamp < timestamp) { - // WARNING: ¡¡¡¡ not implemented (lack of documentation) !!!! - packet_flag = FLAG_CONTINUED; - } - - return result; + return addPacketSegment(block.dataSize); } private boolean addPacketSegment(int size) { + if (size > 65025) { + throw new UnsupportedOperationException("page size cannot be larger than 65025"); + } + int available = (segment_table.length - segment_table_size) * 255; - boolean extra = size == 255; + boolean extra = (size % 255) == 0; if (extra) { // add a zero byte entry in the table - // required to indicate the sample size is exactly 255 + // required to indicate the sample size is multiple of 255 available -= 255; } @@ -528,12 +422,10 @@ public class OggFromWebMWriter implements Closeable { } } - private int calc_crc32(int initial_crc, byte[] buffer, int offset, int size) { - size += offset; - - for (; offset < size; offset++) { + private int calc_crc32(int initial_crc, byte[] buffer, int size) { + for (int i = 0; i < size; i++) { int reg = (initial_crc >>> 24) & 0xff; - initial_crc = (initial_crc << 8) ^ crc32_table[reg ^ (buffer[offset] & 0xff)]; + initial_crc = (initial_crc << 8) ^ crc32_table[reg ^ (buffer[i] & 0xff)]; } return initial_crc; diff --git a/app/src/main/java/org/schabi/newpipe/streams/WebMReader.java b/app/src/main/java/org/schabi/newpipe/streams/WebMReader.java index 13c15370d..4cb96d901 100644 --- a/app/src/main/java/org/schabi/newpipe/streams/WebMReader.java +++ b/app/src/main/java/org/schabi/newpipe/streams/WebMReader.java @@ -368,7 +368,7 @@ public class WebMReader { obj.trackNumber = readEncodedNumber(); obj.relativeTimeCode = stream.readShort(); obj.flags = (byte) stream.read(); - obj.dataSize = (ref.offset + ref.size) - stream.position(); + obj.dataSize = (int) ((ref.offset + ref.size) - stream.position()); obj.createdFromBlock = ref.type == ID_Block; // NOTE: lacing is not implemented, and will be mixed with the stream data @@ -465,7 +465,7 @@ public class WebMReader { public short relativeTimeCode; public long absoluteTimeCodeNs; public byte flags; - public long dataSize; + public int dataSize; private final Element ref; public boolean isKeyframe() { diff --git a/app/src/main/java/us/shandian/giga/get/DownloadInitializer.java b/app/src/main/java/us/shandian/giga/get/DownloadInitializer.java index 593feafa7..17a2a7403 100644 --- a/app/src/main/java/us/shandian/giga/get/DownloadInitializer.java +++ b/app/src/main/java/us/shandian/giga/get/DownloadInitializer.java @@ -14,6 +14,7 @@ import java.nio.channels.ClosedByInterruptException; import us.shandian.giga.util.Utility; import static org.schabi.newpipe.BuildConfig.DEBUG; +import static us.shandian.giga.get.DownloadMission.ERROR_HTTP_FORBIDDEN; public class DownloadInitializer extends Thread { private final static String TAG = "DownloadInitializer"; @@ -29,9 +30,9 @@ public class DownloadInitializer extends Thread { mConn = null; } - private static void safeClose(HttpURLConnection con) { + private void dispose() { try { - con.getInputStream().close(); + mConn.getInputStream().close(); } catch (Exception e) { // nothing to do } @@ -52,9 +53,9 @@ public class DownloadInitializer extends Thread { long lowestSize = Long.MAX_VALUE; for (int i = 0; i < mMission.urls.length && mMission.running; i++) { - mConn = mMission.openConnection(mMission.urls[i], mId, -1, -1); + mConn = mMission.openConnection(mMission.urls[i], true, -1, -1); mMission.establishConnection(mId, mConn); - safeClose(mConn); + dispose(); if (Thread.interrupted()) return; long length = Utility.getContentLength(mConn); @@ -82,9 +83,9 @@ public class DownloadInitializer extends Thread { } } else { // ask for the current resource length - mConn = mMission.openConnection(mId, -1, -1); + mConn = mMission.openConnection(true, -1, -1); mMission.establishConnection(mId, mConn); - safeClose(mConn); + dispose(); if (!mMission.running || Thread.interrupted()) return; @@ -108,9 +109,9 @@ public class DownloadInitializer extends Thread { } } else { // Open again - mConn = mMission.openConnection(mId, mMission.length - 10, mMission.length); + mConn = mMission.openConnection(true, mMission.length - 10, mMission.length); mMission.establishConnection(mId, mConn); - safeClose(mConn); + dispose(); if (!mMission.running || Thread.interrupted()) return; @@ -171,7 +172,14 @@ public class DownloadInitializer extends Thread { } catch (InterruptedIOException | ClosedByInterruptException e) { return; } catch (Exception e) { - if (!mMission.running) return; + if (!mMission.running || super.isInterrupted()) return; + + if (e instanceof DownloadMission.HttpError && ((DownloadMission.HttpError) e).statusCode == ERROR_HTTP_FORBIDDEN) { + // for youtube streams. The url has expired + interrupt(); + mMission.doRecover(e); + return; + } if (e instanceof IOException && e.getMessage().contains("Permission denied")) { mMission.notifyError(DownloadMission.ERROR_PERMISSION_DENIED, e); @@ -194,13 +202,6 @@ public class DownloadInitializer extends Thread { @Override public void interrupt() { super.interrupt(); - - if (mConn != null) { - try { - mConn.disconnect(); - } catch (Exception e) { - // nothing to do - } - } + if (mConn != null) dispose(); } } diff --git a/app/src/main/java/us/shandian/giga/get/DownloadMission.java b/app/src/main/java/us/shandian/giga/get/DownloadMission.java index 77b417118..918d6dbea 100644 --- a/app/src/main/java/us/shandian/giga/get/DownloadMission.java +++ b/app/src/main/java/us/shandian/giga/get/DownloadMission.java @@ -204,22 +204,24 @@ public class DownloadMission extends Mission { /** * Opens a connection * - * @param threadId id of the calling thread, used only for debugging - * @param rangeStart range start - * @param rangeEnd range end + * @param headRequest {@code true} for use {@code HEAD} request method, otherwise, {@code GET} is used + * @param rangeStart range start + * @param rangeEnd range end * @return a {@link java.net.URLConnection URLConnection} linking to the URL. * @throws IOException if an I/O exception occurs. */ - HttpURLConnection openConnection(int threadId, long rangeStart, long rangeEnd) throws IOException { - return openConnection(urls[current], threadId, rangeStart, rangeEnd); + HttpURLConnection openConnection(boolean headRequest, long rangeStart, long rangeEnd) throws IOException { + return openConnection(urls[current], headRequest, rangeStart, rangeEnd); } - HttpURLConnection openConnection(String url, int threadId, long rangeStart, long rangeEnd) throws IOException { + HttpURLConnection openConnection(String url, boolean headRequest, long rangeStart, long rangeEnd) throws IOException { HttpURLConnection conn = (HttpURLConnection) new URL(url).openConnection(); conn.setInstanceFollowRedirects(true); conn.setRequestProperty("User-Agent", DownloaderImpl.USER_AGENT); conn.setRequestProperty("Accept", "*/*"); + if (headRequest) conn.setRequestMethod("HEAD"); + // BUG workaround: switching between networks can freeze the download forever conn.setConnectTimeout(30000); conn.setReadTimeout(10000); @@ -229,10 +231,6 @@ public class DownloadMission extends Mission { if (rangeEnd > 0) req += rangeEnd; conn.setRequestProperty("Range", req); - - if (DEBUG) { - Log.d(TAG, threadId + ":" + conn.getRequestProperty("Range")); - } } return conn; @@ -245,13 +243,14 @@ public class DownloadMission extends Mission { * @throws HttpError if the HTTP Status-Code is not satisfiable */ void establishConnection(int threadId, HttpURLConnection conn) throws IOException, HttpError { - conn.connect(); int statusCode = conn.getResponseCode(); if (DEBUG) { + Log.d(TAG, threadId + ":Range=" + conn.getRequestProperty("Range")); Log.d(TAG, threadId + ":Content-Length=" + conn.getContentLength() + " Code:" + statusCode); } + switch (statusCode) { case 204: case 205: @@ -676,6 +675,15 @@ public class DownloadMission extends Mission { return (isPsFailed() || errCode == ERROR_POSTPROCESSING_HOLD) || isFinished(); } + /** + * Indicates if mission urls has expired and there an attempt to renovate them + * + * @return {@code true} if the mission is running a recovery procedure, otherwise, {@code false} + */ + public boolean isRecovering() { + return threads != null && threads.length > 0 && threads[0] instanceof DownloadRunnable && threads[0].isAlive(); + } + private boolean doPostprocessing() { if (psAlgorithm == null || psState == 2) return true; @@ -742,10 +750,8 @@ public class DownloadMission extends Mission { } } - // set the current download url to null in case if the recovery - // process is canceled. Next time start() method is called the - // recovery will be executed, saving time - urls[current] = null; + errCode = ERROR_NOTHING; + errObject = null; if (recoveryInfo[current].attempts >= maxRetry) { recoveryInfo[current].attempts = 0; diff --git a/app/src/main/java/us/shandian/giga/get/DownloadMissionRecover.java b/app/src/main/java/us/shandian/giga/get/DownloadMissionRecover.java index 9abd93717..5efbd1153 100644 --- a/app/src/main/java/us/shandian/giga/get/DownloadMissionRecover.java +++ b/app/src/main/java/us/shandian/giga/get/DownloadMissionRecover.java @@ -10,10 +10,12 @@ import org.schabi.newpipe.extractor.stream.SubtitlesStream; import org.schabi.newpipe.extractor.stream.VideoStream; import java.io.IOException; +import java.io.InterruptedIOException; import java.net.HttpURLConnection; import java.nio.channels.ClosedByInterruptException; import java.util.List; +import static us.shandian.giga.get.DownloadMission.ERROR_NOTHING; import static us.shandian.giga.get.DownloadMission.ERROR_RESOURCE_GONE; public class DownloadMissionRecover extends Thread { @@ -21,14 +23,17 @@ public class DownloadMissionRecover extends Thread { static final int mID = -3; private final DownloadMission mMission; - private final MissionRecoveryInfo mRecovery; private final Exception mFromError; + private final boolean notInitialized; + private HttpURLConnection mConn; + private MissionRecoveryInfo mRecovery; + private StreamExtractor mExtractor; DownloadMissionRecover(DownloadMission mission, Exception originError) { mMission = mission; mFromError = originError; - mRecovery = mission.recoveryInfo[mission.current]; + notInitialized = mission.blocks == null && mission.current == 0; } @Override @@ -38,28 +43,78 @@ public class DownloadMissionRecover extends Thread { return; } + /*if (mMission.source.startsWith(MissionRecoveryInfo.DIRECT_SOURCE)) { + resolve(mMission.source.substring(MissionRecoveryInfo.DIRECT_SOURCE.length())); + return; + }*/ + try { - /*if (mMission.source.startsWith(MissionRecoveryInfo.DIRECT_SOURCE)) { - resolve(mMission.source.substring(MissionRecoveryInfo.DIRECT_SOURCE.length())); - return; - }*/ - StreamingService svr = NewPipe.getServiceByUrl(mMission.source); - - if (svr == null) { - throw new RuntimeException("Unknown source service"); - } - - StreamExtractor extractor = svr.getStreamExtractor(mMission.source); - extractor.fetchPage(); - + mExtractor = svr.getStreamExtractor(mMission.source); + mExtractor.fetchPage(); + } catch (InterruptedIOException | ClosedByInterruptException e) { + return; + } catch (Exception e) { if (!mMission.running || super.isInterrupted()) return; + mMission.notifyError(e); + return; + } + // maybe the following check is redundant + if (!mMission.running || super.isInterrupted()) return; + + if (!notInitialized) { + // set the current download url to null in case if the recovery + // process is canceled. Next time start() method is called the + // recovery will be executed, saving time + mMission.urls[mMission.current] = null; + + mRecovery = mMission.recoveryInfo[mMission.current]; + resolveStream(); + return; + } + + Log.w(TAG, "mission is not fully initialized, this will take a while"); + + try { + for (; mMission.current < mMission.urls.length; mMission.current++) { + mRecovery = mMission.recoveryInfo[mMission.current]; + + if (test()) continue; + if (!mMission.running) return; + + resolveStream(); + if (!mMission.running) return; + + // before continue, check if the current stream was resolved + if (mMission.urls[mMission.current] == null || mMission.errCode != ERROR_NOTHING) { + break; + } + } + } finally { + mMission.current = 0; + } + + mMission.writeThisToFile(); + + if (!mMission.running || super.isInterrupted()) return; + + mMission.running = false; + mMission.start(); + } + + private void resolveStream() { + if (mExtractor.getErrorMessage() != null) { + mMission.notifyError(mFromError); + return; + } + + try { String url = null; - switch (mMission.kind) { + switch (mRecovery.kind) { case 'a': - for (AudioStream audio : extractor.getAudioStreams()) { + for (AudioStream audio : mExtractor.getAudioStreams()) { if (audio.average_bitrate == mRecovery.desiredBitrate && audio.getFormat() == mRecovery.format) { url = audio.getUrl(); break; @@ -69,9 +124,9 @@ public class DownloadMissionRecover extends Thread { case 'v': List videoStreams; if (mRecovery.desired2) - videoStreams = extractor.getVideoOnlyStreams(); + videoStreams = mExtractor.getVideoOnlyStreams(); else - videoStreams = extractor.getVideoStreams(); + videoStreams = mExtractor.getVideoStreams(); for (VideoStream video : videoStreams) { if (video.resolution.equals(mRecovery.desired) && video.getFormat() == mRecovery.format) { url = video.getUrl(); @@ -80,7 +135,7 @@ public class DownloadMissionRecover extends Thread { } break; case 's': - for (SubtitlesStream subtitles : extractor.getSubtitles(mRecovery.format)) { + for (SubtitlesStream subtitles : mExtractor.getSubtitles(mRecovery.format)) { String tag = subtitles.getLanguageTag(); if (tag.equals(mRecovery.desired) && subtitles.isAutoGenerated() == mRecovery.desired2) { url = subtitles.getURL(); @@ -114,7 +169,7 @@ public class DownloadMissionRecover extends Thread { ////// Validate the http resource doing a range request ///////////////////// try { - mConn = mMission.openConnection(url, mID, mMission.length - 10, mMission.length); + mConn = mMission.openConnection(url, true, mMission.length - 10, mMission.length); mConn.setRequestProperty("If-Range", mRecovery.validateCondition); mMission.establishConnection(mID, mConn); @@ -140,22 +195,24 @@ public class DownloadMissionRecover extends Thread { if (!mMission.running || e instanceof ClosedByInterruptException) return; throw e; } finally { - this.interrupt(); + disconnect(); } } private void recover(String url, boolean stale) { Log.i(TAG, - String.format("download recovered name=%s isStale=%s url=%s", mMission.storage.getName(), stale, url) + String.format("recover() name=%s isStale=%s url=%s", mMission.storage.getName(), stale, url) ); + mMission.urls[mMission.current] = url; + mRecovery.attempts = 0; + if (url == null) { mMission.notifyError(ERROR_RESOURCE_GONE, null); return; } - mMission.urls[mMission.current] = url; - mRecovery.attempts = 0; + if (notInitialized) return; if (stale) { mMission.resetState(false, false, DownloadMission.ERROR_NOTHING); @@ -208,15 +265,40 @@ public class DownloadMissionRecover extends Thread { return range; } + private boolean test() { + if (mMission.urls[mMission.current] == null) return false; + + try { + mConn = mMission.openConnection(mMission.urls[mMission.current], true, -1, -1); + mMission.establishConnection(mID, mConn); + + if (mConn.getResponseCode() == 200) return true; + } catch (Exception e) { + // nothing to do + } finally { + disconnect(); + } + + return false; + } + + private void disconnect() { + try { + try { + mConn.getInputStream().close(); + } finally { + mConn.disconnect(); + } + } catch (Exception e) { + // nothing to do + } finally { + mConn = null; + } + } + @Override public void interrupt() { super.interrupt(); - if (mConn != null) { - try { - mConn.disconnect(); - } catch (Exception e) { - // nothing to do - } - } + if (mConn != null) disconnect(); } } diff --git a/app/src/main/java/us/shandian/giga/get/DownloadRunnable.java b/app/src/main/java/us/shandian/giga/get/DownloadRunnable.java index 1d2a4eee7..b0dc793bc 100644 --- a/app/src/main/java/us/shandian/giga/get/DownloadRunnable.java +++ b/app/src/main/java/us/shandian/giga/get/DownloadRunnable.java @@ -80,7 +80,7 @@ public class DownloadRunnable extends Thread { } try { - mConn = mMission.openConnection(mId, start, end); + mConn = mMission.openConnection(false, start, end); mMission.establishConnection(mId, mConn); // check if the download can be resumed diff --git a/app/src/main/java/us/shandian/giga/get/DownloadRunnableFallback.java b/app/src/main/java/us/shandian/giga/get/DownloadRunnableFallback.java index b5937c577..e64322b48 100644 --- a/app/src/main/java/us/shandian/giga/get/DownloadRunnableFallback.java +++ b/app/src/main/java/us/shandian/giga/get/DownloadRunnableFallback.java @@ -35,7 +35,11 @@ public class DownloadRunnableFallback extends Thread { private void dispose() { try { - if (mIs != null) mIs.close(); + try { + if (mIs != null) mIs.close(); + } finally { + mConn.disconnect(); + } } catch (IOException e) { // nothing to do } @@ -68,7 +72,13 @@ public class DownloadRunnableFallback extends Thread { long rangeStart = (mMission.unknownLength || start < 1) ? -1 : start; int mId = 1; - mConn = mMission.openConnection(mId, rangeStart, -1); + mConn = mMission.openConnection(false, rangeStart, -1); + + if (mRetryCount == 0 && rangeStart == -1) { + // workaround: bypass android connection pool + mConn.setRequestProperty("Range", "bytes=0-"); + } + mMission.establishConnection(mId, mConn); // check if the download can be resumed @@ -96,6 +106,8 @@ public class DownloadRunnableFallback extends Thread { mMission.notifyProgress(len); } + dispose(); + // if thread goes interrupted check if the last part is written. This avoid re-download the whole file done = len == -1; } catch (Exception e) { @@ -107,8 +119,8 @@ public class DownloadRunnableFallback extends Thread { if (e instanceof HttpError && ((HttpError) e).statusCode == ERROR_HTTP_FORBIDDEN) { // for youtube streams. The url has expired, recover - mMission.doRecover(e); dispose(); + mMission.doRecover(e); return; } @@ -125,8 +137,6 @@ public class DownloadRunnableFallback extends Thread { return; } - dispose(); - if (done) { mMission.notifyFinished(); } else { diff --git a/app/src/main/java/us/shandian/giga/get/MissionRecoveryInfo.java b/app/src/main/java/us/shandian/giga/get/MissionRecoveryInfo.java index 553ba6d89..bd1d9bc49 100644 --- a/app/src/main/java/us/shandian/giga/get/MissionRecoveryInfo.java +++ b/app/src/main/java/us/shandian/giga/get/MissionRecoveryInfo.java @@ -16,25 +16,28 @@ public class MissionRecoveryInfo implements Serializable, Parcelable { private static final long serialVersionUID = 0L; //public static final String DIRECT_SOURCE = "direct-source://"; - public MediaFormat format; + MediaFormat format; String desired; boolean desired2; int desiredBitrate; + byte kind; + String validateCondition = null; transient int attempts = 0; - String validateCondition = null; - public MissionRecoveryInfo(@NonNull Stream stream) { if (stream instanceof AudioStream) { desiredBitrate = ((AudioStream) stream).average_bitrate; desired2 = false; + kind = 'a'; } else if (stream instanceof VideoStream) { desired = ((VideoStream) stream).getResolution(); desired2 = ((VideoStream) stream).isVideoOnly(); + kind = 'v'; } else if (stream instanceof SubtitlesStream) { desired = ((SubtitlesStream) stream).getLanguageTag(); desired2 = ((SubtitlesStream) stream).isAutoGenerated(); + kind = 's'; } else { throw new RuntimeException("Unknown stream kind"); } @@ -43,6 +46,38 @@ public class MissionRecoveryInfo implements Serializable, Parcelable { if (format == null) throw new NullPointerException("Stream format cannot be null"); } + @NonNull + @Override + public String toString() { + String info; + StringBuilder str = new StringBuilder(); + str.append("type="); + switch (kind) { + case 'a': + str.append("audio"); + info = "bitrate=" + desiredBitrate; + break; + case 'v': + str.append("video"); + info = "quality=" + desired + " videoOnly=" + desired2; + break; + case 's': + str.append("subtitles"); + info = "language=" + desired + " autoGenerated=" + desired2; + break; + default: + info = ""; + str.append("other"); + } + + str.append(" format=") + .append(format.getName()) + .append(' ') + .append(info); + + return str.toString(); + } + @Override public int describeContents() { return 0; @@ -54,6 +89,7 @@ public class MissionRecoveryInfo implements Serializable, Parcelable { parcel.writeString(this.desired); parcel.writeInt(this.desired2 ? 0x01 : 0x00); parcel.writeInt(this.desiredBitrate); + parcel.writeByte(this.kind); parcel.writeString(this.validateCondition); } @@ -62,6 +98,7 @@ public class MissionRecoveryInfo implements Serializable, Parcelable { this.desired = parcel.readString(); this.desired2 = parcel.readInt() != 0x00; this.desiredBitrate = parcel.readInt(); + this.kind = parcel.readByte(); this.validateCondition = parcel.readString(); } diff --git a/app/src/main/java/us/shandian/giga/ui/adapter/MissionAdapter.java b/app/src/main/java/us/shandian/giga/ui/adapter/MissionAdapter.java index 6c6198750..78fd7ea9d 100644 --- a/app/src/main/java/us/shandian/giga/ui/adapter/MissionAdapter.java +++ b/app/src/main/java/us/shandian/giga/ui/adapter/MissionAdapter.java @@ -36,6 +36,7 @@ import android.widget.Toast; import org.schabi.newpipe.BuildConfig; import org.schabi.newpipe.R; +import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.report.ErrorActivity; import org.schabi.newpipe.report.UserAction; import org.schabi.newpipe.util.NavigationHelper; @@ -44,11 +45,11 @@ import java.io.File; import java.lang.ref.WeakReference; import java.net.URI; import java.util.ArrayList; -import java.util.Collections; import us.shandian.giga.get.DownloadMission; import us.shandian.giga.get.FinishedMission; import us.shandian.giga.get.Mission; +import us.shandian.giga.get.MissionRecoveryInfo; import us.shandian.giga.io.StoredFileHelper; import us.shandian.giga.service.DownloadManager; import us.shandian.giga.service.DownloadManagerService; @@ -234,7 +235,7 @@ public class MissionAdapter extends Adapter implements Handler.Callb // hide on error // show if current resource length is not fetched // show if length is unknown - h.progress.setMarquee(!hasError && (!mission.isInitialized() || mission.unknownLength)); + h.progress.setMarquee(mission.isRecovering() || !hasError && (!mission.isInitialized() || mission.unknownLength)); float progress; if (mission.unknownLength) { @@ -463,13 +464,13 @@ public class MissionAdapter extends Adapter implements Handler.Callb break; case ERROR_POSTPROCESSING: case ERROR_POSTPROCESSING_HOLD: - showError(mission.errObject, UserAction.DOWNLOAD_POSTPROCESSING, R.string.error_postprocessing_failed); + showError(mission, UserAction.DOWNLOAD_POSTPROCESSING, R.string.error_postprocessing_failed); return; case ERROR_INSUFFICIENT_STORAGE: msg = R.string.error_insufficient_storage; break; case ERROR_UNKNOWN_EXCEPTION: - showError(mission.errObject, UserAction.DOWNLOAD_FAILED, R.string.general_error); + showError(mission, UserAction.DOWNLOAD_FAILED, R.string.general_error); return; case ERROR_PROGRESS_LOST: msg = R.string.error_progress_lost; @@ -486,7 +487,7 @@ public class MissionAdapter extends Adapter implements Handler.Callb } else if (mission.errObject == null) { msgEx = "(not_decelerated_error_code)"; } else { - showError(mission.errObject, UserAction.DOWNLOAD_FAILED, msg); + showError(mission, UserAction.DOWNLOAD_FAILED, msg); return; } break; @@ -503,7 +504,7 @@ public class MissionAdapter extends Adapter implements Handler.Callb if (mission.errObject != null && (mission.errCode < 100 || mission.errCode >= 600)) { @StringRes final int mMsg = msg; builder.setPositiveButton(R.string.error_report_title, (dialog, which) -> - showError(mission.errObject, UserAction.DOWNLOAD_FAILED, mMsg) + showError(mission, UserAction.DOWNLOAD_FAILED, mMsg) ); } @@ -513,13 +514,30 @@ public class MissionAdapter extends Adapter implements Handler.Callb .show(); } - private void showError(Exception exception, UserAction action, @StringRes int reason) { + private void showError(DownloadMission mission, UserAction action, @StringRes int reason) { + StringBuilder request = new StringBuilder(256); + request.append(mission.source); + + request.append(" ["); + if (mission.recoveryInfo != null) { + for (MissionRecoveryInfo recovery : mission.recoveryInfo) + request.append(" {").append(recovery.toString()).append("} "); + } + request.append("]"); + + String service; + try { + service = NewPipe.getServiceByUrl(mission.source).getServiceInfo().getName(); + } catch (Exception e) { + service = "-"; + } + ErrorActivity.reportError( mContext, - Collections.singletonList(exception), + mission.errObject, null, null, - ErrorActivity.ErrorInfo.make(action, "-", "-", reason) + ErrorActivity.ErrorInfo.make(action, service, request.toString(), reason) ); } diff --git a/app/src/main/res/values-es/strings.xml b/app/src/main/res/values-es/strings.xml index 2f69e62cb..b14aab94b 100644 --- a/app/src/main/res/values-es/strings.xml +++ b/app/src/main/res/values-es/strings.xml @@ -453,7 +453,7 @@ No hay suficiente espacio disponible en el dispositivo Se perdió el progreso porque el archivo fue eliminado Tiempo de espera excedido - El recurso solicitado ya no esta disponible + No se puede recuperar esta descarga Preguntar dónde descargar Se preguntará dónde guardar cada descarga Se le preguntará dónde guardar cada descarga. diff --git a/app/src/main/res/values/strings.xml b/app/src/main/res/values/strings.xml index 2917fb9fd..f929e0d2b 100644 --- a/app/src/main/res/values/strings.xml +++ b/app/src/main/res/values/strings.xml @@ -557,7 +557,7 @@ No space left on device Progress lost, because the file was deleted Connection timeout - The solicited resource is not available anymore + Cannot recover this download Clear finished downloads Are you sure? Continue your %s pending transfers from Downloads From d092e39c56900358b3caa1897d22df18fafad8da Mon Sep 17 00:00:00 2001 From: kapodamy Date: Tue, 1 Oct 2019 13:00:16 -0300 Subject: [PATCH 08/28] fallback for pending downloads directory --- .../giga/service/DownloadManager.java | 31 +++++++++++++------ 1 file changed, 21 insertions(+), 10 deletions(-) diff --git a/app/src/main/java/us/shandian/giga/service/DownloadManager.java b/app/src/main/java/us/shandian/giga/service/DownloadManager.java index a859a87ca..89c44638d 100644 --- a/app/src/main/java/us/shandian/giga/service/DownloadManager.java +++ b/app/src/main/java/us/shandian/giga/service/DownloadManager.java @@ -37,6 +37,7 @@ public class DownloadManager { public static final String TAG_AUDIO = "audio"; public static final String TAG_VIDEO = "video"; + private static final String DOWNLOADS_METADATA_FOLDER = "pending_downloads"; private final FinishedMissionStore mFinishedMissionStore; @@ -75,24 +76,33 @@ public class DownloadManager { mPendingMissionsDir = getPendingDir(context); if (!Utility.mkdir(mPendingMissionsDir, false)) { - throw new RuntimeException("failed to create pending_downloads in data directory"); + throw new RuntimeException("failed to create " + DOWNLOADS_METADATA_FOLDER + " directory"); } loadPendingMissions(context); } private static File getPendingDir(@NonNull Context context) { - //File dir = new File(ContextCompat.getDataDir(context), "pending_downloads"); - File dir = context.getExternalFilesDir("pending_downloads"); + File dir = context.getExternalFilesDir(DOWNLOADS_METADATA_FOLDER); + if (testDir(dir)) return dir; - if (dir == null) { - // One of the following paths are not accessible ¿unmounted internal memory? - // /storage/emulated/0/Android/data/org.schabi.newpipe[.debug]/pending_downloads - // /sdcard/Android/data/org.schabi.newpipe[.debug]/pending_downloads - Log.w(TAG, "path to pending downloads are not accessible"); + dir = new File(context.getFilesDir(), DOWNLOADS_METADATA_FOLDER); + if (testDir(dir)) return dir; + + throw new RuntimeException("path to pending downloads are not accessible"); + } + + private static boolean testDir(@Nullable File dir) { + if (dir == null) return false; + + try { + File tmp = new File(dir, ".tmp"); + if (!tmp.createNewFile()) return false; + return tmp.delete();// if the file was created, SHOULD BE deleted too + } catch (Exception e) { + Log.e(TAG, "testDir() failed: " + dir.getAbsolutePath(), e); + return false; } - - return dir; } /** @@ -132,6 +142,7 @@ public class DownloadManager { for (File sub : subs) { if (!sub.isFile()) continue; + if (sub.getName().equals(".tmp")) continue; DownloadMission mis = Utility.readFromFile(sub); if (mis == null || mis.isFinished()) { From 9339fc80b4f40c62df65c04de328fc4fd9ae964c Mon Sep 17 00:00:00 2001 From: kapodamy Date: Tue, 1 Oct 2019 15:01:17 -0300 Subject: [PATCH 09/28] update DownloadManager.java * check if the directory pending_downloads was created --- .../java/us/shandian/giga/service/DownloadManager.java | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/app/src/main/java/us/shandian/giga/service/DownloadManager.java b/app/src/main/java/us/shandian/giga/service/DownloadManager.java index 89c44638d..2d1e9cd00 100644 --- a/app/src/main/java/us/shandian/giga/service/DownloadManager.java +++ b/app/src/main/java/us/shandian/giga/service/DownloadManager.java @@ -75,10 +75,6 @@ public class DownloadManager { mMissionsFinished = loadFinishedMissions(); mPendingMissionsDir = getPendingDir(context); - if (!Utility.mkdir(mPendingMissionsDir, false)) { - throw new RuntimeException("failed to create " + DOWNLOADS_METADATA_FOLDER + " directory"); - } - loadPendingMissions(context); } @@ -96,6 +92,11 @@ public class DownloadManager { if (dir == null) return false; try { + if (!Utility.mkdir(dir, false)) { + Log.e(TAG, "testDir() cannot create the directory in path: " + dir.getAbsolutePath()); + return false; + } + File tmp = new File(dir, ".tmp"); if (!tmp.createNewFile()) return false; return tmp.delete();// if the file was created, SHOULD BE deleted too From 94e23142a5cfcca74e0345c76ecccfc54a87c450 Mon Sep 17 00:00:00 2001 From: kapodamy Date: Tue, 1 Oct 2019 16:28:45 -0300 Subject: [PATCH 10/28] update WebMWriter.java fix wrong cue generation --- app/src/main/java/org/schabi/newpipe/streams/WebMWriter.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/src/main/java/org/schabi/newpipe/streams/WebMWriter.java b/app/src/main/java/org/schabi/newpipe/streams/WebMWriter.java index 1bf994b1e..8525fabd2 100644 --- a/app/src/main/java/org/schabi/newpipe/streams/WebMWriter.java +++ b/app/src/main/java/org/schabi/newpipe/streams/WebMWriter.java @@ -249,7 +249,7 @@ public class WebMWriter implements Closeable { nextCueTime += DEFAULT_CUES_EACH_MS; } keyFrames.add( - new KeyFrame(baseSegmentOffset, currentClusterOffset - 7, written, bTimecode.length, bloq.absoluteTimecode) + new KeyFrame(baseSegmentOffset, currentClusterOffset - 8, written, bTimecode.length, bloq.absoluteTimecode) ); } } From 844f80a5f1b0762a043afd2b4aec63b402830e53 Mon Sep 17 00:00:00 2001 From: kapodamy Date: Wed, 2 Oct 2019 13:31:45 -0300 Subject: [PATCH 11/28] update DownloadDialog.java keep *.opus extension --- .../main/java/org/schabi/newpipe/download/DownloadDialog.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/src/main/java/org/schabi/newpipe/download/DownloadDialog.java b/app/src/main/java/org/schabi/newpipe/download/DownloadDialog.java index 0006b3c12..60b6192be 100644 --- a/app/src/main/java/org/schabi/newpipe/download/DownloadDialog.java +++ b/app/src/main/java/org/schabi/newpipe/download/DownloadDialog.java @@ -562,7 +562,7 @@ public class DownloadDialog extends DialogFragment implements RadioGroup.OnCheck mainStorage = mainStorageVideo; format = videoStreamsAdapter.getItem(selectedVideoIndex).getFormat(); mime = format.mimeType; - filename += format == MediaFormat.OPUS ? "ogg" : format.suffix; + filename += format.suffix; break; case R.id.subtitle_button: mainStorage = mainStorageVideo;// subtitle & video files go together From f62a7919a5f592bfe62396dccd054150094498d1 Mon Sep 17 00:00:00 2001 From: kapodamy Date: Wed, 9 Oct 2019 23:49:23 -0300 Subject: [PATCH 12/28] code cleanup * migrate few annotations to androidx * mission recovery: better error handling (except StreamExtractor.getErrorMessage() method always returns an error) * post-processing: more detailed progress [file specific changes] DownloadMission.java * remove redundant/boilerplate code (again) * make few variables volatile * better file "length" approximation * use "done" variable to count the amount of bytes downloaded (simplify percent calc in UI code) Postprocessing.java * if case of error use "ERROR_POSTPROCESSING" instead of "ERROR_UNKNOWN_EXCEPTION" * simplify source stream init DownloadManager.java * move all "service message sending" code to DownloadMission * remove not implemented method "notifyUserPendingDownloads()" also his unused strings DownloadManagerService.java * use START_STICKY instead of START_NOT_STICKY * simplify addMissionEventListener()/removeMissionEventListener() methods (always are called from the main thread) Deleter.java * better method definition MissionAdapter.java * better method definition * code cleanup * the UI is now refreshed every 750ms * simplify download progress calculation * indicates if the download is actually recovering * smooth download speed measure * show estimated remain time MainFragment.java: * check if viewPager is null (issued by "Apply changes" feature of Android Studio) --- .../newpipe/fragments/MainFragment.java | 9 + .../newpipe/streams/OggFromWebMWriter.java | 2 +- .../giga/get/DownloadInitializer.java | 5 +- .../us/shandian/giga/get/DownloadMission.java | 287 ++++++++--------- .../giga/get/DownloadMissionRecover.java | 160 +++++----- .../shandian/giga/get/DownloadRunnable.java | 5 +- .../giga/get/DownloadRunnableFallback.java | 29 +- .../us/shandian/giga/get/FinishedMission.java | 6 +- .../giga/get/MissionRecoveryInfo.java | 10 +- .../giga/io/ChunkFileInputStream.java | 19 +- .../shandian/giga/io/CircularFileWriter.java | 30 +- .../us/shandian/giga/io/ProgressReport.java | 11 + .../postprocessing/OggFromWebmDemuxer.java | 2 +- .../giga/postprocessing/Postprocessing.java | 54 ++-- .../giga/service/DownloadManager.java | 57 +--- .../giga/service/DownloadManagerService.java | 45 ++- .../giga/ui/adapter/MissionAdapter.java | 302 +++++++++--------- .../us/shandian/giga/ui/common/Deleter.java | 9 +- .../giga/ui/common/ProgressDrawable.java | 5 +- .../giga/ui/fragment/MissionsFragment.java | 45 ++- .../java/us/shandian/giga/util/Utility.java | 50 ++- app/src/main/res/values-ar/strings.xml | 1 - app/src/main/res/values-be/strings.xml | 1 - app/src/main/res/values-cmn/strings.xml | 1 - app/src/main/res/values-cs/strings.xml | 1 - app/src/main/res/values-da/strings.xml | 1 - app/src/main/res/values-de/strings.xml | 1 - app/src/main/res/values-el/strings.xml | 1 - app/src/main/res/values-es/strings.xml | 2 +- app/src/main/res/values-et/strings.xml | 1 - app/src/main/res/values-eu/strings.xml | 1 - app/src/main/res/values-fr/strings.xml | 1 - app/src/main/res/values-he/strings.xml | 1 - app/src/main/res/values-hr/strings.xml | 1 - app/src/main/res/values-id/strings.xml | 1 - app/src/main/res/values-it/strings.xml | 1 - app/src/main/res/values-ja/strings.xml | 1 - app/src/main/res/values-ko/strings.xml | 1 - app/src/main/res/values-ms/strings.xml | 1 - app/src/main/res/values-nb-rNO/strings.xml | 1 - app/src/main/res/values-nl-rBE/strings.xml | 1 - app/src/main/res/values-nl/strings.xml | 1 - app/src/main/res/values-pa/strings.xml | 1 - app/src/main/res/values-pl/strings.xml | 1 - app/src/main/res/values-pt-rBR/strings.xml | 1 - app/src/main/res/values-pt/strings.xml | 1 - app/src/main/res/values-ru/strings.xml | 1 - app/src/main/res/values-sk/strings.xml | 1 - app/src/main/res/values-tr/strings.xml | 1 - app/src/main/res/values-uk/strings.xml | 1 - app/src/main/res/values-vi/strings.xml | 1 - app/src/main/res/values-zh-rTW/strings.xml | 1 - app/src/main/res/values/strings.xml | 2 +- 53 files changed, 554 insertions(+), 622 deletions(-) create mode 100644 app/src/main/java/us/shandian/giga/io/ProgressReport.java diff --git a/app/src/main/java/org/schabi/newpipe/fragments/MainFragment.java b/app/src/main/java/org/schabi/newpipe/fragments/MainFragment.java index 720e0f216..70e0d9fb1 100644 --- a/app/src/main/java/org/schabi/newpipe/fragments/MainFragment.java +++ b/app/src/main/java/org/schabi/newpipe/fragments/MainFragment.java @@ -2,6 +2,15 @@ package org.schabi.newpipe.fragments; import android.content.Context; import android.os.Bundle; +import androidx.annotation.NonNull; +import androidx.annotation.Nullable; +import com.google.android.material.tabs.TabLayout; +import androidx.fragment.app.Fragment; +import androidx.fragment.app.FragmentManager; +import androidx.fragment.app.FragmentPagerAdapter; +import androidx.viewpager.widget.ViewPager; +import androidx.appcompat.app.ActionBar; +import androidx.appcompat.app.AppCompatActivity; import android.util.Log; import android.view.LayoutInflater; import android.view.Menu; diff --git a/app/src/main/java/org/schabi/newpipe/streams/OggFromWebMWriter.java b/app/src/main/java/org/schabi/newpipe/streams/OggFromWebMWriter.java index e6363e423..37bf9c6d7 100644 --- a/app/src/main/java/org/schabi/newpipe/streams/OggFromWebMWriter.java +++ b/app/src/main/java/org/schabi/newpipe/streams/OggFromWebMWriter.java @@ -1,6 +1,6 @@ package org.schabi.newpipe.streams; -import android.support.annotation.NonNull; +import androidx.annotation.NonNull; import org.schabi.newpipe.streams.WebMReader.Cluster; import org.schabi.newpipe.streams.WebMReader.Segment; diff --git a/app/src/main/java/us/shandian/giga/get/DownloadInitializer.java b/app/src/main/java/us/shandian/giga/get/DownloadInitializer.java index 17a2a7403..618200f27 100644 --- a/app/src/main/java/us/shandian/giga/get/DownloadInitializer.java +++ b/app/src/main/java/us/shandian/giga/get/DownloadInitializer.java @@ -1,9 +1,10 @@ package us.shandian.giga.get; -import androidx.annotation.NonNull; import android.text.TextUtils; import android.util.Log; +import androidx.annotation.NonNull; + import org.schabi.newpipe.streams.io.SharpStream; import java.io.IOException; @@ -177,7 +178,7 @@ public class DownloadInitializer extends Thread { if (e instanceof DownloadMission.HttpError && ((DownloadMission.HttpError) e).statusCode == ERROR_HTTP_FORBIDDEN) { // for youtube streams. The url has expired interrupt(); - mMission.doRecover(e); + mMission.doRecover(ERROR_HTTP_FORBIDDEN); return; } diff --git a/app/src/main/java/us/shandian/giga/get/DownloadMission.java b/app/src/main/java/us/shandian/giga/get/DownloadMission.java index 918d6dbea..5ef72162c 100644 --- a/app/src/main/java/us/shandian/giga/get/DownloadMission.java +++ b/app/src/main/java/us/shandian/giga/get/DownloadMission.java @@ -4,18 +4,21 @@ import android.os.Handler; import android.util.Log; import androidx.annotation.Nullable; +import androidx.annotation.NonNull; import org.schabi.newpipe.DownloaderImpl; import java.io.File; import java.io.FileNotFoundException; import java.io.IOException; +import java.io.InterruptedIOException; import java.io.Serializable; import java.net.ConnectException; import java.net.HttpURLConnection; import java.net.SocketTimeoutException; import java.net.URL; import java.net.UnknownHostException; +import java.nio.channels.ClosedByInterruptException; import javax.net.ssl.SSLException; @@ -27,7 +30,7 @@ import us.shandian.giga.util.Utility; import static org.schabi.newpipe.BuildConfig.DEBUG; public class DownloadMission extends Mission { - private static final long serialVersionUID = 6L;// last bump: 28 september 2019 + private static final long serialVersionUID = 6L;// last bump: 07 october 2019 static final int BUFFER_SIZE = 64 * 1024; static final int BLOCK_SIZE = 512 * 1024; @@ -61,9 +64,9 @@ public class DownloadMission extends Mission { public String[] urls; /** - * Number of bytes downloaded + * Number of bytes downloaded and written */ - public long done; + public volatile long done; /** * Indicates a file generated dynamically on the web server @@ -119,7 +122,7 @@ public class DownloadMission extends Mission { /** * Download/File resume offset in fallback mode (if applicable) {@link DownloadRunnableFallback} */ - long fallbackResumeOffset; + volatile long fallbackResumeOffset; /** * Maximum of download threads running, chosen by the user @@ -132,22 +135,23 @@ public class DownloadMission extends Mission { public MissionRecoveryInfo[] recoveryInfo; private transient int finishCount; - public transient boolean running; + public transient volatile boolean running; public boolean enqueued; public int errCode = ERROR_NOTHING; public Exception errObject = null; public transient Handler mHandler; - private transient boolean mWritingToFile; private transient boolean[] blockAcquired; + private transient long writingToFileNext; + private transient volatile boolean writingToFile; + final Object LOCK = new Lock(); - private transient boolean deleted; - - public transient volatile Thread[] threads = new Thread[0]; - private transient Thread init = null; + @NonNull + public transient Thread[] threads = new Thread[0]; + public transient Thread init = null; public DownloadMission(String[] urls, StoredFileHelper storage, char kind, Postprocessing psInstance) { if (urls == null) throw new NullPointerException("urls is null"); @@ -246,8 +250,10 @@ public class DownloadMission extends Mission { int statusCode = conn.getResponseCode(); if (DEBUG) { - Log.d(TAG, threadId + ":Range=" + conn.getRequestProperty("Range")); - Log.d(TAG, threadId + ":Content-Length=" + conn.getContentLength() + " Code:" + statusCode); + Log.d(TAG, threadId + ":[request] Range=" + conn.getRequestProperty("Range")); + Log.d(TAG, threadId + ":[response] Code=" + statusCode); + Log.d(TAG, threadId + ":[response] Content-Length=" + conn.getContentLength()); + Log.d(TAG, threadId + ":[response] Content-Range=" + conn.getHeaderField("Content-Range")); } @@ -272,24 +278,19 @@ public class DownloadMission extends Mission { } synchronized void notifyProgress(long deltaLen) { - if (!running) return; - if (unknownLength) { length += deltaLen;// Update length before proceeding } done += deltaLen; - if (done > length) { - done = length; - } + if (metadata == null) return; - if (done != length && !deleted && !mWritingToFile) { - mWritingToFile = true; - runAsync(-2, this::writeThisToFile); + if (!writingToFile && (done > writingToFileNext || deltaLen < 0)) { + writingToFile = true; + writingToFileNext = done + BLOCK_SIZE; + writeThisToFileAsync(); } - - notify(DownloadManagerService.MESSAGE_PROGRESS); } synchronized void notifyError(Exception err) { @@ -342,43 +343,42 @@ public class DownloadMission extends Mission { notify(DownloadManagerService.MESSAGE_ERROR); - if (running) { - running = false; - if (threads != null) selfPause(); - } + if (running) pauseThreads(); } synchronized void notifyFinished() { - if (errCode > ERROR_NOTHING) return; - - finishCount++; - - if (blocks.length < 1 || threads == null || finishCount == threads.length) { - if (errCode != ERROR_NOTHING) return; + if (current < urls.length) { + if (++finishCount < threads.length) return; if (DEBUG) { - Log.d(TAG, "onFinish: " + (current + 1) + "/" + urls.length); - } - - if ((current + 1) < urls.length) { - // prepare next sub-mission - long current_offset = offsets[current++]; - offsets[current] = current_offset + length; - initializer(); - return; + Log.d(TAG, "onFinish: downloaded " + (current + 1) + "/" + urls.length); } current++; - unknownLength = false; - - if (!doPostprocessing()) return; - - enqueued = false; - running = false; - deleteThisFromFile(); - - notify(DownloadManagerService.MESSAGE_FINISHED); + if (current < urls.length) { + // prepare next sub-mission + offsets[current] = offsets[current - 1] + length; + initializer(); + return; + } } + + if (psAlgorithm != null && psState == 0) { + threads = new Thread[]{ + runAsync(1, this::doPostprocessing) + }; + return; + } + + + // this mission is fully finished + + unknownLength = false; + enqueued = false; + running = false; + + deleteThisFromFile(); + notify(DownloadManagerService.MESSAGE_FINISHED); } private void notifyPostProcessing(int state) { @@ -396,10 +396,15 @@ public class DownloadMission extends Mission { Log.d(TAG, action + " postprocessing on " + storage.getName()); + if (state == 2) { + psState = state; + return; + } + synchronized (LOCK) { // don't return without fully write the current state psState = state; - Utility.writeToFile(metadata, DownloadMission.this); + writeThisToFile(); } } @@ -411,12 +416,7 @@ public class DownloadMission extends Mission { if (running || isFinished() || urls.length < 1) return; // ensure that the previous state is completely paused. - int maxWait = 10000;// 10 seconds - joinForThread(init, maxWait); - if (threads != null) { - for (Thread thread : threads) joinForThread(thread, maxWait); - threads = null; - } + joinForThreads(10000); running = true; errCode = ERROR_NOTHING; @@ -427,12 +427,14 @@ public class DownloadMission extends Mission { } if (current >= urls.length) { - runAsync(1, this::notifyFinished); + notifyFinished(); return; } + notify(DownloadManagerService.MESSAGE_RUNNING); + if (urls[current] == null) { - doRecover(null); + doRecover(ERROR_RESOURCE_GONE); return; } @@ -446,18 +448,13 @@ public class DownloadMission extends Mission { blockAcquired = new boolean[blocks.length]; if (blocks.length < 1) { - if (unknownLength) { - done = 0; - length = 0; - } - threads = new Thread[]{runAsync(1, new DownloadRunnableFallback(this))}; } else { int remainingBlocks = 0; for (int block : blocks) if (block >= 0) remainingBlocks++; if (remainingBlocks < 1) { - runAsync(1, this::notifyFinished); + notifyFinished(); return; } @@ -483,6 +480,7 @@ public class DownloadMission extends Mission { } running = false; + notify(DownloadManagerService.MESSAGE_PAUSED); if (init != null && init.isAlive()) { // NOTE: if start() method is running ¡will no have effect! @@ -497,29 +495,14 @@ public class DownloadMission extends Mission { Log.w(TAG, "pausing a download that can not be resumed (range requests not allowed by the server)."); } - // check if the calling thread (alias UI thread) is interrupted - if (Thread.currentThread().isInterrupted()) { - writeThisToFile(); - return; - } - - // wait for all threads are suspended before save the state - if (threads != null) runAsync(-1, this::selfPause); + init = null; + pauseThreads(); } - private void selfPause() { - try { - for (Thread thread : threads) { - if (thread.isAlive()) { - thread.interrupt(); - thread.join(5000); - } - } - } catch (Exception e) { - // nothing to do - } finally { - writeThisToFile(); - } + private void pauseThreads() { + running = false; + joinForThreads(-1); + writeThisToFile(); } /** @@ -527,9 +510,10 @@ public class DownloadMission extends Mission { */ @Override public boolean delete() { - deleted = true; if (psAlgorithm != null) psAlgorithm.cleanupTemporalDir(); + notify(DownloadManagerService.MESSAGE_DELETED); + boolean res = deleteThisFromFile(); if (!super.delete()) return false; @@ -544,35 +528,37 @@ public class DownloadMission extends Mission { * @param persistChanges {@code true} to commit changes to the metadata file, otherwise, {@code false} */ public void resetState(boolean rollback, boolean persistChanges, int errorCode) { - done = 0; + length = 0; errCode = errorCode; errObject = null; unknownLength = false; - threads = null; + threads = new Thread[0]; fallbackResumeOffset = 0; blocks = null; blockAcquired = null; if (rollback) current = 0; - - if (persistChanges) - Utility.writeToFile(metadata, DownloadMission.this); + if (persistChanges) writeThisToFile(); } private void initializer() { init = runAsync(DownloadInitializer.mId, new DownloadInitializer(this)); } + private void writeThisToFileAsync() { + runAsync(-2, this::writeThisToFile); + } + /** * Write this {@link DownloadMission} to the meta file asynchronously * if no thread is already running. */ void writeThisToFile() { synchronized (LOCK) { - if (deleted) return; - Utility.writeToFile(metadata, DownloadMission.this); + if (metadata == null) return; + Utility.writeToFile(metadata, this); + writingToFile = false; } - mWritingToFile = false; } /** @@ -625,11 +611,10 @@ public class DownloadMission extends Mission { public long getLength() { long calculated; if (psState == 1 || psState == 3) { - calculated = length; - } else { - calculated = offsets[current < offsets.length ? current : (offsets.length - 1)] + length; + return length; } + calculated = offsets[current < offsets.length ? current : (offsets.length - 1)] + length; calculated -= offsets[0];// don't count reserved space return calculated > nearLength ? calculated : nearLength; @@ -642,7 +627,7 @@ public class DownloadMission extends Mission { */ public void setEnqueued(boolean queue) { enqueued = queue; - runAsync(-2, this::writeThisToFile); + writeThisToFileAsync(); } /** @@ -681,24 +666,19 @@ public class DownloadMission extends Mission { * @return {@code true} if the mission is running a recovery procedure, otherwise, {@code false} */ public boolean isRecovering() { - return threads != null && threads.length > 0 && threads[0] instanceof DownloadRunnable && threads[0].isAlive(); + return threads.length > 0 && threads[0] instanceof DownloadMissionRecover && threads[0].isAlive(); } - private boolean doPostprocessing() { - if (psAlgorithm == null || psState == 2) return true; - + private void doPostprocessing() { + errCode = ERROR_NOTHING; errObject = null; + Thread thread = Thread.currentThread(); notifyPostProcessing(1); - notifyProgress(0); - if (DEBUG) - Thread.currentThread().setName("[" + TAG + "] ps = " + - psAlgorithm.getClass().getSimpleName() + - " filename = " + storage.getName() - ); - - threads = new Thread[]{Thread.currentThread()}; + if (DEBUG) { + thread.setName("[" + TAG + "] ps = " + psAlgorithm + " filename = " + storage.getName()); + } Exception exception = null; @@ -707,6 +687,11 @@ public class DownloadMission extends Mission { } catch (Exception err) { Log.e(TAG, "Post-processing failed. " + psAlgorithm.toString(), err); + if (err instanceof InterruptedIOException || err instanceof ClosedByInterruptException || thread.isInterrupted()) { + notifyError(DownloadMission.ERROR_POSTPROCESSING_STOPPED, null); + return; + } + if (errCode == ERROR_NOTHING) errCode = ERROR_POSTPROCESSING; exception = err; @@ -717,56 +702,38 @@ public class DownloadMission extends Mission { if (errCode != ERROR_NOTHING) { if (exception == null) exception = errObject; notifyError(ERROR_POSTPROCESSING, exception); - - return false; + return; } - return true; + notifyFinished(); } /** * Attempts to recover the download * - * @param fromError exception which require update the url from the source + * @param errorCode error code which trigger the recovery procedure */ - void doRecover(Exception fromError) { + void doRecover(int errorCode) { Log.i(TAG, "Attempting to recover the mission: " + storage.getName()); if (recoveryInfo == null) { - if (fromError == null) - notifyError(ERROR_RESOURCE_GONE, null); - else - notifyError(fromError); - + notifyError(errorCode, null); urls = new String[0];// mark this mission as dead return; } - if (threads != null) { - for (Thread thread : threads) { - if (thread == Thread.currentThread()) continue; - thread.interrupt(); - joinForThread(thread, 0); - } - } - - errCode = ERROR_NOTHING; - errObject = null; - - if (recoveryInfo[current].attempts >= maxRetry) { - recoveryInfo[current].attempts = 0; - notifyError(fromError); - return; - } + joinForThreads(0); threads = new Thread[]{ - runAsync(DownloadMissionRecover.mID, new DownloadMissionRecover(this, fromError)) + runAsync(DownloadMissionRecover.mID, new DownloadMissionRecover(this, errorCode)) }; } private boolean deleteThisFromFile() { synchronized (LOCK) { - return metadata.delete(); + boolean res = metadata.delete(); + metadata = null; + return res; } } @@ -776,8 +743,8 @@ public class DownloadMission extends Mission { * @param id id of new thread (used for debugging only) * @param who the Runnable whose {@code run} method is invoked. */ - private void runAsync(int id, Runnable who) { - runAsync(id, new Thread(who)); + private Thread runAsync(int id, Runnable who) { + return runAsync(id, new Thread(who)); } /** @@ -806,28 +773,44 @@ public class DownloadMission extends Mission { /** * Waits at most {@code millis} milliseconds for the thread to die * - * @param thread the desired thread * @param millis the time to wait in milliseconds */ - private void joinForThread(Thread thread, int millis) { - if (thread == null || !thread.isAlive()) return; - if (thread == Thread.currentThread()) return; + private void joinForThreads(int millis) { + final Thread currentThread = Thread.currentThread(); - if (DEBUG) { - Log.w(TAG, "a thread is !still alive!: " + thread.getName()); + if (init != null && init != currentThread && init.isAlive()) { + init.interrupt(); + + if (millis > 0) { + try { + init.join(millis); + } catch (InterruptedException e) { + Log.w(TAG, "Initializer thread is still running", e); + return; + } + } } - // still alive, this should not happen. - // Possible reasons: + // if a thread is still alive, possible reasons: // slow device // the user is spamming start/pause buttons // start() method called quickly after pause() + for (Thread thread : threads) { + if (!thread.isAlive() || thread == Thread.currentThread()) continue; + thread.interrupt(); + } + try { - thread.join(millis); + for (Thread thread : threads) { + if (!thread.isAlive()) continue; + if (DEBUG) { + Log.w(TAG, "thread alive: " + thread.getName()); + } + if (millis > 0) thread.join(millis); + } } catch (InterruptedException e) { - Log.d(TAG, "timeout on join : " + thread.getName()); - throw new RuntimeException("A thread is still running:\n" + thread.getName()); + throw new RuntimeException("A download thread is still running", e); } } diff --git a/app/src/main/java/us/shandian/giga/get/DownloadMissionRecover.java b/app/src/main/java/us/shandian/giga/get/DownloadMissionRecover.java index 5efbd1153..eb660e564 100644 --- a/app/src/main/java/us/shandian/giga/get/DownloadMissionRecover.java +++ b/app/src/main/java/us/shandian/giga/get/DownloadMissionRecover.java @@ -4,6 +4,7 @@ import android.util.Log; import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.StreamingService; +import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.stream.AudioStream; import org.schabi.newpipe.extractor.stream.StreamExtractor; import org.schabi.newpipe.extractor.stream.SubtitlesStream; @@ -15,7 +16,8 @@ import java.net.HttpURLConnection; import java.nio.channels.ClosedByInterruptException; import java.util.List; -import static us.shandian.giga.get.DownloadMission.ERROR_NOTHING; +import us.shandian.giga.get.DownloadMission.HttpError; + import static us.shandian.giga.get.DownloadMission.ERROR_RESOURCE_GONE; public class DownloadMissionRecover extends Thread { @@ -23,47 +25,67 @@ public class DownloadMissionRecover extends Thread { static final int mID = -3; private final DownloadMission mMission; - private final Exception mFromError; - private final boolean notInitialized; + private final boolean mNotInitialized; + + private final int mErrCode; private HttpURLConnection mConn; private MissionRecoveryInfo mRecovery; private StreamExtractor mExtractor; - DownloadMissionRecover(DownloadMission mission, Exception originError) { + DownloadMissionRecover(DownloadMission mission, int errCode) { mMission = mission; - mFromError = originError; - notInitialized = mission.blocks == null && mission.current == 0; + mNotInitialized = mission.blocks == null && mission.current == 0; + mErrCode = errCode; } @Override public void run() { if (mMission.source == null) { - mMission.notifyError(mFromError); + mMission.notifyError(mErrCode, null); return; } + Exception err = null; + int attempt = 0; + + while (attempt++ < mMission.maxRetry) { + try { + tryRecover(); + return; + } catch (InterruptedIOException | ClosedByInterruptException e) { + return; + } catch (Exception e) { + if (!mMission.running || super.isInterrupted()) return; + err = e; + } + } + + // give up + mMission.notifyError(mErrCode, err); + } + + private void tryRecover() throws ExtractionException, IOException, HttpError { /*if (mMission.source.startsWith(MissionRecoveryInfo.DIRECT_SOURCE)) { resolve(mMission.source.substring(MissionRecoveryInfo.DIRECT_SOURCE.length())); return; }*/ - try { - StreamingService svr = NewPipe.getServiceByUrl(mMission.source); - mExtractor = svr.getStreamExtractor(mMission.source); - mExtractor.fetchPage(); - } catch (InterruptedIOException | ClosedByInterruptException e) { - return; - } catch (Exception e) { - if (!mMission.running || super.isInterrupted()) return; - mMission.notifyError(e); - return; + if (mExtractor == null) { + try { + StreamingService svr = NewPipe.getServiceByUrl(mMission.source); + mExtractor = svr.getStreamExtractor(mMission.source); + mExtractor.fetchPage(); + } catch (ExtractionException e) { + mExtractor = null; + throw e; + } } // maybe the following check is redundant if (!mMission.running || super.isInterrupted()) return; - if (!notInitialized) { + if (!mNotInitialized) { // set the current download url to null in case if the recovery // process is canceled. Next time start() method is called the // recovery will be executed, saving time @@ -87,7 +109,7 @@ public class DownloadMissionRecover extends Thread { if (!mMission.running) return; // before continue, check if the current stream was resolved - if (mMission.urls[mMission.current] == null || mMission.errCode != ERROR_NOTHING) { + if (mMission.urls[mMission.current] == null) { break; } } @@ -103,59 +125,54 @@ public class DownloadMissionRecover extends Thread { mMission.start(); } - private void resolveStream() { - if (mExtractor.getErrorMessage() != null) { - mMission.notifyError(mFromError); + private void resolveStream() throws IOException, ExtractionException, HttpError { + // FIXME: this getErrorMessage() always returns "video is unavailable" + /*if (mExtractor.getErrorMessage() != null) { + mMission.notifyError(mErrCode, new ExtractionException(mExtractor.getErrorMessage())); return; + }*/ + + String url = null; + + switch (mRecovery.kind) { + case 'a': + for (AudioStream audio : mExtractor.getAudioStreams()) { + if (audio.average_bitrate == mRecovery.desiredBitrate && audio.getFormat() == mRecovery.format) { + url = audio.getUrl(); + break; + } + } + break; + case 'v': + List videoStreams; + if (mRecovery.desired2) + videoStreams = mExtractor.getVideoOnlyStreams(); + else + videoStreams = mExtractor.getVideoStreams(); + for (VideoStream video : videoStreams) { + if (video.resolution.equals(mRecovery.desired) && video.getFormat() == mRecovery.format) { + url = video.getUrl(); + break; + } + } + break; + case 's': + for (SubtitlesStream subtitles : mExtractor.getSubtitles(mRecovery.format)) { + String tag = subtitles.getLanguageTag(); + if (tag.equals(mRecovery.desired) && subtitles.isAutoGenerated() == mRecovery.desired2) { + url = subtitles.getURL(); + break; + } + } + break; + default: + throw new RuntimeException("Unknown stream type"); } - try { - String url = null; - - switch (mRecovery.kind) { - case 'a': - for (AudioStream audio : mExtractor.getAudioStreams()) { - if (audio.average_bitrate == mRecovery.desiredBitrate && audio.getFormat() == mRecovery.format) { - url = audio.getUrl(); - break; - } - } - break; - case 'v': - List videoStreams; - if (mRecovery.desired2) - videoStreams = mExtractor.getVideoOnlyStreams(); - else - videoStreams = mExtractor.getVideoStreams(); - for (VideoStream video : videoStreams) { - if (video.resolution.equals(mRecovery.desired) && video.getFormat() == mRecovery.format) { - url = video.getUrl(); - break; - } - } - break; - case 's': - for (SubtitlesStream subtitles : mExtractor.getSubtitles(mRecovery.format)) { - String tag = subtitles.getLanguageTag(); - if (tag.equals(mRecovery.desired) && subtitles.isAutoGenerated() == mRecovery.desired2) { - url = subtitles.getURL(); - break; - } - } - break; - default: - throw new RuntimeException("Unknown stream type"); - } - - resolve(url); - } catch (Exception e) { - if (!mMission.running || e instanceof ClosedByInterruptException) return; - mRecovery.attempts++; - mMission.notifyError(e); - } + resolve(url); } - private void resolve(String url) throws IOException, DownloadMission.HttpError { + private void resolve(String url) throws IOException, HttpError { if (mRecovery.validateCondition == null) { Log.w(TAG, "validation condition not defined, the resource can be stale"); } @@ -190,10 +207,7 @@ public class DownloadMissionRecover extends Thread { return; } - throw new DownloadMission.HttpError(code); - } catch (Exception e) { - if (!mMission.running || e instanceof ClosedByInterruptException) return; - throw e; + throw new HttpError(code); } finally { disconnect(); } @@ -205,14 +219,14 @@ public class DownloadMissionRecover extends Thread { ); mMission.urls[mMission.current] = url; - mRecovery.attempts = 0; if (url == null) { + mMission.urls = new String[0]; mMission.notifyError(ERROR_RESOURCE_GONE, null); return; } - if (notInitialized) return; + if (mNotInitialized) return; if (stale) { mMission.resetState(false, false, DownloadMission.ERROR_NOTHING); diff --git a/app/src/main/java/us/shandian/giga/get/DownloadRunnable.java b/app/src/main/java/us/shandian/giga/get/DownloadRunnable.java index b0dc793bc..4aa6e912e 100644 --- a/app/src/main/java/us/shandian/giga/get/DownloadRunnable.java +++ b/app/src/main/java/us/shandian/giga/get/DownloadRunnable.java @@ -87,6 +87,7 @@ public class DownloadRunnable extends Thread { if (mConn.getResponseCode() == 416) { if (block.done > 0) { // try again from the start (of the block) + mMission.notifyProgress(-block.done); block.done = 0; retry = true; mConn.disconnect(); @@ -114,7 +115,7 @@ public class DownloadRunnable extends Thread { int len; // use always start <= end - // fixes a deadlock in DownloadRunnable because youtube is sending one byte alone after downloading 26MiB exactly + // fixes a deadlock because in some videos, youtube is sending one byte alone while (start <= end && mMission.running && (len = is.read(buf, 0, buf.length)) != -1) { f.write(buf, 0, len); start += len; @@ -135,7 +136,7 @@ public class DownloadRunnable extends Thread { if (mId == 1) { // only the first thread will execute the recovery procedure - mMission.doRecover(e); + mMission.doRecover(ERROR_HTTP_FORBIDDEN); } return; } diff --git a/app/src/main/java/us/shandian/giga/get/DownloadRunnableFallback.java b/app/src/main/java/us/shandian/giga/get/DownloadRunnableFallback.java index e64322b48..9cb40cb32 100644 --- a/app/src/main/java/us/shandian/giga/get/DownloadRunnableFallback.java +++ b/app/src/main/java/us/shandian/giga/get/DownloadRunnableFallback.java @@ -1,8 +1,9 @@ package us.shandian.giga.get; -import androidx.annotation.NonNull; import android.util.Log; +import androidx.annotation.NonNull; + import org.schabi.newpipe.streams.io.SharpStream; import java.io.IOException; @@ -47,22 +48,10 @@ public class DownloadRunnableFallback extends Thread { if (mF != null) mF.close(); } - private long loadPosition() { - synchronized (mMission.LOCK) { - return mMission.fallbackResumeOffset; - } - } - - private void savePosition(long position) { - synchronized (mMission.LOCK) { - mMission.fallbackResumeOffset = position; - } - } - @Override public void run() { boolean done; - long start = loadPosition(); + long start = mMission.fallbackResumeOffset; if (DEBUG && !mMission.unknownLength && start > 0) { Log.i(TAG, "Resuming a single-thread download at " + start); @@ -83,6 +72,7 @@ public class DownloadRunnableFallback extends Thread { // check if the download can be resumed if (mConn.getResponseCode() == 416 && start > 0) { + mMission.notifyProgress(-start); start = 0; mRetryCount--; throw new DownloadMission.HttpError(416); @@ -92,6 +82,11 @@ public class DownloadRunnableFallback extends Thread { if (!mMission.unknownLength) mMission.unknownLength = Utility.getContentLength(mConn) == -1; + if (mMission.unknownLength || mConn.getResponseCode() == 200) { + // restart amount of bytes downloaded + mMission.done = mMission.offsets[mMission.current] - mMission.offsets[0]; + } + mF = mMission.storage.getStream(); mF.seek(mMission.offsets[mMission.current] + start); @@ -113,14 +108,14 @@ public class DownloadRunnableFallback extends Thread { } catch (Exception e) { dispose(); - savePosition(start); + mMission.fallbackResumeOffset = start; if (!mMission.running || e instanceof ClosedByInterruptException) return; if (e instanceof HttpError && ((HttpError) e).statusCode == ERROR_HTTP_FORBIDDEN) { // for youtube streams. The url has expired, recover dispose(); - mMission.doRecover(e); + mMission.doRecover(ERROR_HTTP_FORBIDDEN); return; } @@ -140,7 +135,7 @@ public class DownloadRunnableFallback extends Thread { if (done) { mMission.notifyFinished(); } else { - savePosition(start); + mMission.fallbackResumeOffset = start; } } diff --git a/app/src/main/java/us/shandian/giga/get/FinishedMission.java b/app/src/main/java/us/shandian/giga/get/FinishedMission.java index b468f3c76..6bc5423b8 100644 --- a/app/src/main/java/us/shandian/giga/get/FinishedMission.java +++ b/app/src/main/java/us/shandian/giga/get/FinishedMission.java @@ -2,17 +2,17 @@ package us.shandian.giga.get; import androidx.annotation.NonNull; -public class FinishedMission extends Mission { +public class FinishedMission extends Mission { public FinishedMission() { } public FinishedMission(@NonNull DownloadMission mission) { source = mission.source; - length = mission.length;// ¿or mission.done? + length = mission.length; timestamp = mission.timestamp; kind = mission.kind; storage = mission.storage; - } + } diff --git a/app/src/main/java/us/shandian/giga/get/MissionRecoveryInfo.java b/app/src/main/java/us/shandian/giga/get/MissionRecoveryInfo.java index bd1d9bc49..f6a3a3984 100644 --- a/app/src/main/java/us/shandian/giga/get/MissionRecoveryInfo.java +++ b/app/src/main/java/us/shandian/giga/get/MissionRecoveryInfo.java @@ -2,7 +2,8 @@ package us.shandian.giga.get; import android.os.Parcel; import android.os.Parcelable; -import android.support.annotation.NonNull; + +import androidx.annotation.NonNull; import org.schabi.newpipe.extractor.MediaFormat; import org.schabi.newpipe.extractor.stream.AudioStream; @@ -23,8 +24,6 @@ public class MissionRecoveryInfo implements Serializable, Parcelable { byte kind; String validateCondition = null; - transient int attempts = 0; - public MissionRecoveryInfo(@NonNull Stream stream) { if (stream instanceof AudioStream) { desiredBitrate = ((AudioStream) stream).average_bitrate; @@ -51,7 +50,7 @@ public class MissionRecoveryInfo implements Serializable, Parcelable { public String toString() { String info; StringBuilder str = new StringBuilder(); - str.append("type="); + str.append("{type="); switch (kind) { case 'a': str.append("audio"); @@ -73,7 +72,8 @@ public class MissionRecoveryInfo implements Serializable, Parcelable { str.append(" format=") .append(format.getName()) .append(' ') - .append(info); + .append(info) + .append('}'); return str.toString(); } diff --git a/app/src/main/java/us/shandian/giga/io/ChunkFileInputStream.java b/app/src/main/java/us/shandian/giga/io/ChunkFileInputStream.java index 16a90fcee..98015e37e 100644 --- a/app/src/main/java/us/shandian/giga/io/ChunkFileInputStream.java +++ b/app/src/main/java/us/shandian/giga/io/ChunkFileInputStream.java @@ -5,21 +5,23 @@ import org.schabi.newpipe.streams.io.SharpStream; import java.io.IOException; public class ChunkFileInputStream extends SharpStream { + private static final int REPORT_INTERVAL = 256 * 1024; private SharpStream source; private final long offset; private final long length; private long position; - public ChunkFileInputStream(SharpStream target, long start) throws IOException { - this(target, start, target.length()); - } + private long progressReport; + private final ProgressReport onProgress; - public ChunkFileInputStream(SharpStream target, long start, long end) throws IOException { + public ChunkFileInputStream(SharpStream target, long start, long end, ProgressReport callback) throws IOException { source = target; offset = start; length = end - start; position = 0; + onProgress = callback; + progressReport = REPORT_INTERVAL; if (length < 1) { source.close(); @@ -60,12 +62,12 @@ public class ChunkFileInputStream extends SharpStream { } @Override - public int read(byte b[]) throws IOException { + public int read(byte[] b) throws IOException { return read(b, 0, b.length); } @Override - public int read(byte b[], int off, int len) throws IOException { + public int read(byte[] b, int off, int len) throws IOException { if ((position + len) > length) { len = (int) (length - position); } @@ -76,6 +78,11 @@ public class ChunkFileInputStream extends SharpStream { int res = source.read(b, off, len); position += res; + if (onProgress != null && position > progressReport) { + onProgress.report(position); + progressReport = position + REPORT_INTERVAL; + } + return res; } diff --git a/app/src/main/java/us/shandian/giga/io/CircularFileWriter.java b/app/src/main/java/us/shandian/giga/io/CircularFileWriter.java index e2afb9202..102580570 100644 --- a/app/src/main/java/us/shandian/giga/io/CircularFileWriter.java +++ b/app/src/main/java/us/shandian/giga/io/CircularFileWriter.java @@ -174,12 +174,12 @@ public class CircularFileWriter extends SharpStream { } @Override - public void write(byte b[]) throws IOException { + public void write(byte[] b) throws IOException { write(b, 0, b.length); } @Override - public void write(byte b[], int off, int len) throws IOException { + public void write(byte[] b, int off, int len) throws IOException { if (len == 0) { return; } @@ -261,7 +261,7 @@ public class CircularFileWriter extends SharpStream { @Override public void rewind() throws IOException { if (onProgress != null) { - onProgress.report(-out.length - aux.length);// rollback the whole progress + onProgress.report(0);// rollback the whole progress } seek(0); @@ -357,16 +357,6 @@ public class CircularFileWriter extends SharpStream { long check(); } - public interface ProgressReport { - - /** - * Report the size of the new file - * - * @param progress the new size - */ - void report(long progress); - } - public interface WriteErrorHandle { /** @@ -381,10 +371,10 @@ public class CircularFileWriter extends SharpStream { class BufferedFile { - protected final SharpStream target; + final SharpStream target; private long offset; - protected long length; + long length; private byte[] queue = new byte[QUEUE_BUFFER_SIZE]; private int queueSize; @@ -397,16 +387,16 @@ public class CircularFileWriter extends SharpStream { this.target = target; } - protected long getOffset() { + long getOffset() { return offset + queueSize;// absolute offset in the file } - protected void close() { + void close() { queue = null; target.close(); } - protected void write(byte b[], int off, int len) throws IOException { + void write(byte[] b, int off, int len) throws IOException { while (len > 0) { // if the queue is full, the method available() will flush the queue int read = Math.min(available(), len); @@ -436,7 +426,7 @@ public class CircularFileWriter extends SharpStream { target.seek(0); } - protected int available() throws IOException { + int available() throws IOException { if (queueSize >= queue.length) { flush(); return queue.length; @@ -451,7 +441,7 @@ public class CircularFileWriter extends SharpStream { target.seek(0); } - protected void seek(long absoluteOffset) throws IOException { + void seek(long absoluteOffset) throws IOException { if (absoluteOffset == offset) { return;// nothing to do } diff --git a/app/src/main/java/us/shandian/giga/io/ProgressReport.java b/app/src/main/java/us/shandian/giga/io/ProgressReport.java new file mode 100644 index 000000000..14ae9ded9 --- /dev/null +++ b/app/src/main/java/us/shandian/giga/io/ProgressReport.java @@ -0,0 +1,11 @@ +package us.shandian.giga.io; + +public interface ProgressReport { + + /** + * Report the size of the new file + * + * @param progress the new size + */ + void report(long progress); +} \ No newline at end of file diff --git a/app/src/main/java/us/shandian/giga/postprocessing/OggFromWebmDemuxer.java b/app/src/main/java/us/shandian/giga/postprocessing/OggFromWebmDemuxer.java index 605c0a88b..04958c495 100644 --- a/app/src/main/java/us/shandian/giga/postprocessing/OggFromWebmDemuxer.java +++ b/app/src/main/java/us/shandian/giga/postprocessing/OggFromWebmDemuxer.java @@ -1,6 +1,6 @@ package us.shandian.giga.postprocessing; -import android.support.annotation.NonNull; +import androidx.annotation.NonNull; import org.schabi.newpipe.streams.OggFromWebMWriter; import org.schabi.newpipe.streams.io.SharpStream; diff --git a/app/src/main/java/us/shandian/giga/postprocessing/Postprocessing.java b/app/src/main/java/us/shandian/giga/postprocessing/Postprocessing.java index 92510c3df..773ff92d1 100644 --- a/app/src/main/java/us/shandian/giga/postprocessing/Postprocessing.java +++ b/app/src/main/java/us/shandian/giga/postprocessing/Postprocessing.java @@ -1,9 +1,9 @@ package us.shandian.giga.postprocessing; -import android.os.Message; -import androidx.annotation.NonNull; import android.util.Log; +import androidx.annotation.NonNull; + import org.schabi.newpipe.streams.io.SharpStream; import java.io.File; @@ -14,11 +14,11 @@ import us.shandian.giga.get.DownloadMission; import us.shandian.giga.io.ChunkFileInputStream; import us.shandian.giga.io.CircularFileWriter; import us.shandian.giga.io.CircularFileWriter.OffsetChecker; -import us.shandian.giga.service.DownloadManagerService; +import us.shandian.giga.io.ProgressReport; import static us.shandian.giga.get.DownloadMission.ERROR_NOTHING; +import static us.shandian.giga.get.DownloadMission.ERROR_POSTPROCESSING; import static us.shandian.giga.get.DownloadMission.ERROR_POSTPROCESSING_HOLD; -import static us.shandian.giga.get.DownloadMission.ERROR_UNKNOWN_EXCEPTION; public abstract class Postprocessing implements Serializable { @@ -63,22 +63,22 @@ public abstract class Postprocessing implements Serializable { * Get a boolean value that indicate if the given algorithm work on the same * file */ - public final boolean worksOnSameFile; + public boolean worksOnSameFile; /** * Indicates whether the selected algorithm needs space reserved at the beginning of the file */ - public final boolean reserveSpace; + public boolean reserveSpace; /** * Gets the given algorithm short name */ - private final String name; + private String name; private String[] args; - protected transient DownloadMission mission; + private transient DownloadMission mission; private File tempFile; @@ -109,16 +109,24 @@ public abstract class Postprocessing implements Serializable { long finalLength = -1; mission.done = 0; - mission.length = mission.storage.length(); + + long length = mission.storage.length() - mission.offsets[0]; + mission.length = length > mission.nearLength ? length : mission.nearLength; + + final ProgressReport readProgress = (long position) -> { + position -= mission.offsets[0]; + if (position > mission.done) mission.done = position; + }; if (worksOnSameFile) { ChunkFileInputStream[] sources = new ChunkFileInputStream[mission.urls.length]; try { - int i = 0; - for (; i < sources.length - 1; i++) { - sources[i] = new ChunkFileInputStream(mission.storage.getStream(), mission.offsets[i], mission.offsets[i + 1]); + for (int i = 0, j = 1; i < sources.length; i++, j++) { + SharpStream source = mission.storage.getStream(); + long end = j < sources.length ? mission.offsets[j] : source.length(); + + sources[i] = new ChunkFileInputStream(source, mission.offsets[i], end, readProgress); } - sources[i] = new ChunkFileInputStream(mission.storage.getStream(), mission.offsets[i]); if (test(sources)) { for (SharpStream source : sources) source.rewind(); @@ -140,7 +148,7 @@ public abstract class Postprocessing implements Serializable { }; out = new CircularFileWriter(mission.storage.getStream(), tempFile, checker); - out.onProgress = this::progressReport; + out.onProgress = (long position) -> mission.done = position; out.onWriteError = (err) -> { mission.psState = 3; @@ -187,11 +195,10 @@ public abstract class Postprocessing implements Serializable { if (result == OK_RESULT) { if (finalLength != -1) { - mission.done = finalLength; mission.length = finalLength; } } else { - mission.errCode = ERROR_UNKNOWN_EXCEPTION; + mission.errCode = ERROR_POSTPROCESSING; mission.errObject = new RuntimeException("post-processing algorithm returned " + result); } @@ -229,23 +236,12 @@ public abstract class Postprocessing implements Serializable { return args[index]; } - private void progressReport(long done) { - mission.done = done; - if (mission.length < mission.done) mission.length = mission.done; - - Message m = new Message(); - m.what = DownloadManagerService.MESSAGE_PROGRESS; - m.obj = mission; - - mission.mHandler.sendMessage(m); - } - @NonNull @Override public String toString() { StringBuilder str = new StringBuilder(); - str.append("name=").append(name).append('['); + str.append("{ name=").append(name).append('['); if (args != null) { for (String arg : args) { @@ -255,6 +251,6 @@ public abstract class Postprocessing implements Serializable { str.delete(0, 1); } - return str.append(']').toString(); + return str.append("] }").toString(); } } diff --git a/app/src/main/java/us/shandian/giga/service/DownloadManager.java b/app/src/main/java/us/shandian/giga/service/DownloadManager.java index 2d1e9cd00..e8bc468e9 100644 --- a/app/src/main/java/us/shandian/giga/service/DownloadManager.java +++ b/app/src/main/java/us/shandian/giga/service/DownloadManager.java @@ -2,13 +2,11 @@ package us.shandian.giga.service; import android.content.Context; import android.os.Handler; +import android.util.Log; + import androidx.annotation.NonNull; import androidx.annotation.Nullable; import androidx.recyclerview.widget.DiffUtil; -import android.util.Log; -import android.widget.Toast; - -import org.schabi.newpipe.R; import java.io.File; import java.io.IOException; @@ -152,6 +150,8 @@ public class DownloadManager { continue; } + mis.threads = new Thread[0]; + boolean exists; try { mis.storage = StoredFileHelper.deserialize(mis.storage, ctx); @@ -170,8 +170,6 @@ public class DownloadManager { // is Java IO (avoid showing the "Save as..." dialog) if (exists && mis.storage.isDirect() && !mis.storage.delete()) Log.w(TAG, "Unable to delete incomplete download file: " + sub.getPath()); - - exists = true; } mis.psState = 0; @@ -243,7 +241,6 @@ public class DownloadManager { boolean start = !mPrefQueueLimit || getRunningMissionsCount() < 1; if (canDownloadInCurrentNetwork() && start) { - mHandler.sendEmptyMessage(DownloadManagerService.MESSAGE_PROGRESS); mission.start(); } } @@ -252,7 +249,6 @@ public class DownloadManager { public void resumeMission(DownloadMission mission) { if (!mission.running) { - mHandler.sendEmptyMessage(DownloadManagerService.MESSAGE_PROGRESS); mission.start(); } } @@ -261,7 +257,6 @@ public class DownloadManager { if (mission.running) { mission.setEnqueued(false); mission.pause(); - mHandler.sendEmptyMessage(DownloadManagerService.MESSAGE_PAUSED); } } @@ -274,7 +269,6 @@ public class DownloadManager { mFinishedMissionStore.deleteMission(mission); } - mHandler.sendEmptyMessage(DownloadManagerService.MESSAGE_DELETED); mission.delete(); } } @@ -291,7 +285,6 @@ public class DownloadManager { mFinishedMissionStore.deleteMission(mission); } - mHandler.sendEmptyMessage(DownloadManagerService.MESSAGE_DELETED); mission.storage = null; mission.delete(); } @@ -374,35 +367,29 @@ public class DownloadManager { } public void pauseAllMissions(boolean force) { - boolean flag = false; - synchronized (this) { for (DownloadMission mission : mMissionsPending) { if (!mission.running || mission.isPsRunning() || mission.isFinished()) continue; - if (force) mission.threads = null;// avoid waiting for threads + if (force) { + // avoid waiting for threads + mission.init = null; + mission.threads = new Thread[0]; + } mission.pause(); - flag = true; } } - - if (flag) mHandler.sendEmptyMessage(DownloadManagerService.MESSAGE_PAUSED); } public void startAllMissions() { - boolean flag = false; - synchronized (this) { for (DownloadMission mission : mMissionsPending) { if (mission.running || mission.isCorrupt()) continue; - flag = true; mission.start(); } } - - if (flag) mHandler.sendEmptyMessage(DownloadManagerService.MESSAGE_PROGRESS); } /** @@ -483,28 +470,18 @@ public class DownloadManager { boolean isMetered = mPrefMeteredDownloads && mLastNetworkStatus == NetworkState.MeteredOperating; - int running = 0; - int paused = 0; synchronized (this) { for (DownloadMission mission : mMissionsPending) { if (mission.isCorrupt() || mission.isPsRunning()) continue; if (mission.running && isMetered) { - paused++; mission.pause(); } else if (!mission.running && !isMetered && mission.enqueued) { - running++; mission.start(); if (mPrefQueueLimit) break; } } } - - if (running > 0) { - mHandler.sendEmptyMessage(DownloadManagerService.MESSAGE_PROGRESS); - return; - } - if (paused > 0) mHandler.sendEmptyMessage(DownloadManagerService.MESSAGE_PAUSED); } void updateMaximumAttempts() { @@ -513,22 +490,6 @@ public class DownloadManager { } } - /** - * Fast check for pending downloads. If exists, the user will be notified - * TODO: call this method in somewhere - * - * @param context the application context - */ - public static void notifyUserPendingDownloads(Context context) { - int pending = getPendingDir(context).list().length; - if (pending < 1) return; - - Toast.makeText(context, context.getString( - R.string.msg_pending_downloads, - String.valueOf(pending) - ), Toast.LENGTH_LONG).show(); - } - public MissionState checkForExistingMission(StoredFileHelper storage) { synchronized (this) { DownloadMission pending = getPendingMission(storage); diff --git a/app/src/main/java/us/shandian/giga/service/DownloadManagerService.java b/app/src/main/java/us/shandian/giga/service/DownloadManagerService.java index ea9029c0b..3da0e75b8 100755 --- a/app/src/main/java/us/shandian/giga/service/DownloadManagerService.java +++ b/app/src/main/java/us/shandian/giga/service/DownloadManagerService.java @@ -25,14 +25,15 @@ import android.os.IBinder; import android.os.Message; import android.os.Parcelable; import android.preference.PreferenceManager; +import android.util.Log; +import android.util.SparseArray; +import android.widget.Toast; + import androidx.annotation.NonNull; import androidx.annotation.Nullable; import androidx.annotation.StringRes; import androidx.core.app.NotificationCompat; import androidx.core.app.NotificationCompat.Builder; -import android.util.Log; -import android.util.SparseArray; -import android.widget.Toast; import org.schabi.newpipe.R; import org.schabi.newpipe.download.DownloadActivity; @@ -41,8 +42,6 @@ import org.schabi.newpipe.player.helper.LockManager; import java.io.File; import java.io.IOException; import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; import us.shandian.giga.get.DownloadMission; import us.shandian.giga.get.MissionRecoveryInfo; @@ -58,11 +57,11 @@ public class DownloadManagerService extends Service { private static final String TAG = "DownloadManagerService"; + public static final int MESSAGE_RUNNING = 0; public static final int MESSAGE_PAUSED = 1; public static final int MESSAGE_FINISHED = 2; - public static final int MESSAGE_PROGRESS = 3; - public static final int MESSAGE_ERROR = 4; - public static final int MESSAGE_DELETED = 5; + public static final int MESSAGE_ERROR = 3; + public static final int MESSAGE_DELETED = 4; private static final int FOREGROUND_NOTIFICATION_ID = 1000; private static final int DOWNLOADS_NOTIFICATION_ID = 1001; @@ -217,9 +216,11 @@ public class DownloadManagerService extends Service { .addFlags(Intent.FLAG_ACTIVITY_NEW_TASK) ); } + return START_NOT_STICKY; } } - return START_NOT_STICKY; + + return START_STICKY; } @Override @@ -250,6 +251,7 @@ public class DownloadManagerService extends Service { if (icDownloadFailed != null) icDownloadFailed.recycle(); if (icLauncher != null) icLauncher.recycle(); + mHandler = null; mManager.pauseAllMissions(true); } @@ -274,6 +276,8 @@ public class DownloadManagerService extends Service { } private boolean handleMessage(@NonNull Message msg) { + if (mHandler == null) return true; + DownloadMission mission = (DownloadMission) msg.obj; switch (msg.what) { @@ -284,7 +288,7 @@ public class DownloadManagerService extends Service { handleConnectivityState(false); updateForegroundState(mManager.runMissions()); break; - case MESSAGE_PROGRESS: + case MESSAGE_RUNNING: updateForegroundState(true); break; case MESSAGE_ERROR: @@ -300,11 +304,8 @@ public class DownloadManagerService extends Service { if (msg.what != MESSAGE_ERROR) mFailedDownloads.delete(mFailedDownloads.indexOfValue(mission)); - synchronized (mEchoObservers) { - for (Callback observer : mEchoObservers) { - observer.handleMessage(msg); - } - } + for (Callback observer : mEchoObservers) + observer.handleMessage(msg); return true; } @@ -523,16 +524,6 @@ public class DownloadManagerService extends Service { return PendingIntent.getService(this, intent.hashCode(), intent, PendingIntent.FLAG_UPDATE_CURRENT); } - private void manageObservers(Callback handler, boolean add) { - synchronized (mEchoObservers) { - if (add) { - mEchoObservers.add(handler); - } else { - mEchoObservers.remove(handler); - } - } - } - private void manageLock(boolean acquire) { if (acquire == mLockAcquired) return; @@ -605,11 +596,11 @@ public class DownloadManagerService extends Service { } public void addMissionEventListener(Callback handler) { - manageObservers(handler, true); + mEchoObservers.add(handler); } public void removeMissionEventListener(Callback handler) { - manageObservers(handler, false); + mEchoObservers.remove(handler); } public void clearDownloadNotifications() { diff --git a/app/src/main/java/us/shandian/giga/ui/adapter/MissionAdapter.java b/app/src/main/java/us/shandian/giga/ui/adapter/MissionAdapter.java index 78fd7ea9d..e3a7f112a 100644 --- a/app/src/main/java/us/shandian/giga/ui/adapter/MissionAdapter.java +++ b/app/src/main/java/us/shandian/giga/ui/adapter/MissionAdapter.java @@ -10,16 +10,6 @@ import android.os.AsyncTask; import android.os.Build; import android.os.Handler; import android.os.Message; -import androidx.annotation.NonNull; -import androidx.annotation.Nullable; -import androidx.annotation.StringRes; -import androidx.core.content.FileProvider; -import androidx.core.view.ViewCompat; -import androidx.appcompat.app.AlertDialog; -import androidx.recyclerview.widget.DiffUtil; -import androidx.recyclerview.widget.RecyclerView; -import androidx.recyclerview.widget.RecyclerView.Adapter; -import androidx.recyclerview.widget.RecyclerView.ViewHolder; import android.util.Log; import android.util.SparseArray; import android.view.HapticFeedbackConstants; @@ -34,6 +24,17 @@ import android.widget.PopupMenu; import android.widget.TextView; import android.widget.Toast; +import androidx.annotation.NonNull; +import androidx.annotation.Nullable; +import androidx.annotation.StringRes; +import androidx.appcompat.app.AlertDialog; +import androidx.core.content.FileProvider; +import androidx.core.view.ViewCompat; +import androidx.recyclerview.widget.DiffUtil; +import androidx.recyclerview.widget.RecyclerView; +import androidx.recyclerview.widget.RecyclerView.Adapter; +import androidx.recyclerview.widget.RecyclerView.ViewHolder; + import org.schabi.newpipe.BuildConfig; import org.schabi.newpipe.R; import org.schabi.newpipe.extractor.NewPipe; @@ -82,6 +83,7 @@ public class MissionAdapter extends Adapter implements Handler.Callb private static final String TAG = "MissionAdapter"; private static final String UNDEFINED_PROGRESS = "--.-%"; private static final String DEFAULT_MIME_TYPE = "*/*"; + private static final String UNDEFINED_ETA = "--:--"; static { @@ -103,10 +105,11 @@ public class MissionAdapter extends Adapter implements Handler.Callb private View mEmptyMessage; private RecoverHelper mRecover; - public MissionAdapter(Context context, @NonNull DownloadManager downloadManager, View emptyMessage) { + private final Runnable rUpdater = this::updater; + + public MissionAdapter(Context context, @NonNull DownloadManager downloadManager, View emptyMessage, View root) { mContext = context; mDownloadManager = downloadManager; - mDeleter = null; mInflater = (LayoutInflater) mContext.getSystemService(Context.LAYOUT_INFLATER_SERVICE); mLayout = R.layout.mission_item; @@ -117,7 +120,10 @@ public class MissionAdapter extends Adapter implements Handler.Callb mIterator = downloadManager.getIterator(); + mDeleter = new Deleter(root, mContext, this, mDownloadManager, mIterator, mHandler); + checkEmptyMessageVisibility(); + onResume(); } @Override @@ -142,17 +148,13 @@ public class MissionAdapter extends Adapter implements Handler.Callb if (h.item.mission instanceof DownloadMission) { mPendingDownloadsItems.remove(h); if (mPendingDownloadsItems.size() < 1) { - setAutoRefresh(false); checkMasterButtonsVisibility(); } } h.popupMenu.dismiss(); h.item = null; - h.lastTimeStamp = -1; - h.lastDone = -1; - h.lastCurrent = -1; - h.state = 0; + h.resetSpeedMeasure(); } @Override @@ -191,7 +193,6 @@ public class MissionAdapter extends Adapter implements Handler.Callb h.size.setText(length); h.pause.setTitle(mission.unknownLength ? R.string.stop : R.string.pause); - h.lastCurrent = mission.current; updateProgress(h); mPendingDownloadsItems.add(h); } else { @@ -216,20 +217,10 @@ public class MissionAdapter extends Adapter implements Handler.Callb private void updateProgress(ViewHolderItem h) { if (h == null || h.item == null || h.item.mission instanceof FinishedMission) return; - long now = System.currentTimeMillis(); DownloadMission mission = (DownloadMission) h.item.mission; - - if (h.lastCurrent != mission.current) { - h.lastCurrent = mission.current; - h.lastTimeStamp = now; - h.lastDone = 0; - } else { - if (h.lastTimeStamp == -1) h.lastTimeStamp = now; - if (h.lastDone == -1) h.lastDone = mission.done; - } - - long deltaTime = now - h.lastTimeStamp; - long deltaDone = mission.done - h.lastDone; + double done = mission.done; + long length = mission.getLength(); + long now = System.currentTimeMillis(); boolean hasError = mission.errCode != ERROR_NOTHING; // hide on error @@ -237,19 +228,16 @@ public class MissionAdapter extends Adapter implements Handler.Callb // show if length is unknown h.progress.setMarquee(mission.isRecovering() || !hasError && (!mission.isInitialized() || mission.unknownLength)); - float progress; + double progress; if (mission.unknownLength) { - progress = Float.NaN; + progress = Double.NaN; h.progress.setProgress(0f); } else { - progress = (float) ((double) mission.done / mission.length); - if (mission.urls.length > 1 && mission.current < mission.urls.length) { - progress = (progress / mission.urls.length) + ((float) mission.current / mission.urls.length); - } + progress = done / length; } if (hasError) { - h.progress.setProgress(isNotFinite(progress) ? 1f : progress); + h.progress.setProgress(isNotFinite(progress) ? 1d : progress); h.status.setText(R.string.msg_error); } else if (isNotFinite(progress)) { h.status.setText(UNDEFINED_PROGRESS); @@ -258,59 +246,78 @@ public class MissionAdapter extends Adapter implements Handler.Callb h.progress.setProgress(progress); } - long length = mission.getLength(); + @StringRes int state; + String sizeStr = Utility.formatBytes(length).concat(" "); - int state; if (mission.isPsFailed() || mission.errCode == ERROR_POSTPROCESSING_HOLD) { - state = 0; + h.size.setText(sizeStr); + return; } else if (!mission.running) { - state = mission.enqueued ? 1 : 2; + state = mission.enqueued ? R.string.queued : R.string.paused; } else if (mission.isPsRunning()) { - state = 3; + state = R.string.post_processing; + } else if (mission.isRecovering()) { + state = R.string.recovering; } else { state = 0; } if (state != 0) { // update state without download speed - if (h.state != state) { - String statusStr; - h.state = state; + h.size.setText(sizeStr.concat("(").concat(mContext.getString(state)).concat(")")); + h.resetSpeedMeasure(); + return; + } - switch (state) { - case 1: - statusStr = mContext.getString(R.string.queued); - break; - case 2: - statusStr = mContext.getString(R.string.paused); - break; - case 3: - statusStr = mContext.getString(R.string.post_processing); - break; - default: - statusStr = "?"; - break; - } + if (h.lastTimestamp < 0) { + h.size.setText(sizeStr); + h.lastTimestamp = now; + h.lastDone = done; + return; + } - h.size.setText(Utility.formatBytes(length).concat(" (").concat(statusStr).concat(")")); - } else if (deltaDone > 0) { - h.lastTimeStamp = now; - h.lastDone = mission.done; - } + long deltaTime = now - h.lastTimestamp; + double deltaDone = done - h.lastDone; + if (h.lastDone > done) { + h.lastDone = done; + h.size.setText(sizeStr); return; } if (deltaDone > 0 && deltaTime > 0) { - float speed = (deltaDone * 1000f) / deltaTime; + float speed = (float) ((deltaDone * 1000d) / deltaTime); + float averageSpeed = speed; - String speedStr = Utility.formatSpeed(speed); - String sizeStr = Utility.formatBytes(length); + if (h.lastSpeedIdx < 0) { + for (int i = 0; i < h.lastSpeed.length; i++) { + h.lastSpeed[i] = speed; + } + h.lastSpeedIdx = 0; + } else { + for (int i = 0; i < h.lastSpeed.length; i++) { + averageSpeed += h.lastSpeed[i]; + } + averageSpeed /= h.lastSpeed.length + 1f; + } - h.size.setText(sizeStr.concat(" ").concat(speedStr)); + String speedStr = Utility.formatSpeed(averageSpeed); + String etaStr; - h.lastTimeStamp = now; - h.lastDone = mission.done; + if (mission.unknownLength) { + etaStr = ""; + } else { + long eta = (long) Math.ceil((length - done) / averageSpeed); + etaStr = " @ ".concat(Utility.stringifySeconds(eta)); + } + + h.size.setText(sizeStr.concat(speedStr).concat(etaStr)); + + h.lastTimestamp = now; + h.lastDone = done; + h.lastSpeed[h.lastSpeedIdx++] = speed; + + if (h.lastSpeedIdx >= h.lastSpeed.length) h.lastSpeedIdx = 0; } } @@ -389,6 +396,13 @@ public class MissionAdapter extends Adapter implements Handler.Callb return true; } + private ViewHolderItem getViewHolder(Object mission) { + for (ViewHolderItem h : mPendingDownloadsItems) { + if (h.item.mission == mission) return h; + } + return null; + } + @Override public boolean handleMessage(@NonNull Message msg) { if (mStartButton != null && mPauseButton != null) { @@ -396,33 +410,28 @@ public class MissionAdapter extends Adapter implements Handler.Callb } switch (msg.what) { - case DownloadManagerService.MESSAGE_PROGRESS: case DownloadManagerService.MESSAGE_ERROR: case DownloadManagerService.MESSAGE_FINISHED: + case DownloadManagerService.MESSAGE_DELETED: + case DownloadManagerService.MESSAGE_PAUSED: break; default: return false; } - if (msg.what == DownloadManagerService.MESSAGE_PROGRESS) { - setAutoRefresh(true); - return true; - } + ViewHolderItem h = getViewHolder(msg.obj); + if (h == null) return false; - for (ViewHolderItem h : mPendingDownloadsItems) { - if (h.item.mission != msg.obj) continue; - - if (msg.what == DownloadManagerService.MESSAGE_FINISHED) { + switch (msg.what) { + case DownloadManagerService.MESSAGE_FINISHED: + case DownloadManagerService.MESSAGE_DELETED: // DownloadManager should mark the download as finished applyChanges(); return true; - } - - updateProgress(h); - return true; } - return false; + updateProgress(h); + return true; } private void showError(@NonNull DownloadMission mission) { @@ -470,8 +479,13 @@ public class MissionAdapter extends Adapter implements Handler.Callb msg = R.string.error_insufficient_storage; break; case ERROR_UNKNOWN_EXCEPTION: - showError(mission, UserAction.DOWNLOAD_FAILED, R.string.general_error); - return; + if (mission.errObject != null) { + showError(mission, UserAction.DOWNLOAD_FAILED, R.string.general_error); + return; + } else { + msg = R.string.msg_error; + break; + } case ERROR_PROGRESS_LOST: msg = R.string.error_progress_lost; break; @@ -521,7 +535,9 @@ public class MissionAdapter extends Adapter implements Handler.Callb request.append(" ["); if (mission.recoveryInfo != null) { for (MissionRecoveryInfo recovery : mission.recoveryInfo) - request.append(" {").append(recovery.toString()).append("} "); + request.append(' ') + .append(recovery.toString()) + .append(' '); } request.append("]"); @@ -556,16 +572,10 @@ public class MissionAdapter extends Adapter implements Handler.Callb switch (id) { case R.id.start: h.status.setText(UNDEFINED_PROGRESS); - h.state = -1; - h.size.setText(Utility.formatBytes(mission.getLength())); mDownloadManager.resumeMission(mission); return true; case R.id.pause: - h.state = -1; mDownloadManager.pauseMission(mission); - updateProgress(h); - h.lastTimeStamp = -1; - h.lastDone = -1; return true; case R.id.error_message_view: showError(mission); @@ -598,12 +608,9 @@ public class MissionAdapter extends Adapter implements Handler.Callb shareFile(h.item.mission); return true; case R.id.delete: - if (mDeleter == null) { - mDownloadManager.deleteMission(h.item.mission); - } else { - mDeleter.append(h.item.mission); - } + mDeleter.append(h.item.mission); applyChanges(); + checkMasterButtonsVisibility(); return true; case R.id.md5: case R.id.sha1: @@ -639,7 +646,7 @@ public class MissionAdapter extends Adapter implements Handler.Callb mIterator.end(); for (ViewHolderItem item : mPendingDownloadsItems) { - item.lastTimeStamp = -1; + item.resetSpeedMeasure(); } notifyDataSetChanged(); @@ -672,6 +679,7 @@ public class MissionAdapter extends Adapter implements Handler.Callb public void checkMasterButtonsVisibility() { boolean[] state = mIterator.hasValidPendingMissions(); + Log.d(TAG, "checkMasterButtonsVisibility() running=" + state[0] + " paused=" + state[1]); setButtonVisible(mPauseButton, state[0]); setButtonVisible(mStartButton, state[1]); } @@ -681,86 +689,57 @@ public class MissionAdapter extends Adapter implements Handler.Callb button.setVisible(visible); } - public void ensurePausedMissions() { + public void refreshMissionItems() { for (ViewHolderItem h : mPendingDownloadsItems) { if (((DownloadMission) h.item.mission).running) continue; updateProgress(h); - h.lastTimeStamp = -1; - h.lastDone = -1; + h.resetSpeedMeasure(); } } - public void deleterDispose(boolean commitChanges) { - if (mDeleter != null) mDeleter.dispose(commitChanges); + public void onDestroy() { + mDeleter.dispose(); } - public void deleterLoad(View view) { - if (mDeleter == null) - mDeleter = new Deleter(view, mContext, this, mDownloadManager, mIterator, mHandler); + public void onResume() { + mDeleter.resume(); + mHandler.post(rUpdater); } - public void deleterResume() { - if (mDeleter != null) mDeleter.resume(); - } - - public void recoverMission(DownloadMission mission) { - for (ViewHolderItem h : mPendingDownloadsItems) { - if (mission != h.item.mission) continue; - - mission.errObject = null; - mission.resetState(true, false, DownloadMission.ERROR_NOTHING); - - h.status.setText(UNDEFINED_PROGRESS); - h.state = -1; - h.size.setText(Utility.formatBytes(mission.getLength())); - h.progress.setMarquee(true); - - mDownloadManager.resumeMission(mission); - return; - } - - } - - - private boolean mUpdaterRunning = false; - private final Runnable rUpdater = this::updater; - public void onPaused() { - setAutoRefresh(false); + mDeleter.pause(); + mHandler.removeCallbacks(rUpdater); } - private void setAutoRefresh(boolean enabled) { - if (enabled && !mUpdaterRunning) { - mUpdaterRunning = true; - updater(); - } else if (!enabled && mUpdaterRunning) { - mUpdaterRunning = false; - mHandler.removeCallbacks(rUpdater); - } + + public void recoverMission(DownloadMission mission) { + ViewHolderItem h = getViewHolder(mission); + if (h == null) return; + + mission.errObject = null; + mission.resetState(true, false, DownloadMission.ERROR_NOTHING); + + h.status.setText(UNDEFINED_PROGRESS); + h.size.setText(Utility.formatBytes(mission.getLength())); + h.progress.setMarquee(true); + + mDownloadManager.resumeMission(mission); } private void updater() { - if (!mUpdaterRunning) return; - - boolean running = false; for (ViewHolderItem h : mPendingDownloadsItems) { // check if the mission is running first if (!((DownloadMission) h.item.mission).running) continue; updateProgress(h); - running = true; } - if (running) { - mHandler.postDelayed(rUpdater, 1000); - } else { - mUpdaterRunning = false; - } + mHandler.postDelayed(rUpdater, 1000); } - private boolean isNotFinite(Float value) { - return Float.isNaN(value) || Float.isInfinite(value); + private boolean isNotFinite(double value) { + return Double.isNaN(value) || Double.isInfinite(value); } public void setRecover(@NonNull RecoverHelper callback) { @@ -789,10 +768,11 @@ public class MissionAdapter extends Adapter implements Handler.Callb MenuItem source; MenuItem checksum; - long lastTimeStamp = -1; - long lastDone = -1; - int lastCurrent = -1; - int state = 0; + long lastTimestamp = -1; + double lastDone; + int lastSpeedIdx; + float[] lastSpeed = new float[3]; + String estimatedTimeArrival = UNDEFINED_ETA; ViewHolderItem(View view) { super(view); @@ -902,6 +882,12 @@ public class MissionAdapter extends Adapter implements Handler.Callb return popup; } + + private void resetSpeedMeasure() { + estimatedTimeArrival = UNDEFINED_ETA; + lastTimestamp = -1; + lastSpeedIdx = -1; + } } class ViewHolderHeader extends RecyclerView.ViewHolder { diff --git a/app/src/main/java/us/shandian/giga/ui/common/Deleter.java b/app/src/main/java/us/shandian/giga/ui/common/Deleter.java index 81b4e33e8..a0828c23d 100644 --- a/app/src/main/java/us/shandian/giga/ui/common/Deleter.java +++ b/app/src/main/java/us/shandian/giga/ui/common/Deleter.java @@ -4,9 +4,10 @@ import android.content.Context; import android.content.Intent; import android.graphics.Color; import android.os.Handler; -import com.google.android.material.snackbar.Snackbar; import android.view.View; +import com.google.android.material.snackbar.Snackbar; + import org.schabi.newpipe.R; import java.util.ArrayList; @@ -113,7 +114,7 @@ public class Deleter { show(); } - private void pause() { + public void pause() { running = false; mHandler.removeCallbacks(rNext); mHandler.removeCallbacks(rShow); @@ -126,13 +127,11 @@ public class Deleter { mHandler.postDelayed(rShow, DELAY_RESUME); } - public void dispose(boolean commitChanges) { + public void dispose() { if (items.size() < 1) return; pause(); - if (!commitChanges) return; - for (Mission mission : items) mDownloadManager.deleteMission(mission); items = null; } diff --git a/app/src/main/java/us/shandian/giga/ui/common/ProgressDrawable.java b/app/src/main/java/us/shandian/giga/ui/common/ProgressDrawable.java index a0ff24aaa..3f638d418 100644 --- a/app/src/main/java/us/shandian/giga/ui/common/ProgressDrawable.java +++ b/app/src/main/java/us/shandian/giga/ui/common/ProgressDrawable.java @@ -9,6 +9,7 @@ import android.graphics.Rect; import android.graphics.drawable.Drawable; import android.os.Handler; import android.os.Looper; + import androidx.annotation.ColorInt; import androidx.annotation.NonNull; @@ -35,8 +36,8 @@ public class ProgressDrawable extends Drawable { mForegroundColor = foreground; } - public void setProgress(float progress) { - mProgress = progress; + public void setProgress(double progress) { + mProgress = (float) progress; invalidateSelf(); } diff --git a/app/src/main/java/us/shandian/giga/ui/fragment/MissionsFragment.java b/app/src/main/java/us/shandian/giga/ui/fragment/MissionsFragment.java index 26da47b1f..921eaff5c 100644 --- a/app/src/main/java/us/shandian/giga/ui/fragment/MissionsFragment.java +++ b/app/src/main/java/us/shandian/giga/ui/fragment/MissionsFragment.java @@ -12,11 +12,6 @@ import android.os.Bundle; import android.os.Environment; import android.os.IBinder; import android.preference.PreferenceManager; -import androidx.annotation.NonNull; -import androidx.fragment.app.Fragment; -import androidx.recyclerview.widget.GridLayoutManager; -import androidx.recyclerview.widget.LinearLayoutManager; -import androidx.recyclerview.widget.RecyclerView; import android.view.LayoutInflater; import android.view.Menu; import android.view.MenuItem; @@ -24,6 +19,12 @@ import android.view.View; import android.view.ViewGroup; import android.widget.Toast; +import androidx.annotation.NonNull; +import androidx.fragment.app.Fragment; +import androidx.recyclerview.widget.GridLayoutManager; +import androidx.recyclerview.widget.LinearLayoutManager; +import androidx.recyclerview.widget.RecyclerView; + import com.nononsenseapps.filepicker.Utils; import org.schabi.newpipe.R; @@ -72,8 +73,7 @@ public class MissionsFragment extends Fragment { mBinder = (DownloadManagerBinder) binder; mBinder.clearDownloadNotifications(); - mAdapter = new MissionAdapter(mContext, mBinder.getDownloadManager(), mEmpty); - mAdapter.deleterLoad(getView()); + mAdapter = new MissionAdapter(mContext, mBinder.getDownloadManager(), mEmpty, getView()); mAdapter.setRecover(MissionsFragment.this::recoverMission); @@ -132,7 +132,7 @@ public class MissionsFragment extends Fragment { * Added in API level 23. */ @Override - public void onAttach(Context context) { + public void onAttach(@NonNull Context context) { super.onAttach(context); // Bug: in api< 23 this is never called @@ -147,7 +147,7 @@ public class MissionsFragment extends Fragment { */ @SuppressWarnings("deprecation") @Override - public void onAttach(Activity activity) { + public void onAttach(@NonNull Activity activity) { super.onAttach(activity); mContext = activity; @@ -162,7 +162,7 @@ public class MissionsFragment extends Fragment { mBinder.removeMissionEventListener(mAdapter); mBinder.enableNotifications(true); mContext.unbindService(mConnection); - mAdapter.deleterDispose(true); + mAdapter.onDestroy(); mBinder = null; mAdapter = null; @@ -196,13 +196,11 @@ public class MissionsFragment extends Fragment { prompt.create().show(); return true; case R.id.start_downloads: - item.setVisible(false); mBinder.getDownloadManager().startAllMissions(); return true; case R.id.pause_downloads: - item.setVisible(false); mBinder.getDownloadManager().pauseAllMissions(false); - mAdapter.ensurePausedMissions();// update items view + mAdapter.refreshMissionItems();// update items view default: return super.onOptionsItemSelected(item); } @@ -271,23 +269,12 @@ public class MissionsFragment extends Fragment { } } - @Override - public void onSaveInstanceState(@NonNull Bundle outState) { - super.onSaveInstanceState(outState); - - if (mAdapter != null) { - mAdapter.deleterDispose(false); - mForceUpdate = true; - mBinder.removeMissionEventListener(mAdapter); - } - } - @Override public void onResume() { super.onResume(); if (mAdapter != null) { - mAdapter.deleterResume(); + mAdapter.onResume(); if (mForceUpdate) { mForceUpdate = false; @@ -303,7 +290,13 @@ public class MissionsFragment extends Fragment { @Override public void onPause() { super.onPause(); - if (mAdapter != null) mAdapter.onPaused(); + + if (mAdapter != null) { + mForceUpdate = true; + mBinder.removeMissionEventListener(mAdapter); + mAdapter.onPaused(); + } + if (mBinder != null) mBinder.enableNotifications(true); } diff --git a/app/src/main/java/us/shandian/giga/util/Utility.java b/app/src/main/java/us/shandian/giga/util/Utility.java index 21fdd72ad..46207777a 100644 --- a/app/src/main/java/us/shandian/giga/util/Utility.java +++ b/app/src/main/java/us/shandian/giga/util/Utility.java @@ -4,13 +4,14 @@ import android.content.ClipData; import android.content.ClipboardManager; import android.content.Context; import android.os.Build; +import android.util.Log; +import android.widget.Toast; + import androidx.annotation.ColorInt; import androidx.annotation.DrawableRes; import androidx.annotation.NonNull; import androidx.annotation.Nullable; import androidx.core.content.ContextCompat; -import android.util.Log; -import android.widget.Toast; import org.schabi.newpipe.R; import org.schabi.newpipe.streams.io.SharpStream; @@ -26,6 +27,7 @@ import java.io.Serializable; import java.net.HttpURLConnection; import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; +import java.util.Locale; import us.shandian.giga.io.StoredFileHelper; @@ -39,26 +41,28 @@ public class Utility { } public static String formatBytes(long bytes) { + Locale locale = Locale.getDefault(); if (bytes < 1024) { - return String.format("%d B", bytes); + return String.format(locale, "%d B", bytes); } else if (bytes < 1024 * 1024) { - return String.format("%.2f kB", bytes / 1024d); + return String.format(locale, "%.2f kB", bytes / 1024d); } else if (bytes < 1024 * 1024 * 1024) { - return String.format("%.2f MB", bytes / 1024d / 1024d); + return String.format(locale, "%.2f MB", bytes / 1024d / 1024d); } else { - return String.format("%.2f GB", bytes / 1024d / 1024d / 1024d); + return String.format(locale, "%.2f GB", bytes / 1024d / 1024d / 1024d); } } - public static String formatSpeed(float speed) { + public static String formatSpeed(double speed) { + Locale locale = Locale.getDefault(); if (speed < 1024) { - return String.format("%.2f B/s", speed); + return String.format(locale, "%.2f B/s", speed); } else if (speed < 1024 * 1024) { - return String.format("%.2f kB/s", speed / 1024); + return String.format(locale, "%.2f kB/s", speed / 1024); } else if (speed < 1024 * 1024 * 1024) { - return String.format("%.2f MB/s", speed / 1024 / 1024); + return String.format(locale, "%.2f MB/s", speed / 1024 / 1024); } else { - return String.format("%.2f GB/s", speed / 1024 / 1024 / 1024); + return String.format(locale, "%.2f GB/s", speed / 1024 / 1024 / 1024); } } @@ -188,12 +192,11 @@ public class Utility { switch (type) { case MUSIC: return R.drawable.music; + default: case VIDEO: return R.drawable.video; case SUBTITLE: return R.drawable.subtitle; - default: - return R.drawable.video; } } @@ -274,4 +277,25 @@ public class Utility { return -1; } + + private static String pad(int number) { + return number < 10 ? ("0" + number) : String.valueOf(number); + } + + public static String stringifySeconds(double seconds) { + int h = (int) Math.floor(seconds / 3600); + int m = (int) Math.floor((seconds - (h * 3600)) / 60); + int s = (int) (seconds - (h * 3600) - (m * 60)); + + String str = ""; + + if (h < 1 && m < 1) { + str = "00:"; + } else { + if (h > 0) str = pad(h) + ":"; + if (m > 0) str += pad(m) + ":"; + } + + return str + pad(s); + } } diff --git a/app/src/main/res/values-ar/strings.xml b/app/src/main/res/values-ar/strings.xml index 43b45d15e..86cbbb59a 100644 --- a/app/src/main/res/values-ar/strings.xml +++ b/app/src/main/res/values-ar/strings.xml @@ -471,7 +471,6 @@ غير موجود فشلت المعالجة الاولية حذف التنزيلات المنتهية - "قم بإستكمال %s حيثما يتم التحويل من التنزيلات" توقف أقصى عدد للمحاولات الحد الأقصى لعدد محاولات قبل إلغاء التحميل diff --git a/app/src/main/res/values-be/strings.xml b/app/src/main/res/values-be/strings.xml index 3c79a96d3..1cf3abd7e 100644 --- a/app/src/main/res/values-be/strings.xml +++ b/app/src/main/res/values-be/strings.xml @@ -458,7 +458,6 @@ Не знойдзена Пасляапрацоўка не ўдалася Ачысціць завершаныя - Аднавіць прыпыненыя загрузкі (%s) Спыніць Максімум спробаў Колькасць спробаў перад адменай загрузкі diff --git a/app/src/main/res/values-cmn/strings.xml b/app/src/main/res/values-cmn/strings.xml index bcb145c16..3ff479bfd 100644 --- a/app/src/main/res/values-cmn/strings.xml +++ b/app/src/main/res/values-cmn/strings.xml @@ -460,7 +460,6 @@ NewPipe 更新可用! 无法创建目标文件夹 服务器不接受多线程下载, 请使用 @string/msg_threads = 1重试 - 继续进行%s个待下载转移 切换至移动数据时有用，尽管一些下载无法被暂停 显示评论 禁用停止显示评论 diff --git a/app/src/main/res/values-cs/strings.xml b/app/src/main/res/values-cs/strings.xml index b741e0d16..9a9cc8654 100644 --- a/app/src/main/res/values-cs/strings.xml +++ b/app/src/main/res/values-cs/strings.xml @@ -466,7 +466,6 @@ otevření ve vyskakovacím okně Nenalezeno Post-processing selhal Vyčistit dokončená stahování - Pokračovat ve stahování %s souborů, čekajících na stažení Zastavit Maximální počet pokusů o opakování Maximální počet pokusů před zrušením stahování diff --git a/app/src/main/res/values-da/strings.xml b/app/src/main/res/values-da/strings.xml index 199c2f85d..5e44aab61 100644 --- a/app/src/main/res/values-da/strings.xml +++ b/app/src/main/res/values-da/strings.xml @@ -447,7 +447,6 @@ sat på pause sat i kø Ryd færdige downloads - Fortsæt dine %s ventende overførsler fra Downloads Maksimalt antal genforsøg Maksimalt antal forsøg før downloaden opgives Sæt på pause ved skift til mobildata diff --git a/app/src/main/res/values-de/strings.xml b/app/src/main/res/values-de/strings.xml index 3279e919c..0dc0de8b4 100644 --- a/app/src/main/res/values-de/strings.xml +++ b/app/src/main/res/values-de/strings.xml @@ -457,7 +457,6 @@ Nicht gefunden Nachbearbeitung fehlgeschlagen Um fertige Downloads bereinigen - Setze deine %s ausstehenden Übertragungen von Downloads fort Anhalten Maximale Wiederholungen Maximalanzahl der Versuche, bevor der Download abgebrochen wird diff --git a/app/src/main/res/values-el/strings.xml b/app/src/main/res/values-el/strings.xml index 372cbb1a2..115b8d0b3 100644 --- a/app/src/main/res/values-el/strings.xml +++ b/app/src/main/res/values-el/strings.xml @@ -459,7 +459,6 @@ Δεν βρέθηκε Μετεπεξεργασία απέτυχε Εκκαθάριση ολοκληρωμένων λήψεων - Συνέχιση των %s εκκρεμών σας λήψεων Διακοπή Μέγιστες επαναπροσπάθειες Μέγιστος αριθμός προσπαθειών προτού γίνει ακύρωση της λήψης diff --git a/app/src/main/res/values-es/strings.xml b/app/src/main/res/values-es/strings.xml index b14aab94b..6fcbc9fa7 100644 --- a/app/src/main/res/values-es/strings.xml +++ b/app/src/main/res/values-es/strings.xml @@ -406,6 +406,7 @@ pausado en cola posprocesamiento + recuperando Añadir a cola Acción denegada por el sistema Se eliminó el archivo @@ -424,7 +425,6 @@ Mostrar como grilla Mostrar como lista Limpiar descargas finalizadas - Tienes %s descargas pendientes, ve a Descargas para continuarlas ¿Lo confirma\? Detener Intentos máximos diff --git a/app/src/main/res/values-et/strings.xml b/app/src/main/res/values-et/strings.xml index 4dfcc3d0e..99dc6cc80 100644 --- a/app/src/main/res/values-et/strings.xml +++ b/app/src/main/res/values-et/strings.xml @@ -460,7 +460,6 @@ Ei leitud Järeltöötlemine nurjus Eemalda lõpetatud allalaadimised - Jätka %s pooleliolevat allalaadimist Stopp Korduskatseid Suurim katsete arv enne allalaadimise tühistamist diff --git a/app/src/main/res/values-eu/strings.xml b/app/src/main/res/values-eu/strings.xml index 7b636d383..743c6b3fb 100644 --- a/app/src/main/res/values-eu/strings.xml +++ b/app/src/main/res/values-eu/strings.xml @@ -459,7 +459,6 @@ Ez aurkitua Post-prozesuak huts egin du Garbitu amaitutako deskargak - Berrekin burutzeke dauden %s transferentzia deskargetatik Gelditu Gehienezko saiakerak Deskarga ezeztatu aurretik saiatu beharreko aldi kopurua diff --git a/app/src/main/res/values-fr/strings.xml b/app/src/main/res/values-fr/strings.xml index b4388e39f..2091a62fe 100644 --- a/app/src/main/res/values-fr/strings.xml +++ b/app/src/main/res/values-fr/strings.xml @@ -466,7 +466,6 @@ Nombre maximum de tentatives avant d’annuler le téléchargement Utilisation des onglets par défaut, erreur lors de la lecture des onglets enregistrés Le serveur n’accepte pas les téléchargements multi-fils, veuillez réessayer avec @string/msg_threads = 1 - Continuer vos %s transferts en attente depuis Téléchargement Afficher les commentaires Désactiver pour ne pas afficher les commentaires Lecture automatique diff --git a/app/src/main/res/values-he/strings.xml b/app/src/main/res/values-he/strings.xml index 5e340d8b3..565f815a1 100644 --- a/app/src/main/res/values-he/strings.xml +++ b/app/src/main/res/values-he/strings.xml @@ -464,7 +464,6 @@ לא נמצא העיבוד המאוחר נכשל פינוי ההורדות שהסתיימו - ניתן להמשיך את %s ההורדות הממתינות שלך דרך ההורדות עצירה מספר הניסיונות החוזרים המרבי מספר הניסיונות החוזרים המרבי בטרם ביטול ההורדה diff --git a/app/src/main/res/values-hr/strings.xml b/app/src/main/res/values-hr/strings.xml index aa4ff9113..a981dcf5e 100644 --- a/app/src/main/res/values-hr/strings.xml +++ b/app/src/main/res/values-hr/strings.xml @@ -457,7 +457,6 @@ Nije pronađeno Naknadna obrada nije uspjela Obriši završena preuzimanja - Nastavite s prijenosima na čekanju za %s s preuzimanja Stop Maksimalnih ponovnih pokušaja Maksimalni broj pokušaja prije poništavanja preuzimanja diff --git a/app/src/main/res/values-id/strings.xml b/app/src/main/res/values-id/strings.xml index d52f5fafa..5fbdcffc1 100644 --- a/app/src/main/res/values-id/strings.xml +++ b/app/src/main/res/values-id/strings.xml @@ -453,7 +453,6 @@ Tidak ditemukan Pengolahan-pasca gagal Hapus unduhan yang sudah selesai - Lanjutkan %s transfer anda yang tertunda dari Unduhan Berhenti Percobaan maksimum Jumlah upaya maksimum sebelum membatalkan unduhan diff --git a/app/src/main/res/values-it/strings.xml b/app/src/main/res/values-it/strings.xml index c92292f99..73633ab03 100644 --- a/app/src/main/res/values-it/strings.xml +++ b/app/src/main/res/values-it/strings.xml @@ -457,7 +457,6 @@ Non trovato Post-processing fallito Pulisci i download completati - Continua i %s trasferimenti in corso dai Download Ferma Tentativi massimi Tentativi massimi prima di cancellare il download diff --git a/app/src/main/res/values-ja/strings.xml b/app/src/main/res/values-ja/strings.xml index 58ca2ebff..4c3aeb5c1 100644 --- a/app/src/main/res/values-ja/strings.xml +++ b/app/src/main/res/values-ja/strings.xml @@ -456,7 +456,6 @@ デフォルトのタブを使用します。保存されたタブの読み込みエラーが発生しました メインページに表示されるタブ 新しいバージョンが利用可能なときにアプリの更新を確認する通知を表示します - ダウンロードから %s の保留中の転送を続行します 従量制課金ネットワークの割り込み モバイルデータ通信に切り替える場合に便利ですが、一部のダウンロードは一時停止できません コメントを表示 diff --git a/app/src/main/res/values-ko/strings.xml b/app/src/main/res/values-ko/strings.xml index fdc76b04e..39b08347c 100644 --- a/app/src/main/res/values-ko/strings.xml +++ b/app/src/main/res/values-ko/strings.xml @@ -454,7 +454,6 @@ HTTP 찾을 수 없습니다 후처리 작업이 실패하였습니다 완료된 다운로드 비우기 - 대기중인 %s 다운로드를 지속하세요 멈추기 최대 재시도 횟수 다운로드를 취소하기 전까지 다시 시도할 최대 횟수 diff --git a/app/src/main/res/values-ms/strings.xml b/app/src/main/res/values-ms/strings.xml index daa120ea2..354e7b7de 100644 --- a/app/src/main/res/values-ms/strings.xml +++ b/app/src/main/res/values-ms/strings.xml @@ -453,7 +453,6 @@ Tidak ditemui Pemprosesan-pasca gagal Hapuskan senarai muat turun yang selesai - Teruskan %s pemindahan anda yang menunggu dari muat turun Berhenti Percubaan maksimum Jumlah percubaan maksimum sebelum membatalkan muat turun diff --git a/app/src/main/res/values-nb-rNO/strings.xml b/app/src/main/res/values-nb-rNO/strings.xml index 6262480b0..e0a08d0a7 100644 --- a/app/src/main/res/values-nb-rNO/strings.xml +++ b/app/src/main/res/values-nb-rNO/strings.xml @@ -458,7 +458,6 @@ Ikke funnet Etterbehandling mislyktes Tøm fullførte nedlastinger - Fortsett dine %s ventende overføringer fra Nedlastinger Stopp Maksimalt antall forsøk Maksimalt antall tilkoblingsforsøk før nedlastingen avblåses diff --git a/app/src/main/res/values-nl-rBE/strings.xml b/app/src/main/res/values-nl-rBE/strings.xml index f64ff6bf9..5c42bfd23 100644 --- a/app/src/main/res/values-nl-rBE/strings.xml +++ b/app/src/main/res/values-nl-rBE/strings.xml @@ -457,7 +457,6 @@ Niet gevonden Nabewerking mislukt Voltooide downloads wissen - Zet uw %s wachtende downloads verder via Downloads Stoppen Maximaal aantal pogingen Maximaal aantal pogingen vooraleer dat den download wordt geannuleerd diff --git a/app/src/main/res/values-nl/strings.xml b/app/src/main/res/values-nl/strings.xml index 6aecc2cd1..b9b86a292 100644 --- a/app/src/main/res/values-nl/strings.xml +++ b/app/src/main/res/values-nl/strings.xml @@ -457,7 +457,6 @@ Niet gevonden Nabewerking mislukt Voltooide downloads wissen - Zet je %s wachtende downloads voort via Downloads Stop Maximum aantal keer proberen Maximum aantal pogingen voordat de download wordt geannuleerd diff --git a/app/src/main/res/values-pa/strings.xml b/app/src/main/res/values-pa/strings.xml index b57564eba..0e579720a 100644 --- a/app/src/main/res/values-pa/strings.xml +++ b/app/src/main/res/values-pa/strings.xml @@ -453,7 +453,6 @@ ਨਹੀਂ ਲਭਿਆ Post-processing ਫੇਲ੍ਹ ਮੁਕੰਮਲ ਹੋਈਆਂ ਡਾਊਨਲੋਡ ਸਾਫ਼ ਕਰੋ - ਡਾਉਨਲੋਡਸ ਤੋਂ ਆਪਣੀਆਂ %s ਬਕਾਇਆ ਟ੍ਰਾਂਸਫਰ ਜਾਰੀ ਰੱਖੋ ਰੁੱਕੋ ਵੱਧ ਤੋਂ ਵੱਧ ਕੋਸ਼ਿਸ਼ਾਂ ਡਾਉਨਲੋਡ ਰੱਦ ਕਰਨ ਤੋਂ ਪਹਿਲਾਂ ਵੱਧ ਤੋਂ ਵੱਧ ਕੋਸ਼ਿਸ਼ਾਂ diff --git a/app/src/main/res/values-pl/strings.xml b/app/src/main/res/values-pl/strings.xml index ca1e52ff2..b7086b34f 100644 --- a/app/src/main/res/values-pl/strings.xml +++ b/app/src/main/res/values-pl/strings.xml @@ -459,7 +459,6 @@ Nie znaleziono Przetwarzanie końcowe nie powiodło się Wyczyść ukończone pobieranie - Kontynuuj %s oczekujące transfery z plików do pobrania Zatrzymaj Maksymalna liczba powtórzeń Maksymalna liczba prób przed anulowaniem pobierania diff --git a/app/src/main/res/values-pt-rBR/strings.xml b/app/src/main/res/values-pt-rBR/strings.xml index 0bdf4d006..5de1e6610 100644 --- a/app/src/main/res/values-pt-rBR/strings.xml +++ b/app/src/main/res/values-pt-rBR/strings.xml @@ -466,7 +466,6 @@ abrir em modo popup Não encontrado Falha no pós processamento Limpar downloads finalizados - Continuar seus %s downloads pendentes Parar Tentativas Máximas Número máximo de tentativas antes de cancelar o download diff --git a/app/src/main/res/values-pt/strings.xml b/app/src/main/res/values-pt/strings.xml index 6d55023d1..88fbb72a6 100644 --- a/app/src/main/res/values-pt/strings.xml +++ b/app/src/main/res/values-pt/strings.xml @@ -455,7 +455,6 @@ Não encontrado Pós-processamento falhado Limpar transferências concluídas - Continue as suas %s transferências pendentes das Transferências Parar Tentativas máximas Número máximo de tentativas antes de cancelar a transferência diff --git a/app/src/main/res/values-ru/strings.xml b/app/src/main/res/values-ru/strings.xml index 51771e1b1..80b587657 100644 --- a/app/src/main/res/values-ru/strings.xml +++ b/app/src/main/res/values-ru/strings.xml @@ -464,7 +464,6 @@ Загрузка завершена %s загрузок завершено Создать уникальное имя - Возобновить приостановленные загрузки (%s) Максимум попыток Количество попыток перед отменой загрузки Некоторые загрузки не поддерживают докачку и начнутся с начала diff --git a/app/src/main/res/values-sk/strings.xml b/app/src/main/res/values-sk/strings.xml index 36c0afd84..cbc201fd5 100644 --- a/app/src/main/res/values-sk/strings.xml +++ b/app/src/main/res/values-sk/strings.xml @@ -465,7 +465,6 @@ Nenájdené Post-spracovanie zlyhalo Vyčistiť dokončené sťahovania - Pokračujte v preberaní %s zo súborov na prevzatie Stop Maximum opakovaní Maximálny počet pokusov pred zrušením stiahnutia diff --git a/app/src/main/res/values-tr/strings.xml b/app/src/main/res/values-tr/strings.xml index 6c9c66f69..1cb6fafd4 100644 --- a/app/src/main/res/values-tr/strings.xml +++ b/app/src/main/res/values-tr/strings.xml @@ -452,7 +452,6 @@ Bulunamadı İşlem sonrası başarısız Tamamlanan indirmeleri temizle - Beklemedeki %s transferinize İndirmeler\'den devam edin Durdur Azami deneme sayısı İndirmeyi iptal etmeden önce maksimum deneme sayısı diff --git a/app/src/main/res/values-uk/strings.xml b/app/src/main/res/values-uk/strings.xml index fcce99e89..d43b8be66 100644 --- a/app/src/main/res/values-uk/strings.xml +++ b/app/src/main/res/values-uk/strings.xml @@ -471,7 +471,6 @@ Помилка зчитування збережених вкладок. Використовую типові вкладки. Вкладки, що відображаються на головній сторінці Показувати сповіщення з пропозицією оновити застосунок за наявності нової версії - Продовжити ваші %s відкладених переміщень із Завантажень Корисно під час переходу на мобільні дані, хоча деякі завантаження не можуть бути призупинені Показувати коментарі Вимнути відображення дописів diff --git a/app/src/main/res/values-vi/strings.xml b/app/src/main/res/values-vi/strings.xml index f8860acfd..ab0983e7a 100644 --- a/app/src/main/res/values-vi/strings.xml +++ b/app/src/main/res/values-vi/strings.xml @@ -452,7 +452,6 @@ Không tìm thấy Xử lý thất bại Dọn các tải về đã hoàn thành - Hãy tiếp tục %s tải về đang chờ Dừng Số lượt thử lại tối đa Số lượt thử lại trước khi hủy tải về diff --git a/app/src/main/res/values-zh-rTW/strings.xml b/app/src/main/res/values-zh-rTW/strings.xml index 310bae3a3..98b9cf381 100644 --- a/app/src/main/res/values-zh-rTW/strings.xml +++ b/app/src/main/res/values-zh-rTW/strings.xml @@ -450,7 +450,6 @@ 找不到 後處理失敗 清除已結束的下載 - 繼續從您所擱置中的下載 %s 傳輸 停止 最大重試次數 在取消下載前的最大嘗試數 diff --git a/app/src/main/res/values/strings.xml b/app/src/main/res/values/strings.xml index f929e0d2b..c2d8d70f2 100644 --- a/app/src/main/res/values/strings.xml +++ b/app/src/main/res/values/strings.xml @@ -526,6 +526,7 @@ paused queued post-processing + recovering Queue Action denied by the system @@ -560,7 +561,6 @@ Cannot recover this download Clear finished downloads Are you sure? - Continue your %s pending transfers from Downloads Stop Maximum retries Maximum number of attempts before canceling the download From ea1be11a8031e25dcf90c9af54a35372dd92744d Mon Sep 17 00:00:00 2001 From: kapodamy Date: Sun, 24 Nov 2019 14:00:22 -0300 Subject: [PATCH 13/28] Merge branch 'dev' into dl-last-features --- .../us/shandian/giga/get/DownloadMission.java | 26 +++++++++++++++---- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/app/src/main/java/us/shandian/giga/get/DownloadMission.java b/app/src/main/java/us/shandian/giga/get/DownloadMission.java index 5ef72162c..917a0a148 100644 --- a/app/src/main/java/us/shandian/giga/get/DownloadMission.java +++ b/app/src/main/java/us/shandian/giga/get/DownloadMission.java @@ -1,6 +1,9 @@ package us.shandian.giga.get; +import android.os.Build; import android.os.Handler; +import android.system.ErrnoException; +import android.system.OsConstants; import android.util.Log; import androidx.annotation.Nullable; @@ -35,9 +38,6 @@ public class DownloadMission extends Mission { static final int BUFFER_SIZE = 64 * 1024; static final int BLOCK_SIZE = 512 * 1024; - @SuppressWarnings("SpellCheckingInspection") - private static final String INSUFFICIENT_STORAGE = "ENOSPC"; - private static final String TAG = "DownloadMission"; public static final int ERROR_NOTHING = -1; @@ -315,13 +315,29 @@ public class DownloadMission extends Mission { public synchronized void notifyError(int code, Exception err) { Log.e(TAG, "notifyError() code = " + code, err); + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.LOLLIPOP) { + if (err.getCause() instanceof ErrnoException) { + int errno = ((ErrnoException) err.getCause()).errno; + if (errno == OsConstants.ENOSPC) { + code = ERROR_INSUFFICIENT_STORAGE; + err = null; + } else if (errno == OsConstants.EACCES) { + code = ERROR_PERMISSION_DENIED; + err = null; + } + } + } + if (err instanceof IOException) { - if (!storage.canWrite() || err.getMessage().contains("Permission denied")) { + if (err.getMessage().contains("Permission denied")) { code = ERROR_PERMISSION_DENIED; err = null; - } else if (err.getMessage().contains(INSUFFICIENT_STORAGE)) { + } else if (err.getMessage().contains("ENOSPC")) { code = ERROR_INSUFFICIENT_STORAGE; err = null; + } else if (!storage.canWrite()) { + code = ERROR_FILE_CREATION; + err = null; } } From 773aa1eff006f5de824bd5739ac7e78ed2f35168 Mon Sep 17 00:00:00 2001 From: kapodamy Date: Mon, 23 Sep 2019 21:38:29 -0300 Subject: [PATCH 14/28] implement webm to ogg demuxer * used for opus audio stream * update WebMReader and WebMWriter * new post-processing algorithm --- .../newpipe/download/DownloadDialog.java | 4 +- .../newpipe/streams/OggFromWebMWriter.java | 488 ++++++++++++++++++ .../schabi/newpipe/streams/WebMReader.java | 55 +- .../schabi/newpipe/streams/WebMWriter.java | 27 +- .../postprocessing/OggFromWebmDemuxer.java | 44 ++ .../giga/postprocessing/Postprocessing.java | 6 +- 6 files changed, 595 insertions(+), 29 deletions(-) create mode 100644 app/src/main/java/org/schabi/newpipe/streams/OggFromWebMWriter.java create mode 100644 app/src/main/java/us/shandian/giga/postprocessing/OggFromWebmDemuxer.java diff --git a/app/src/main/java/org/schabi/newpipe/download/DownloadDialog.java b/app/src/main/java/org/schabi/newpipe/download/DownloadDialog.java index 59bffa933..90258a6dc 100644 --- a/app/src/main/java/org/schabi/newpipe/download/DownloadDialog.java +++ b/app/src/main/java/org/schabi/newpipe/download/DownloadDialog.java @@ -561,7 +561,7 @@ public class DownloadDialog extends DialogFragment implements RadioGroup.OnCheck mainStorage = mainStorageVideo; format = videoStreamsAdapter.getItem(selectedVideoIndex).getFormat(); mime = format.mimeType; - filename += format.suffix; + filename += format == MediaFormat.OPUS ? "ogg" : format.suffix; break; case R.id.subtitle_button: mainStorage = mainStorageVideo;// subtitle & video files go together @@ -778,6 +778,8 @@ public class DownloadDialog extends DialogFragment implements RadioGroup.OnCheck if (selectedStream.getFormat() == MediaFormat.M4A) { psName = Postprocessing.ALGORITHM_M4A_NO_DASH; + } else if (selectedStream.getFormat() == MediaFormat.OPUS) { + psName = Postprocessing.ALGORITHM_OGG_FROM_WEBM_DEMUXER; } break; case R.id.video_button: diff --git a/app/src/main/java/org/schabi/newpipe/streams/OggFromWebMWriter.java b/app/src/main/java/org/schabi/newpipe/streams/OggFromWebMWriter.java new file mode 100644 index 000000000..2b3d778c6 --- /dev/null +++ b/app/src/main/java/org/schabi/newpipe/streams/OggFromWebMWriter.java @@ -0,0 +1,488 @@ +package org.schabi.newpipe.streams; + +import android.support.annotation.NonNull; + +import org.schabi.newpipe.streams.WebMReader.Cluster; +import org.schabi.newpipe.streams.WebMReader.Segment; +import org.schabi.newpipe.streams.WebMReader.SimpleBlock; +import org.schabi.newpipe.streams.WebMReader.WebMTrack; +import org.schabi.newpipe.streams.io.SharpStream; + +import java.io.Closeable; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.util.ArrayList; +import java.util.Random; + +import javax.annotation.Nullable; + +/** + * @author kapodamy + */ +public class OggFromWebMWriter implements Closeable { + + private static final byte FLAG_UNSET = 0x00; + //private static final byte FLAG_CONTINUED = 0x01; + private static final byte FLAG_FIRST = 0x02; + private static final byte FLAG_LAST = 0x04; + + private final static byte SEGMENTS_PER_PACKET = 50;// used in ffmpeg, which is near 1 second at 48kHz + private final static byte HEADER_CHECKSUM_OFFSET = 22; + + private boolean done = false; + private boolean parsed = false; + + private SharpStream source; + private SharpStream output; + + private int sequence_count = 0; + private final int STREAM_ID; + + private WebMReader webm = null; + private WebMTrack webm_track = null; + private int track_index = 0; + + public OggFromWebMWriter(@NonNull SharpStream source, @NonNull SharpStream target) { + if (!source.canRead() || !source.canRewind()) { + throw new IllegalArgumentException("source stream must be readable and allows seeking"); + } + if (!target.canWrite() || !target.canRewind()) { + throw new IllegalArgumentException("output stream must be writable and allows seeking"); + } + + this.source = source; + this.output = target; + + this.STREAM_ID = (new Random(System.currentTimeMillis())).nextInt(); + + populate_crc32_table(); + } + + public boolean isDone() { + return done; + } + + public boolean isParsed() { + return parsed; + } + + public WebMTrack[] getTracksFromSource() throws IllegalStateException { + if (!parsed) { + throw new IllegalStateException("source must be parsed first"); + } + + return webm.getAvailableTracks(); + } + + public void parseSource() throws IOException, IllegalStateException { + if (done) { + throw new IllegalStateException("already done"); + } + if (parsed) { + throw new IllegalStateException("already parsed"); + } + + try { + webm = new WebMReader(source); + webm.parse(); + webm_segment = webm.getNextSegment(); + } finally { + parsed = true; + } + } + + public void selectTrack(int trackIndex) throws IOException { + if (!parsed) { + throw new IllegalStateException("source must be parsed first"); + } + if (done) { + throw new IOException("already done"); + } + if (webm_track != null) { + throw new IOException("tracks already selected"); + } + + switch (webm.getAvailableTracks()[trackIndex].kind) { + case Audio: + case Video: + break; + default: + throw new UnsupportedOperationException("the track must an audio or video stream"); + } + + try { + webm_track = webm.selectTrack(trackIndex); + track_index = trackIndex; + } finally { + parsed = true; + } + } + + @Override + public void close() throws IOException { + done = true; + parsed = true; + + webm_track = null; + webm = null; + + if (!output.isClosed()) { + output.flush(); + } + + source.close(); + output.close(); + } + + public void build() throws IOException { + float resolution; + int read; + byte[] buffer; + int checksum; + byte flag = FLAG_FIRST;// obligatory + + switch (webm_track.kind) { + case Audio: + resolution = getSampleFrequencyFromTrack(webm_track.bMetadata); + if (resolution == 0f) { + throw new RuntimeException("cannot get the audio sample rate"); + } + break; + case Video: + // WARNING: untested + if (webm_track.defaultDuration == 0) { + throw new RuntimeException("missing default frame time"); + } + resolution = 1000f / ((float) webm_track.defaultDuration / webm_segment.info.timecodeScale); + break; + default: + throw new RuntimeException("not implemented"); + } + + /* step 1.1: write codec init data, in most cases must be present */ + if (webm_track.codecPrivate != null) { + addPacketSegment(webm_track.codecPrivate.length); + dump_packetHeader(flag, 0x00, webm_track.codecPrivate); + flag = FLAG_UNSET; + } + + /* step 1.2: write metadata */ + buffer = make_metadata(); + if (buffer != null) { + addPacketSegment(buffer.length); + dump_packetHeader(flag, 0x00, buffer); + flag = FLAG_UNSET; + } + + buffer = new byte[8 * 1024]; + + /* step 1.3: write headers */ + long approx_packets = webm_segment.info.duration / webm_segment.info.timecodeScale; + approx_packets = approx_packets / (approx_packets / SEGMENTS_PER_PACKET); + + ArrayList pending_offsets = new ArrayList<>((int) approx_packets); + ArrayList pending_checksums = new ArrayList<>((int) approx_packets); + ArrayList data_offsets = new ArrayList<>((int) approx_packets); + + int page_size = 0; + SimpleBlock bloq; + + while (webm_segment != null) { + bloq = getNextBlock(); + + if (bloq != null && addPacketSegment(bloq.dataSize)) { + page_size += bloq.dataSize; + + if (segment_table_size < SEGMENTS_PER_PACKET) { + continue; + } + + // calculate the current packet duration using the next block + bloq = getNextBlock(); + } + + double elapsed_ns = webm_track.codecDelay; + + if (bloq == null) { + flag = FLAG_LAST; + elapsed_ns += webm_block_last_timecode; + + if (webm_track.defaultDuration > 0) { + elapsed_ns += webm_track.defaultDuration; + } else { + // hardcoded way, guess the sample duration + elapsed_ns += webm_block_near_duration; + } + } else { + elapsed_ns += bloq.absoluteTimeCodeNs; + } + + // get the sample count in the page + elapsed_ns = (elapsed_ns / 1000000000d) * resolution; + elapsed_ns = Math.ceil(elapsed_ns); + + long offset = output_offset + HEADER_CHECKSUM_OFFSET; + pending_offsets.add(offset); + + checksum = dump_packetHeader(flag, (long) elapsed_ns, null); + pending_checksums.add(checksum); + + data_offsets.add((short) (output_offset - offset)); + + // reserve space in the page + while (page_size > 0) { + int write = Math.min(page_size, buffer.length); + out_write(buffer, write); + page_size -= write; + } + + webm_block = bloq; + } + + /* step 2.1: write stream data */ + output.rewind(); + output_offset = 0; + + source.rewind(); + + webm = new WebMReader(source); + webm.parse(); + webm_track = webm.selectTrack(track_index); + + for (int i = 0; i < pending_offsets.size(); i++) { + checksum = pending_checksums.get(i); + segment_table_size = 0; + + out_seek(pending_offsets.get(i) + data_offsets.get(i)); + + while (segment_table_size < SEGMENTS_PER_PACKET) { + bloq = getNextBlock(); + + if (bloq == null || !addPacketSegment(bloq.dataSize)) { + webm_block = bloq;// use this block later (if not null) + break; + } + + // NOTE: calling bloq.data.close() is unnecessary + while ((read = bloq.data.read(buffer)) != -1) { + out_write(buffer, read); + checksum = calc_crc32(checksum, buffer, read); + } + } + + pending_checksums.set(i, checksum); + } + + /* step 2.2: write every checksum */ + output.rewind(); + output_offset = 0; + buffer = new byte[4]; + + ByteBuffer buff = ByteBuffer.wrap(buffer); + buff.order(ByteOrder.LITTLE_ENDIAN); + + for (int i = 0; i < pending_checksums.size(); i++) { + out_seek(pending_offsets.get(i)); + buff.putInt(0, pending_checksums.get(i)); + out_write(buffer); + } + } + + private int dump_packetHeader(byte flag, long gran_pos, byte[] immediate_page) throws IOException { + ByteBuffer buffer = ByteBuffer.allocate(27 + segment_table_size); + + buffer.putInt(0x4F676753);// "OggS" binary string + buffer.put((byte) 0x00);// version + buffer.put(flag);// type + + buffer.order(ByteOrder.LITTLE_ENDIAN); + + buffer.putLong(gran_pos);// granulate position + + buffer.putInt(STREAM_ID);// bitstream serial number + buffer.putInt(sequence_count++);// page sequence number + + buffer.putInt(0x00);// page checksum + + buffer.order(ByteOrder.BIG_ENDIAN); + + buffer.put((byte) segment_table_size);// segment table + buffer.put(segment_table, 0, segment_table_size);// segment size + + segment_table_size = 0;// clear segment table for next header + + byte[] buff = buffer.array(); + int checksum_crc32 = calc_crc32(0x00, buff, buff.length); + + if (immediate_page != null) { + checksum_crc32 = calc_crc32(checksum_crc32, immediate_page, immediate_page.length); + buffer.order(ByteOrder.LITTLE_ENDIAN); + buffer.putInt(HEADER_CHECKSUM_OFFSET, checksum_crc32); + + out_write(buff); + out_write(immediate_page); + return 0; + } + + out_write(buff); + return checksum_crc32; + } + + @Nullable + private byte[] make_metadata() { + if ("A_OPUS".equals(webm_track.codecId)) { + return new byte[]{ + 0x4F, 0x70, 0x75, 0x73, 0x54, 0x61, 0x67, 0x73,// "OpusTags" binary string + 0x07, 0x00, 0x00, 0x00,// writting application string size + 0x4E, 0x65, 0x77, 0x50, 0x69, 0x70, 0x65,// "NewPipe" binary string + 0x00, 0x00, 0x00, 0x00// additional tags count (zero means no tags) + }; + } else if ("A_VORBIS".equals(webm_track.codecId)) { + return new byte[]{ + 0x03,// ???????? + 0x76, 0x6f, 0x72, 0x62, 0x69, 0x73,// "vorbis" binary string + 0x07, 0x00, 0x00, 0x00,// writting application string size + 0x4E, 0x65, 0x77, 0x50, 0x69, 0x70, 0x65,// "NewPipe" binary string + 0x01, 0x00, 0x00, 0x00,// additional tags count (zero means no tags) + + /* + // whole file duration (not implemented) + 0x44,// tag string size + 0x55, 0x52, 0x41, 0x54, 0x49, 0x4F, 0x4E, 0x3D, 0x30, 0x30, 0x3A, 0x30, 0x30, 0x3A, 0x30, + 0x30, 0x2E, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30 + */ + 0x0F,// tag string size + 0x00, 0x00, 0x00, 0x45, 0x4E, 0x43, 0x4F, 0x44, 0x45, 0x52, 0x3D,// "ENCODER=" binary string + 0x4E, 0x65, 0x77, 0x50, 0x69, 0x70, 0x65,// "NewPipe" binary string + 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00// ???????? + }; + } + + // not implemented for the desired codec + return null; + } + + // + private Segment webm_segment = null; + private Cluster webm_cluter = null; + private SimpleBlock webm_block = null; + private long webm_block_last_timecode = 0; + private long webm_block_near_duration = 0; + + private SimpleBlock getNextBlock() throws IOException { + SimpleBlock res; + + if (webm_block != null) { + res = webm_block; + webm_block = null; + return res; + } + + if (webm_segment == null) { + webm_segment = webm.getNextSegment(); + if (webm_segment == null) { + return null;// no more blocks in the selected track + } + } + + if (webm_cluter == null) { + webm_cluter = webm_segment.getNextCluster(); + if (webm_cluter == null) { + webm_segment = null; + return getNextBlock(); + } + } + + res = webm_cluter.getNextSimpleBlock(); + if (res == null) { + webm_cluter = null; + return getNextBlock(); + } + + webm_block_near_duration = res.absoluteTimeCodeNs - webm_block_last_timecode; + webm_block_last_timecode = res.absoluteTimeCodeNs; + + return res; + } + + private float getSampleFrequencyFromTrack(byte[] bMetadata) { + // hardcoded way + ByteBuffer buffer = ByteBuffer.wrap(bMetadata); + + while (buffer.remaining() >= 6) { + int id = buffer.getShort() & 0xFFFF; + if (id == 0x0000B584) { + return buffer.getFloat(); + } + } + + return 0f; + } + // + + // + private int segment_table_size = 0; + private final byte[] segment_table = new byte[255]; + + private boolean addPacketSegment(long size) { + // check if possible add the segment, without overflow the table + int available = (segment_table.length - segment_table_size) * 255; + if (available < size) { + return false;// not enough space on the page + } + + while (size > 0) { + segment_table[segment_table_size++] = (byte) Math.min(size, 255); + size -= 255; + } + + return true; + } + // + + // + private long output_offset = 0; + + private void out_write(byte[] buffer) throws IOException { + output.write(buffer); + output_offset += buffer.length; + } + + private void out_write(byte[] buffer, int size) throws IOException { + output.write(buffer, 0, size); + output_offset += size; + } + + private void out_seek(long offset) throws IOException { + //if (output.canSeek()) { output.seek(offset); } + output.skip(offset - output_offset); + output_offset = offset; + } + // + + // + private final int[] crc32_table = new int[256]; + + private void populate_crc32_table() { + for (int i = 0; i < 0x100; i++) { + int crc = i << 24; + for (int j = 0; j < 8; j++) { + long b = crc >>> 31; + crc <<= 1; + crc ^= (int) (0x100000000L - b) & 0x04c11db7; + } + crc32_table[i] = crc; + } + } + + private int calc_crc32(int initial_crc, byte[] buffer, int size) { + for (int i = 0; i < size; i++) { + int reg = (initial_crc >>> 24) & 0xff; + initial_crc = (initial_crc << 8) ^ crc32_table[reg ^ (buffer[i] & 0xff)]; + } + + return initial_crc; + } + // +} diff --git a/app/src/main/java/org/schabi/newpipe/streams/WebMReader.java b/app/src/main/java/org/schabi/newpipe/streams/WebMReader.java index 0c635ebe3..13c15370d 100644 --- a/app/src/main/java/org/schabi/newpipe/streams/WebMReader.java +++ b/app/src/main/java/org/schabi/newpipe/streams/WebMReader.java @@ -15,7 +15,7 @@ import java.util.NoSuchElementException; */ public class WebMReader { - // + // private final static int ID_EMBL = 0x0A45DFA3; private final static int ID_EMBLReadVersion = 0x02F7; private final static int ID_EMBLDocType = 0x0282; @@ -37,10 +37,13 @@ public class WebMReader { private final static int ID_Audio = 0x61; private final static int ID_DefaultDuration = 0x3E383; private final static int ID_FlagLacing = 0x1C; + private final static int ID_CodecDelay = 0x16AA; private final static int ID_Cluster = 0x0F43B675; private final static int ID_Timecode = 0x67; private final static int ID_SimpleBlock = 0x23; + private final static int ID_Block = 0x21; + private final static int ID_GroupBlock = 0x20; // public enum TrackKind { @@ -96,7 +99,7 @@ public class WebMReader { } ensure(segment.ref); - + // WARNING: track cannot be the same or have different index in new segments Element elem = untilElement(null, ID_Segment); if (elem == null) { done = true; @@ -189,6 +192,9 @@ public class WebMReader { Element elem; while (ref == null ? stream.available() : (stream.position() < (ref.offset + ref.size))) { elem = readElement(); + if (expected.length < 1) { + return elem; + } for (int type : expected) { if (elem.type == type) { return elem; @@ -300,9 +306,7 @@ public class WebMReader { WebMTrack entry = new WebMTrack(); boolean drop = false; Element elem; - while ((elem = untilElement(elem_trackEntry, - ID_TrackNumber, ID_TrackType, ID_CodecID, ID_CodecPrivate, ID_FlagLacing, ID_DefaultDuration, ID_Audio, ID_Video - )) != null) { + while ((elem = untilElement(elem_trackEntry)) != null) { switch (elem.type) { case ID_TrackNumber: entry.trackNumber = readNumber(elem); @@ -326,8 +330,9 @@ public class WebMReader { case ID_FlagLacing: drop = readNumber(elem) != lacingExpected; break; + case ID_CodecDelay: + entry.codecDelay = readNumber(elem); default: - System.out.println(); break; } ensure(elem); @@ -360,12 +365,13 @@ public class WebMReader { private SimpleBlock readSimpleBlock(Element ref) throws IOException { SimpleBlock obj = new SimpleBlock(ref); - obj.dataSize = stream.position(); obj.trackNumber = readEncodedNumber(); obj.relativeTimeCode = stream.readShort(); obj.flags = (byte) stream.read(); obj.dataSize = (ref.offset + ref.size) - stream.position(); + obj.createdFromBlock = ref.type == ID_Block; + // NOTE: lacing is not implemented, and will be mixed with the stream data if (obj.dataSize < 0) { throw new IOException(String.format("Unexpected SimpleBlock element size, missing %s bytes", -obj.dataSize)); } @@ -409,6 +415,7 @@ public class WebMReader { public byte[] bMetadata; public TrackKind kind; public long defaultDuration; + public long codecDelay; } public class Segment { @@ -448,6 +455,7 @@ public class WebMReader { public class SimpleBlock { public InputStream data; + public boolean createdFromBlock; SimpleBlock(Element ref) { this.ref = ref; @@ -455,6 +463,7 @@ public class WebMReader { public long trackNumber; public short relativeTimeCode; + public long absoluteTimeCodeNs; public byte flags; public long dataSize; private final Element ref; @@ -468,33 +477,55 @@ public class WebMReader { Element ref; SimpleBlock currentSimpleBlock = null; + Element currentBlockGroup = null; public long timecode; Cluster(Element ref) { this.ref = ref; } - boolean check() { + boolean insideClusterBounds() { return stream.position() >= (ref.offset + ref.size); } public SimpleBlock getNextSimpleBlock() throws IOException { - if (check()) { + if (insideClusterBounds()) { return null; } - if (currentSimpleBlock != null) { + + if (currentBlockGroup != null) { + ensure(currentBlockGroup); + currentBlockGroup = null; + currentSimpleBlock = null; + } else if (currentSimpleBlock != null) { ensure(currentSimpleBlock.ref); } - while (!check()) { - Element elem = untilElement(ref, ID_SimpleBlock); + while (!insideClusterBounds()) { + Element elem = untilElement(ref, ID_SimpleBlock, ID_GroupBlock); if (elem == null) { return null; } + if (elem.type == ID_GroupBlock) { + currentBlockGroup = elem; + elem = untilElement(currentBlockGroup, ID_Block); + + if (elem == null) { + ensure(currentBlockGroup); + currentBlockGroup = null; + continue; + } + } + currentSimpleBlock = readSimpleBlock(elem); if (currentSimpleBlock.trackNumber == tracks[selectedTrack].trackNumber) { currentSimpleBlock.data = stream.getView((int) currentSimpleBlock.dataSize); + + // calculate the timestamp in nanoseconds + currentSimpleBlock.absoluteTimeCodeNs = currentSimpleBlock.relativeTimeCode + this.timecode; + currentSimpleBlock.absoluteTimeCodeNs *= segment.info.timecodeScale; + return currentSimpleBlock; } diff --git a/app/src/main/java/org/schabi/newpipe/streams/WebMWriter.java b/app/src/main/java/org/schabi/newpipe/streams/WebMWriter.java index e5881fd0b..1bf994b1e 100644 --- a/app/src/main/java/org/schabi/newpipe/streams/WebMWriter.java +++ b/app/src/main/java/org/schabi/newpipe/streams/WebMWriter.java @@ -8,6 +8,7 @@ import org.schabi.newpipe.streams.WebMReader.SimpleBlock; import org.schabi.newpipe.streams.WebMReader.WebMTrack; import org.schabi.newpipe.streams.io.SharpStream; +import java.io.Closeable; import java.io.IOException; import java.io.InputStream; import java.nio.ByteBuffer; @@ -17,7 +18,7 @@ import java.util.ArrayList; /** * @author kapodamy */ -public class WebMWriter { +public class WebMWriter implements Closeable { private final static int BUFFER_SIZE = 8 * 1024; private final static int DEFAULT_TIMECODE_SCALE = 1000000; @@ -35,7 +36,7 @@ public class WebMWriter { private long written = 0; private Segment[] readersSegment; - private Cluster[] readersCluter; + private Cluster[] readersCluster; private int[] predefinedDurations; @@ -81,7 +82,7 @@ public class WebMWriter { public void selectTracks(int... trackIndex) throws IOException { try { readersSegment = new Segment[readers.length]; - readersCluter = new Cluster[readers.length]; + readersCluster = new Cluster[readers.length]; predefinedDurations = new int[readers.length]; for (int i = 0; i < readers.length; i++) { @@ -102,6 +103,7 @@ public class WebMWriter { return parsed; } + @Override public void close() { done = true; parsed = true; @@ -114,7 +116,7 @@ public class WebMWriter { readers = null; infoTracks = null; readersSegment = null; - readersCluter = null; + readersCluster = null; outBuffer = null; } @@ -334,17 +336,17 @@ public class WebMWriter { } } - if (readersCluter[internalTrackId] == null) { - readersCluter[internalTrackId] = readersSegment[internalTrackId].getNextCluster(); - if (readersCluter[internalTrackId] == null) { + if (readersCluster[internalTrackId] == null) { + readersCluster[internalTrackId] = readersSegment[internalTrackId].getNextCluster(); + if (readersCluster[internalTrackId] == null) { readersSegment[internalTrackId] = null; return getNextBlockFrom(internalTrackId); } } - SimpleBlock res = readersCluter[internalTrackId].getNextSimpleBlock(); + SimpleBlock res = readersCluster[internalTrackId].getNextSimpleBlock(); if (res == null) { - readersCluter[internalTrackId] = null; + readersCluster[internalTrackId] = null; return new Block();// fake block to indicate the end of the cluster } @@ -353,16 +355,11 @@ public class WebMWriter { bloq.dataSize = (int) res.dataSize; bloq.trackNumber = internalTrackId; bloq.flags = res.flags; - bloq.absoluteTimecode = convertTimecode(res.relativeTimeCode, readersSegment[internalTrackId].info.timecodeScale); - bloq.absoluteTimecode += readersCluter[internalTrackId].timecode; + bloq.absoluteTimecode = res.absoluteTimeCodeNs / DEFAULT_TIMECODE_SCALE; return bloq; } - private short convertTimecode(int time, long oldTimeScale) { - return (short) (time * (DEFAULT_TIMECODE_SCALE / oldTimeScale)); - } - private void seekTo(SharpStream stream, long offset) throws IOException { if (stream.canSeek()) { stream.seek(offset); diff --git a/app/src/main/java/us/shandian/giga/postprocessing/OggFromWebmDemuxer.java b/app/src/main/java/us/shandian/giga/postprocessing/OggFromWebmDemuxer.java new file mode 100644 index 000000000..65aa30fa3 --- /dev/null +++ b/app/src/main/java/us/shandian/giga/postprocessing/OggFromWebmDemuxer.java @@ -0,0 +1,44 @@ +package us.shandian.giga.postprocessing; + +import android.support.annotation.NonNull; + +import org.schabi.newpipe.streams.OggFromWebMWriter; +import org.schabi.newpipe.streams.io.SharpStream; + +import java.io.IOException; +import java.nio.ByteBuffer; + +class OggFromWebmDemuxer extends Postprocessing { + + OggFromWebmDemuxer() { + super(false, true, ALGORITHM_OGG_FROM_WEBM_DEMUXER); + } + + @Override + boolean test(SharpStream... sources) throws IOException { + ByteBuffer buffer = ByteBuffer.allocate(4); + sources[0].read(buffer.array()); + + // youtube uses WebM as container, but the file extension (format suffix) is "*.opus" + // check if the file is a webm/mkv file before proceed + + switch (buffer.getInt()) { + case 0x1a45dfa3: + return true;// webm + case 0x4F676753: + return false;// ogg + } + + throw new UnsupportedOperationException("file not recognized, failed to demux the audio stream"); + } + + @Override + int process(SharpStream out, @NonNull SharpStream... sources) throws IOException { + OggFromWebMWriter demuxer = new OggFromWebMWriter(sources[0], out); + demuxer.parseSource(); + demuxer.selectTrack(0); + demuxer.build(); + + return OK_RESULT; + } +} diff --git a/app/src/main/java/us/shandian/giga/postprocessing/Postprocessing.java b/app/src/main/java/us/shandian/giga/postprocessing/Postprocessing.java index 22cc325d5..92510c3df 100644 --- a/app/src/main/java/us/shandian/giga/postprocessing/Postprocessing.java +++ b/app/src/main/java/us/shandian/giga/postprocessing/Postprocessing.java @@ -28,6 +28,7 @@ public abstract class Postprocessing implements Serializable { public transient static final String ALGORITHM_WEBM_MUXER = "webm"; public transient static final String ALGORITHM_MP4_FROM_DASH_MUXER = "mp4D-mp4"; public transient static final String ALGORITHM_M4A_NO_DASH = "mp4D-m4a"; + public transient static final String ALGORITHM_OGG_FROM_WEBM_DEMUXER = "webm-ogg-d"; public static Postprocessing getAlgorithm(@NonNull String algorithmName, String[] args) { Postprocessing instance; @@ -45,6 +46,9 @@ public abstract class Postprocessing implements Serializable { case ALGORITHM_M4A_NO_DASH: instance = new M4aNoDash(); break; + case ALGORITHM_OGG_FROM_WEBM_DEMUXER: + instance = new OggFromWebmDemuxer(); + break; /*case "example-algorithm": instance = new ExampleAlgorithm();*/ default: @@ -212,7 +216,7 @@ public abstract class Postprocessing implements Serializable { * * @param out output stream * @param sources files to be processed - * @return a error code, 0 means the operation was successful + * @return an error code, {@code OK_RESULT} means the operation was successful * @throws IOException if an I/O error occurs. */ abstract int process(SharpStream out, SharpStream... sources) throws IOException; From dab53450c13ad1c7ddf58097541f12681ddbdb39 Mon Sep 17 00:00:00 2001 From: kapodamy Date: Wed, 25 Sep 2019 16:24:52 -0300 Subject: [PATCH 15/28] rewrite OggFromWebMWriter * reduce the number of iterations over the output file (less seeking) * fix audio samples with size of 255 do not handled correctly in the segment table (allows writing audio streams with 70kbps and 160kbps bitrate) * add support for VORBIS codec metadata * write packets based on the timestamp --- .../newpipe/streams/OggFromWebMWriter.java | 348 ++++++++++-------- .../postprocessing/OggFromWebmDemuxer.java | 4 +- 2 files changed, 203 insertions(+), 149 deletions(-) diff --git a/app/src/main/java/org/schabi/newpipe/streams/OggFromWebMWriter.java b/app/src/main/java/org/schabi/newpipe/streams/OggFromWebMWriter.java index 2b3d778c6..091ae6d2a 100644 --- a/app/src/main/java/org/schabi/newpipe/streams/OggFromWebMWriter.java +++ b/app/src/main/java/org/schabi/newpipe/streams/OggFromWebMWriter.java @@ -23,12 +23,16 @@ import javax.annotation.Nullable; public class OggFromWebMWriter implements Closeable { private static final byte FLAG_UNSET = 0x00; - //private static final byte FLAG_CONTINUED = 0x01; + private static final byte FLAG_CONTINUED = 0x01; private static final byte FLAG_FIRST = 0x02; private static final byte FLAG_LAST = 0x04; - private final static byte SEGMENTS_PER_PACKET = 50;// used in ffmpeg, which is near 1 second at 48kHz private final static byte HEADER_CHECKSUM_OFFSET = 22; + private final static byte HEADER_SIZE = 27; + + private final static short BUFFER_SIZE = 8 * 1024;// 8KiB + + private final static int TIME_SCALE_NS = 1000000000; private boolean done = false; private boolean parsed = false; @@ -38,10 +42,23 @@ public class OggFromWebMWriter implements Closeable { private int sequence_count = 0; private final int STREAM_ID; + private byte packet_flag = FLAG_FIRST; + private int track_index = 0; private WebMReader webm = null; private WebMTrack webm_track = null; - private int track_index = 0; + private Segment webm_segment = null; + private Cluster webm_cluster = null; + private SimpleBlock webm_block = null; + + private long webm_block_last_timecode = 0; + private long webm_block_near_duration = 0; + + private short segment_table_size = 0; + private final byte[] segment_table = new byte[255]; + private long segment_table_next_timestamp = TIME_SCALE_NS; + + private final int[] crc32_table = new int[256]; public OggFromWebMWriter(@NonNull SharpStream source, @NonNull SharpStream target) { if (!source.canRead() || !source.canRewind()) { @@ -139,9 +156,8 @@ public class OggFromWebMWriter implements Closeable { float resolution; int read; byte[] buffer; - int checksum; - byte flag = FLAG_FIRST;// obligatory + /* step 1: get the amount of frames per seconds */ switch (webm_track.kind) { case Audio: resolution = getSampleFrequencyFromTrack(webm_track.bMetadata); @@ -160,52 +176,65 @@ public class OggFromWebMWriter implements Closeable { throw new RuntimeException("not implemented"); } - /* step 1.1: write codec init data, in most cases must be present */ + /* step 2a: create packet with code init data */ + ArrayList data_extra = new ArrayList<>(4); + if (webm_track.codecPrivate != null) { addPacketSegment(webm_track.codecPrivate.length); - dump_packetHeader(flag, 0x00, webm_track.codecPrivate); - flag = FLAG_UNSET; + ByteBuffer buff = byte_buffer(HEADER_SIZE + segment_table_size + webm_track.codecPrivate.length); + + make_packetHeader(0x00, buff, webm_track.codecPrivate); + data_extra.add(buff.array()); } - /* step 1.2: write metadata */ + /* step 2b: create packet with metadata */ buffer = make_metadata(); if (buffer != null) { addPacketSegment(buffer.length); - dump_packetHeader(flag, 0x00, buffer); - flag = FLAG_UNSET; + ByteBuffer buff = byte_buffer(HEADER_SIZE + segment_table_size + buffer.length); + + make_packetHeader(0x00, buff, buffer); + data_extra.add(buff.array()); } - buffer = new byte[8 * 1024]; - /* step 1.3: write headers */ - long approx_packets = webm_segment.info.duration / webm_segment.info.timecodeScale; - approx_packets = approx_packets / (approx_packets / SEGMENTS_PER_PACKET); - - ArrayList pending_offsets = new ArrayList<>((int) approx_packets); - ArrayList pending_checksums = new ArrayList<>((int) approx_packets); - ArrayList data_offsets = new ArrayList<>((int) approx_packets); - - int page_size = 0; + /* step 3: calculate amount of packets */ SimpleBlock bloq; + int reserve_header = 0; + int headers_amount = 0; while (webm_segment != null) { bloq = getNextBlock(); - if (bloq != null && addPacketSegment(bloq.dataSize)) { - page_size += bloq.dataSize; - - if (segment_table_size < SEGMENTS_PER_PACKET) { - continue; - } - - // calculate the current packet duration using the next block - bloq = getNextBlock(); + if (addPacketSegment(bloq)) { + continue; } + reserve_header += HEADER_SIZE + segment_table_size;// header size + clearSegmentTable(); + webm_block = bloq; + headers_amount++; + } + + /* step 4: create packet headers */ + rewind_source(); + + ByteBuffer headers = byte_buffer(reserve_header); + short[] headers_size = new short[headers_amount]; + int header_index = 0; + + while (webm_segment != null) { + bloq = getNextBlock(); + + if (addPacketSegment(bloq)) { + continue; + } + + // calculate the current packet duration using the next block double elapsed_ns = webm_track.codecDelay; if (bloq == null) { - flag = FLAG_LAST; + packet_flag = FLAG_LAST;// note: if the flag is FLAG_CONTINUED, is changed elapsed_ns += webm_block_last_timecode; if (webm_track.defaultDuration > 0) { @@ -219,84 +248,83 @@ public class OggFromWebMWriter implements Closeable { } // get the sample count in the page - elapsed_ns = (elapsed_ns / 1000000000d) * resolution; - elapsed_ns = Math.ceil(elapsed_ns); - - long offset = output_offset + HEADER_CHECKSUM_OFFSET; - pending_offsets.add(offset); - - checksum = dump_packetHeader(flag, (long) elapsed_ns, null); - pending_checksums.add(checksum); - - data_offsets.add((short) (output_offset - offset)); - - // reserve space in the page - while (page_size > 0) { - int write = Math.min(page_size, buffer.length); - out_write(buffer, write); - page_size -= write; - } + elapsed_ns = elapsed_ns / TIME_SCALE_NS; + elapsed_ns = Math.ceil(elapsed_ns * resolution); + // create header + headers_size[header_index++] = make_packetHeader((long) elapsed_ns, headers, null); webm_block = bloq; } - /* step 2.1: write stream data */ - output.rewind(); - output_offset = 0; - source.rewind(); + /* step 5: calculate checksums */ + rewind_source(); - webm = new WebMReader(source); - webm.parse(); - webm_track = webm.selectTrack(track_index); + int offset = 0; + buffer = new byte[BUFFER_SIZE]; - for (int i = 0; i < pending_offsets.size(); i++) { - checksum = pending_checksums.get(i); - segment_table_size = 0; + for (header_index = 0; header_index < headers_size.length; header_index++) { + int checksum_offset = offset + HEADER_CHECKSUM_OFFSET; + int checksum = headers.getInt(checksum_offset); - out_seek(pending_offsets.get(i) + data_offsets.get(i)); - - while (segment_table_size < SEGMENTS_PER_PACKET) { + while (webm_segment != null) { bloq = getNextBlock(); - if (bloq == null || !addPacketSegment(bloq.dataSize)) { - webm_block = bloq;// use this block later (if not null) + if (!addPacketSegment(bloq)) { + clearSegmentTable(); + webm_block = bloq; break; } - // NOTE: calling bloq.data.close() is unnecessary - while ((read = bloq.data.read(buffer)) != -1) { - out_write(buffer, read); - checksum = calc_crc32(checksum, buffer, read); + // calculate page checksum + while ((read = bloq.data.read(buffer)) > 0) { + checksum = calc_crc32(checksum, buffer, 0, read); } } - pending_checksums.set(i, checksum); + headers.putInt(checksum_offset, checksum); + offset += headers_size[header_index]; } - /* step 2.2: write every checksum */ - output.rewind(); - output_offset = 0; - buffer = new byte[4]; + /* step 6: write extra headers */ + rewind_source(); - ByteBuffer buff = ByteBuffer.wrap(buffer); - buff.order(ByteOrder.LITTLE_ENDIAN); + for (byte[] buff : data_extra) { + output.write(buff); + } - for (int i = 0; i < pending_checksums.size(); i++) { - out_seek(pending_offsets.get(i)); - buff.putInt(0, pending_checksums.get(i)); - out_write(buffer); + /* step 7: write stream packets */ + byte[] headers_buffers = headers.array(); + offset = 0; + buffer = new byte[BUFFER_SIZE]; + + for (header_index = 0; header_index < headers_size.length; header_index++) { + output.write(headers_buffers, offset, headers_size[header_index]); + offset += headers_size[header_index]; + + while (webm_segment != null) { + bloq = getNextBlock(); + + if (addPacketSegment(bloq)) { + while ((read = bloq.data.read(buffer)) > 0) { + output.write(buffer, 0, read); + } + } else { + clearSegmentTable(); + webm_block = bloq; + break; + } + } } } - private int dump_packetHeader(byte flag, long gran_pos, byte[] immediate_page) throws IOException { - ByteBuffer buffer = ByteBuffer.allocate(27 + segment_table_size); + private short make_packetHeader(long gran_pos, ByteBuffer buffer, byte[] immediate_page) { + int offset = buffer.position(); + short length = HEADER_SIZE; - buffer.putInt(0x4F676753);// "OggS" binary string + buffer.putInt(0x5367674f);// "OggS" binary string in little-endian buffer.put((byte) 0x00);// version - buffer.put(flag);// type - - buffer.order(ByteOrder.LITTLE_ENDIAN); + buffer.put(packet_flag);// type buffer.putLong(gran_pos);// granulate position @@ -305,28 +333,24 @@ public class OggFromWebMWriter implements Closeable { buffer.putInt(0x00);// page checksum - buffer.order(ByteOrder.BIG_ENDIAN); - buffer.put((byte) segment_table_size);// segment table buffer.put(segment_table, 0, segment_table_size);// segment size - segment_table_size = 0;// clear segment table for next header + length += segment_table_size; - byte[] buff = buffer.array(); - int checksum_crc32 = calc_crc32(0x00, buff, buff.length); + clearSegmentTable();// clear segment table for next header + + int checksum_crc32 = calc_crc32(0x00, buffer.array(), offset, length); if (immediate_page != null) { - checksum_crc32 = calc_crc32(checksum_crc32, immediate_page, immediate_page.length); - buffer.order(ByteOrder.LITTLE_ENDIAN); - buffer.putInt(HEADER_CHECKSUM_OFFSET, checksum_crc32); - - out_write(buff); - out_write(immediate_page); - return 0; + checksum_crc32 = calc_crc32(checksum_crc32, immediate_page, 0, immediate_page.length); + System.arraycopy(immediate_page, 0, buffer.array(), length, immediate_page.length); + segment_table_next_timestamp -= TIME_SCALE_NS; } - out_write(buff); - return checksum_crc32; + buffer.putInt(offset + HEADER_CHECKSUM_OFFSET, checksum_crc32); + + return length; } @Nullable @@ -334,7 +358,7 @@ public class OggFromWebMWriter implements Closeable { if ("A_OPUS".equals(webm_track.codecId)) { return new byte[]{ 0x4F, 0x70, 0x75, 0x73, 0x54, 0x61, 0x67, 0x73,// "OpusTags" binary string - 0x07, 0x00, 0x00, 0x00,// writting application string size + 0x07, 0x00, 0x00, 0x00,// writing application string size 0x4E, 0x65, 0x77, 0x50, 0x69, 0x70, 0x65,// "NewPipe" binary string 0x00, 0x00, 0x00, 0x00// additional tags count (zero means no tags) }; @@ -342,15 +366,15 @@ public class OggFromWebMWriter implements Closeable { return new byte[]{ 0x03,// ???????? 0x76, 0x6f, 0x72, 0x62, 0x69, 0x73,// "vorbis" binary string - 0x07, 0x00, 0x00, 0x00,// writting application string size + 0x07, 0x00, 0x00, 0x00,// writing application string size 0x4E, 0x65, 0x77, 0x50, 0x69, 0x70, 0x65,// "NewPipe" binary string 0x01, 0x00, 0x00, 0x00,// additional tags count (zero means no tags) /* - // whole file duration (not implemented) - 0x44,// tag string size - 0x55, 0x52, 0x41, 0x54, 0x49, 0x4F, 0x4E, 0x3D, 0x30, 0x30, 0x3A, 0x30, 0x30, 0x3A, 0x30, - 0x30, 0x2E, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30 + // whole file duration (not implemented) + 0x44,// tag string size + 0x55, 0x52, 0x41, 0x54, 0x49, 0x4F, 0x4E, 0x3D, 0x30, 0x30, 0x3A, 0x30, 0x30, 0x3A, 0x30, + 0x30, 0x2E, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30 */ 0x0F,// tag string size 0x00, 0x00, 0x00, 0x45, 0x4E, 0x43, 0x4F, 0x44, 0x45, 0x52, 0x3D,// "ENCODER=" binary string @@ -363,13 +387,26 @@ public class OggFromWebMWriter implements Closeable { return null; } - // - private Segment webm_segment = null; - private Cluster webm_cluter = null; - private SimpleBlock webm_block = null; - private long webm_block_last_timecode = 0; - private long webm_block_near_duration = 0; + private void rewind_source() throws IOException { + source.rewind(); + webm = new WebMReader(source); + webm.parse(); + webm_track = webm.selectTrack(track_index); + webm_segment = webm.getNextSegment(); + webm_cluster = null; + webm_block = null; + webm_block_last_timecode = 0L; + + segment_table_next_timestamp = TIME_SCALE_NS; + } + + private ByteBuffer byte_buffer(int size) { + return ByteBuffer.allocate(size).order(ByteOrder.LITTLE_ENDIAN); + } + + // + @Nullable private SimpleBlock getNextBlock() throws IOException { SimpleBlock res; @@ -386,17 +423,17 @@ public class OggFromWebMWriter implements Closeable { } } - if (webm_cluter == null) { - webm_cluter = webm_segment.getNextCluster(); - if (webm_cluter == null) { + if (webm_cluster == null) { + webm_cluster = webm_segment.getNextCluster(); + if (webm_cluster == null) { webm_segment = null; return getNextBlock(); } } - res = webm_cluter.getNextSimpleBlock(); + res = webm_cluster.getNextSimpleBlock(); if (res == null) { - webm_cluter = null; + webm_cluster = null; return getNextBlock(); } @@ -421,49 +458,64 @@ public class OggFromWebMWriter implements Closeable { } // - // - private int segment_table_size = 0; - private final byte[] segment_table = new byte[255]; + // + private void clearSegmentTable() { + if (packet_flag != FLAG_CONTINUED) { + segment_table_next_timestamp += TIME_SCALE_NS; + packet_flag = FLAG_UNSET; + } + segment_table_size = 0; + } - private boolean addPacketSegment(long size) { - // check if possible add the segment, without overflow the table + private boolean addPacketSegment(SimpleBlock block) { + if (block == null) { + return false; + } + + long timestamp = block.absoluteTimeCodeNs + webm_track.codecDelay; + + if (timestamp >= segment_table_next_timestamp) { + return false; + } + + boolean result = addPacketSegment((int) block.dataSize); + + if (!result && segment_table_next_timestamp < timestamp) { + // WARNING: ¡¡¡¡ not implemented (lack of documentation) !!!! + packet_flag = FLAG_CONTINUED; + } + + return result; + } + + private boolean addPacketSegment(int size) { int available = (segment_table.length - segment_table_size) * 255; + boolean extra = size == 255; + + if (extra) { + // add a zero byte entry in the table + // required to indicate the sample size is exactly 255 + available -= 255; + } + + // check if possible add the segment, without overflow the table if (available < size) { return false;// not enough space on the page } - while (size > 0) { + for (; size > 0; size -= 255) { segment_table[segment_table_size++] = (byte) Math.min(size, 255); - size -= 255; + } + + if (extra) { + segment_table[segment_table_size++] = 0x00; } return true; } // - // - private long output_offset = 0; - - private void out_write(byte[] buffer) throws IOException { - output.write(buffer); - output_offset += buffer.length; - } - - private void out_write(byte[] buffer, int size) throws IOException { - output.write(buffer, 0, size); - output_offset += size; - } - - private void out_seek(long offset) throws IOException { - //if (output.canSeek()) { output.seek(offset); } - output.skip(offset - output_offset); - output_offset = offset; - } - // - // - private final int[] crc32_table = new int[256]; - private void populate_crc32_table() { for (int i = 0; i < 0x100; i++) { int crc = i << 24; @@ -476,10 +528,12 @@ public class OggFromWebMWriter implements Closeable { } } - private int calc_crc32(int initial_crc, byte[] buffer, int size) { - for (int i = 0; i < size; i++) { + private int calc_crc32(int initial_crc, byte[] buffer, int offset, int size) { + size += offset; + + for (; offset < size; offset++) { int reg = (initial_crc >>> 24) & 0xff; - initial_crc = (initial_crc << 8) ^ crc32_table[reg ^ (buffer[i] & 0xff)]; + initial_crc = (initial_crc << 8) ^ crc32_table[reg ^ (buffer[offset] & 0xff)]; } return initial_crc; diff --git a/app/src/main/java/us/shandian/giga/postprocessing/OggFromWebmDemuxer.java b/app/src/main/java/us/shandian/giga/postprocessing/OggFromWebmDemuxer.java index 65aa30fa3..605c0a88b 100644 --- a/app/src/main/java/us/shandian/giga/postprocessing/OggFromWebmDemuxer.java +++ b/app/src/main/java/us/shandian/giga/postprocessing/OggFromWebmDemuxer.java @@ -11,7 +11,7 @@ import java.nio.ByteBuffer; class OggFromWebmDemuxer extends Postprocessing { OggFromWebmDemuxer() { - super(false, true, ALGORITHM_OGG_FROM_WEBM_DEMUXER); + super(true, true, ALGORITHM_OGG_FROM_WEBM_DEMUXER); } @Override @@ -24,7 +24,7 @@ class OggFromWebmDemuxer extends Postprocessing { switch (buffer.getInt()) { case 0x1a45dfa3: - return true;// webm + return true;// webm/mkv case 0x4F676753: return false;// ogg } From 86dafdd92b200e295494e335d024f99d950af6cb Mon Sep 17 00:00:00 2001 From: kapodamy Date: Sat, 28 Sep 2019 18:11:05 -0300 Subject: [PATCH 16/28] long-term downloads resume * recovery infrastructure * bump serialVersionUID of DownloadMission * misc cleanup in DownloadMission.java * remove unused/redundant from strings.xml --- .../newpipe/download/DownloadDialog.java | 34 ++- .../giga/get/DownloadInitializer.java | 15 ++ .../us/shandian/giga/get/DownloadMission.java | 96 ++++++-- .../giga/get/DownloadMissionRecover.java | 222 ++++++++++++++++++ .../shandian/giga/get/DownloadRunnable.java | 27 ++- .../giga/get/DownloadRunnableFallback.java | 11 +- .../giga/get/MissionRecoveryInfo.java | 79 +++++++ .../giga/service/DownloadManager.java | 1 - .../giga/service/DownloadManagerService.java | 36 ++- .../giga/ui/adapter/MissionAdapter.java | 12 +- app/src/main/res/values-ar/strings.xml | 1 - app/src/main/res/values-be/strings.xml | 1 - app/src/main/res/values-cmn/strings.xml | 1 - app/src/main/res/values-cs/strings.xml | 1 - app/src/main/res/values-da/strings.xml | 1 - app/src/main/res/values-de/strings.xml | 1 - app/src/main/res/values-el/strings.xml | 1 - app/src/main/res/values-es/strings.xml | 9 +- app/src/main/res/values-et/strings.xml | 1 - app/src/main/res/values-eu/strings.xml | 1 - app/src/main/res/values-fr/strings.xml | 1 - app/src/main/res/values-he/strings.xml | 1 - app/src/main/res/values-hr/strings.xml | 1 - app/src/main/res/values-id/strings.xml | 1 - app/src/main/res/values-it/strings.xml | 1 - app/src/main/res/values-ja/strings.xml | 1 - app/src/main/res/values-ko/strings.xml | 1 - app/src/main/res/values-ms/strings.xml | 1 - app/src/main/res/values-nb-rNO/strings.xml | 2 +- app/src/main/res/values-nl-rBE/strings.xml | 1 - app/src/main/res/values-nl/strings.xml | 1 - app/src/main/res/values-pa/strings.xml | 1 - app/src/main/res/values-pl/strings.xml | 1 - app/src/main/res/values-pt-rBR/strings.xml | 1 - app/src/main/res/values-pt/strings.xml | 1 - app/src/main/res/values-ru/strings.xml | 1 - app/src/main/res/values-sk/strings.xml | 1 - app/src/main/res/values-tr/strings.xml | 1 - app/src/main/res/values-uk/strings.xml | 1 - app/src/main/res/values-vi/strings.xml | 1 - app/src/main/res/values-zh-rTW/strings.xml | 1 - app/src/main/res/values/strings.xml | 2 +- 42 files changed, 478 insertions(+), 97 deletions(-) create mode 100644 app/src/main/java/us/shandian/giga/get/DownloadMissionRecover.java create mode 100644 app/src/main/java/us/shandian/giga/get/MissionRecoveryInfo.java diff --git a/app/src/main/java/org/schabi/newpipe/download/DownloadDialog.java b/app/src/main/java/org/schabi/newpipe/download/DownloadDialog.java index 90258a6dc..0006b3c12 100644 --- a/app/src/main/java/org/schabi/newpipe/download/DownloadDialog.java +++ b/app/src/main/java/org/schabi/newpipe/download/DownloadDialog.java @@ -68,6 +68,7 @@ import java.util.Locale; import icepick.Icepick; import icepick.State; import io.reactivex.disposables.CompositeDisposable; +import us.shandian.giga.get.MissionRecoveryInfo; import us.shandian.giga.io.StoredDirectoryHelper; import us.shandian.giga.io.StoredFileHelper; import us.shandian.giga.postprocessing.Postprocessing; @@ -762,12 +763,13 @@ public class DownloadDialog extends DialogFragment implements RadioGroup.OnCheck } Stream selectedStream; + Stream secondaryStream = null; char kind; int threads = threadsSeekBar.getProgress() + 1; String[] urls; + MissionRecoveryInfo[] recoveryInfo; String psName = null; String[] psArgs = null; - String secondaryStreamUrl = null; long nearLength = 0; // more download logic: select muxer, subtitle converter, etc. @@ -786,12 +788,12 @@ public class DownloadDialog extends DialogFragment implements RadioGroup.OnCheck kind = 'v'; selectedStream = videoStreamsAdapter.getItem(selectedVideoIndex); - SecondaryStreamHelper secondaryStream = videoStreamsAdapter + SecondaryStreamHelper secondary = videoStreamsAdapter .getAllSecondary() .get(wrappedVideoStreams.getStreamsList().indexOf(selectedStream)); - if (secondaryStream != null) { - secondaryStreamUrl = secondaryStream.getStream().getUrl(); + if (secondary != null) { + secondaryStream = secondary.getStream(); if (selectedStream.getFormat() == MediaFormat.MPEG_4) psName = Postprocessing.ALGORITHM_MP4_FROM_DASH_MUXER; @@ -803,8 +805,8 @@ public class DownloadDialog extends DialogFragment implements RadioGroup.OnCheck // set nearLength, only, if both sizes are fetched or known. This probably // does not work on slow networks but is later updated in the downloader - if (secondaryStream.getSizeInBytes() > 0 && videoSize > 0) { - nearLength = secondaryStream.getSizeInBytes() + videoSize; + if (secondary.getSizeInBytes() > 0 && videoSize > 0) { + nearLength = secondary.getSizeInBytes() + videoSize; } } break; @@ -826,13 +828,25 @@ public class DownloadDialog extends DialogFragment implements RadioGroup.OnCheck return; } - if (secondaryStreamUrl == null) { - urls = new String[]{selectedStream.getUrl()}; + if (secondaryStream == null) { + urls = new String[]{ + selectedStream.getUrl() + }; + recoveryInfo = new MissionRecoveryInfo[]{ + new MissionRecoveryInfo(selectedStream) + }; } else { - urls = new String[]{selectedStream.getUrl(), secondaryStreamUrl}; + urls = new String[]{ + selectedStream.getUrl(), secondaryStream.getUrl() + }; + recoveryInfo = new MissionRecoveryInfo[]{ + new MissionRecoveryInfo(selectedStream), new MissionRecoveryInfo(secondaryStream) + }; } - DownloadManagerService.startMission(context, urls, storage, kind, threads, currentInfo.getUrl(), psName, psArgs, nearLength); + DownloadManagerService.startMission( + context, urls, storage, kind, threads, currentInfo.getUrl(), psName, psArgs, nearLength, recoveryInfo + ); dismiss(); } diff --git a/app/src/main/java/us/shandian/giga/get/DownloadInitializer.java b/app/src/main/java/us/shandian/giga/get/DownloadInitializer.java index 247faeb6d..593feafa7 100644 --- a/app/src/main/java/us/shandian/giga/get/DownloadInitializer.java +++ b/app/src/main/java/us/shandian/giga/get/DownloadInitializer.java @@ -1,6 +1,7 @@ package us.shandian.giga.get; import androidx.annotation.NonNull; +import android.text.TextUtils; import android.util.Log; import org.schabi.newpipe.streams.io.SharpStream; @@ -151,6 +152,20 @@ public class DownloadInitializer extends Thread { if (!mMission.running || Thread.interrupted()) return; + if (!mMission.unknownLength && mMission.recoveryInfo != null) { + String entityTag = mConn.getHeaderField("ETAG"); + String lastModified = mConn.getHeaderField("Last-Modified"); + MissionRecoveryInfo recovery = mMission.recoveryInfo[mMission.current]; + + if (!TextUtils.isEmpty(entityTag)) { + recovery.validateCondition = entityTag; + } else if (!TextUtils.isEmpty(lastModified)) { + recovery.validateCondition = lastModified;// Note: this is less precise + } else { + recovery.validateCondition = null; + } + } + mMission.running = false; break; } catch (InterruptedIOException | ClosedByInterruptException e) { diff --git a/app/src/main/java/us/shandian/giga/get/DownloadMission.java b/app/src/main/java/us/shandian/giga/get/DownloadMission.java index d78f8e32b..77b417118 100644 --- a/app/src/main/java/us/shandian/giga/get/DownloadMission.java +++ b/app/src/main/java/us/shandian/giga/get/DownloadMission.java @@ -27,7 +27,7 @@ import us.shandian.giga.util.Utility; import static org.schabi.newpipe.BuildConfig.DEBUG; public class DownloadMission extends Mission { - private static final long serialVersionUID = 5L;// last bump: 30 june 2019 + private static final long serialVersionUID = 6L;// last bump: 28 september 2019 static final int BUFFER_SIZE = 64 * 1024; static final int BLOCK_SIZE = 512 * 1024; @@ -51,8 +51,9 @@ public class DownloadMission extends Mission { public static final int ERROR_INSUFFICIENT_STORAGE = 1010; public static final int ERROR_PROGRESS_LOST = 1011; public static final int ERROR_TIMEOUT = 1012; + public static final int ERROR_RESOURCE_GONE = 1013; public static final int ERROR_HTTP_NO_CONTENT = 204; - public static final int ERROR_HTTP_UNSUPPORTED_RANGE = 206; + static final int ERROR_HTTP_FORBIDDEN = 403; /** * The urls of the file to download @@ -125,6 +126,11 @@ public class DownloadMission extends Mission { */ public int threadCount = 3; + /** + * information required to recover a download + */ + public MissionRecoveryInfo[] recoveryInfo; + private transient int finishCount; public transient boolean running; public boolean enqueued; @@ -132,7 +138,6 @@ public class DownloadMission extends Mission { public int errCode = ERROR_NOTHING; public Exception errObject = null; - public transient boolean recovered; public transient Handler mHandler; private transient boolean mWritingToFile; private transient boolean[] blockAcquired; @@ -197,9 +202,9 @@ public class DownloadMission extends Mission { } /** - * Open connection + * Opens a connection * - * @param threadId id of the calling thread, used only for debug + * @param threadId id of the calling thread, used only for debugging * @param rangeStart range start * @param rangeEnd range end * @return a {@link java.net.URLConnection URLConnection} linking to the URL. @@ -251,7 +256,7 @@ public class DownloadMission extends Mission { case 204: case 205: case 207: - throw new HttpError(conn.getResponseCode()); + throw new HttpError(statusCode); case 416: return;// let the download thread handle this error default: @@ -270,10 +275,6 @@ public class DownloadMission extends Mission { synchronized void notifyProgress(long deltaLen) { if (!running) return; - if (recovered) { - recovered = false; - } - if (unknownLength) { length += deltaLen;// Update length before proceeding } @@ -344,7 +345,6 @@ public class DownloadMission extends Mission { if (running) { running = false; - recovered = true; if (threads != null) selfPause(); } } @@ -409,12 +409,13 @@ public class DownloadMission extends Mission { * Start downloading with multiple threads. */ public void start() { - if (running || isFinished()) return; + if (running || isFinished() || urls.length < 1) return; // ensure that the previous state is completely paused. - joinForThread(init); + int maxWait = 10000;// 10 seconds + joinForThread(init, maxWait); if (threads != null) { - for (Thread thread : threads) joinForThread(thread); + for (Thread thread : threads) joinForThread(thread, maxWait); threads = null; } @@ -431,6 +432,11 @@ public class DownloadMission extends Mission { return; } + if (urls[current] == null) { + doRecover(null); + return; + } + if (blocks == null) { initializer(); return; @@ -478,7 +484,6 @@ public class DownloadMission extends Mission { } running = false; - recovered = true; if (init != null && init.isAlive()) { // NOTE: if start() method is running ¡will no have effect! @@ -563,7 +568,7 @@ public class DownloadMission extends Mission { * Write this {@link DownloadMission} to the meta file asynchronously * if no thread is already running. */ - private void writeThisToFile() { + void writeThisToFile() { synchronized (LOCK) { if (deleted) return; Utility.writeToFile(metadata, DownloadMission.this); @@ -667,6 +672,7 @@ public class DownloadMission extends Mission { * @return {@code true} is this mission its "healthy", otherwise, {@code false} */ public boolean isCorrupt() { + if (urls.length < 1) return false; return (isPsFailed() || errCode == ERROR_POSTPROCESSING_HOLD) || isFinished(); } @@ -710,6 +716,48 @@ public class DownloadMission extends Mission { return true; } + /** + * Attempts to recover the download + * + * @param fromError exception which require update the url from the source + */ + void doRecover(Exception fromError) { + Log.i(TAG, "Attempting to recover the mission: " + storage.getName()); + + if (recoveryInfo == null) { + if (fromError == null) + notifyError(ERROR_RESOURCE_GONE, null); + else + notifyError(fromError); + + urls = new String[0];// mark this mission as dead + return; + } + + if (threads != null) { + for (Thread thread : threads) { + if (thread == Thread.currentThread()) continue; + thread.interrupt(); + joinForThread(thread, 0); + } + } + + // set the current download url to null in case if the recovery + // process is canceled. Next time start() method is called the + // recovery will be executed, saving time + urls[current] = null; + + if (recoveryInfo[current].attempts >= maxRetry) { + recoveryInfo[current].attempts = 0; + notifyError(fromError); + return; + } + + threads = new Thread[]{ + runAsync(DownloadMissionRecover.mID, new DownloadMissionRecover(this, fromError)) + }; + } + private boolean deleteThisFromFile() { synchronized (LOCK) { return metadata.delete(); @@ -749,7 +797,13 @@ public class DownloadMission extends Mission { return who; } - private void joinForThread(Thread thread) { + /** + * Waits at most {@code millis} milliseconds for the thread to die + * + * @param thread the desired thread + * @param millis the time to wait in milliseconds + */ + private void joinForThread(Thread thread, int millis) { if (thread == null || !thread.isAlive()) return; if (thread == Thread.currentThread()) return; @@ -764,7 +818,7 @@ public class DownloadMission extends Mission { // start() method called quickly after pause() try { - thread.join(10000); + thread.join(millis); } catch (InterruptedException e) { Log.d(TAG, "timeout on join : " + thread.getName()); throw new RuntimeException("A thread is still running:\n" + thread.getName()); @@ -785,9 +839,9 @@ public class DownloadMission extends Mission { } } - static class Block { - int position; - int done; + public static class Block { + public int position; + public int done; } private static class Lock implements Serializable { diff --git a/app/src/main/java/us/shandian/giga/get/DownloadMissionRecover.java b/app/src/main/java/us/shandian/giga/get/DownloadMissionRecover.java new file mode 100644 index 000000000..9abd93717 --- /dev/null +++ b/app/src/main/java/us/shandian/giga/get/DownloadMissionRecover.java @@ -0,0 +1,222 @@ +package us.shandian.giga.get; + +import android.util.Log; + +import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.StreamingService; +import org.schabi.newpipe.extractor.stream.AudioStream; +import org.schabi.newpipe.extractor.stream.StreamExtractor; +import org.schabi.newpipe.extractor.stream.SubtitlesStream; +import org.schabi.newpipe.extractor.stream.VideoStream; + +import java.io.IOException; +import java.net.HttpURLConnection; +import java.nio.channels.ClosedByInterruptException; +import java.util.List; + +import static us.shandian.giga.get.DownloadMission.ERROR_RESOURCE_GONE; + +public class DownloadMissionRecover extends Thread { + private static final String TAG = "DownloadMissionRecover"; + static final int mID = -3; + + private final DownloadMission mMission; + private final MissionRecoveryInfo mRecovery; + private final Exception mFromError; + private HttpURLConnection mConn; + + DownloadMissionRecover(DownloadMission mission, Exception originError) { + mMission = mission; + mFromError = originError; + mRecovery = mission.recoveryInfo[mission.current]; + } + + @Override + public void run() { + if (mMission.source == null) { + mMission.notifyError(mFromError); + return; + } + + try { + /*if (mMission.source.startsWith(MissionRecoveryInfo.DIRECT_SOURCE)) { + resolve(mMission.source.substring(MissionRecoveryInfo.DIRECT_SOURCE.length())); + return; + }*/ + + StreamingService svr = NewPipe.getServiceByUrl(mMission.source); + + if (svr == null) { + throw new RuntimeException("Unknown source service"); + } + + StreamExtractor extractor = svr.getStreamExtractor(mMission.source); + extractor.fetchPage(); + + if (!mMission.running || super.isInterrupted()) return; + + String url = null; + + switch (mMission.kind) { + case 'a': + for (AudioStream audio : extractor.getAudioStreams()) { + if (audio.average_bitrate == mRecovery.desiredBitrate && audio.getFormat() == mRecovery.format) { + url = audio.getUrl(); + break; + } + } + break; + case 'v': + List videoStreams; + if (mRecovery.desired2) + videoStreams = extractor.getVideoOnlyStreams(); + else + videoStreams = extractor.getVideoStreams(); + for (VideoStream video : videoStreams) { + if (video.resolution.equals(mRecovery.desired) && video.getFormat() == mRecovery.format) { + url = video.getUrl(); + break; + } + } + break; + case 's': + for (SubtitlesStream subtitles : extractor.getSubtitles(mRecovery.format)) { + String tag = subtitles.getLanguageTag(); + if (tag.equals(mRecovery.desired) && subtitles.isAutoGenerated() == mRecovery.desired2) { + url = subtitles.getURL(); + break; + } + } + break; + default: + throw new RuntimeException("Unknown stream type"); + } + + resolve(url); + } catch (Exception e) { + if (!mMission.running || e instanceof ClosedByInterruptException) return; + mRecovery.attempts++; + mMission.notifyError(e); + } + } + + private void resolve(String url) throws IOException, DownloadMission.HttpError { + if (mRecovery.validateCondition == null) { + Log.w(TAG, "validation condition not defined, the resource can be stale"); + } + + if (mMission.unknownLength || mRecovery.validateCondition == null) { + recover(url, false); + return; + } + + /////////////////////////////////////////////////////////////////////// + ////// Validate the http resource doing a range request + ///////////////////// + try { + mConn = mMission.openConnection(url, mID, mMission.length - 10, mMission.length); + mConn.setRequestProperty("If-Range", mRecovery.validateCondition); + mMission.establishConnection(mID, mConn); + + int code = mConn.getResponseCode(); + + switch (code) { + case 200: + case 413: + // stale + recover(url, true); + return; + case 206: + // in case of validation using the Last-Modified date, check the resource length + long[] contentRange = parseContentRange(mConn.getHeaderField("Content-Range")); + boolean lengthMismatch = contentRange[2] != -1 && contentRange[2] != mMission.length; + + recover(url, lengthMismatch); + return; + } + + throw new DownloadMission.HttpError(code); + } catch (Exception e) { + if (!mMission.running || e instanceof ClosedByInterruptException) return; + throw e; + } finally { + this.interrupt(); + } + } + + private void recover(String url, boolean stale) { + Log.i(TAG, + String.format("download recovered name=%s isStale=%s url=%s", mMission.storage.getName(), stale, url) + ); + + if (url == null) { + mMission.notifyError(ERROR_RESOURCE_GONE, null); + return; + } + + mMission.urls[mMission.current] = url; + mRecovery.attempts = 0; + + if (stale) { + mMission.resetState(false, false, DownloadMission.ERROR_NOTHING); + } + + mMission.writeThisToFile(); + + if (!mMission.running || super.isInterrupted()) return; + + mMission.running = false; + mMission.start(); + } + + private long[] parseContentRange(String value) { + long[] range = new long[3]; + + if (value == null) { + // this never should happen + return range; + } + + try { + value = value.trim(); + + if (!value.startsWith("bytes")) { + return range;// unknown range type + } + + int space = value.lastIndexOf(' ') + 1; + int dash = value.indexOf('-', space) + 1; + int bar = value.indexOf('/', dash); + + // start + range[0] = Long.parseLong(value.substring(space, dash - 1)); + + // end + range[1] = Long.parseLong(value.substring(dash, bar)); + + // resource length + value = value.substring(bar + 1); + if (value.equals("*")) { + range[2] = -1;// unknown length received from the server but should be valid + } else { + range[2] = Long.parseLong(value); + } + } catch (Exception e) { + // nothing to do + } + + return range; + } + + @Override + public void interrupt() { + super.interrupt(); + if (mConn != null) { + try { + mConn.disconnect(); + } catch (Exception e) { + // nothing to do + } + } + } +} diff --git a/app/src/main/java/us/shandian/giga/get/DownloadRunnable.java b/app/src/main/java/us/shandian/giga/get/DownloadRunnable.java index f5b9b06d4..1d2a4eee7 100644 --- a/app/src/main/java/us/shandian/giga/get/DownloadRunnable.java +++ b/app/src/main/java/us/shandian/giga/get/DownloadRunnable.java @@ -10,8 +10,10 @@ import java.net.HttpURLConnection; import java.nio.channels.ClosedByInterruptException; import us.shandian.giga.get.DownloadMission.Block; +import us.shandian.giga.get.DownloadMission.HttpError; import static org.schabi.newpipe.BuildConfig.DEBUG; +import static us.shandian.giga.get.DownloadMission.ERROR_HTTP_FORBIDDEN; /** @@ -19,7 +21,7 @@ import static org.schabi.newpipe.BuildConfig.DEBUG; * an error occurs or the process is stopped. */ public class DownloadRunnable extends Thread { - private static final String TAG = DownloadRunnable.class.getSimpleName(); + private static final String TAG = "DownloadRunnable"; private final DownloadMission mMission; private final int mId; @@ -41,13 +43,7 @@ public class DownloadRunnable extends Thread { public void run() { boolean retry = false; Block block = null; - int retryCount = 0; - - if (DEBUG) { - Log.d(TAG, mId + ":recovered: " + mMission.recovered); - } - SharpStream f; try { @@ -133,6 +129,17 @@ public class DownloadRunnable extends Thread { } catch (Exception e) { if (!mMission.running || e instanceof ClosedByInterruptException) break; + if (e instanceof HttpError && ((HttpError) e).statusCode == ERROR_HTTP_FORBIDDEN) { + // for youtube streams. The url has expired, recover + f.close(); + + if (mId == 1) { + // only the first thread will execute the recovery procedure + mMission.doRecover(e); + } + return; + } + if (retryCount++ >= mMission.maxRetry) { mMission.notifyError(e); break; @@ -144,11 +151,7 @@ public class DownloadRunnable extends Thread { } } - try { - f.close(); - } catch (Exception err) { - // ¿ejected media storage? ¿file deleted? ¿storage ran out of space? - } + f.close(); if (DEBUG) { Log.d(TAG, "thread " + mId + " exited from main download loop"); diff --git a/app/src/main/java/us/shandian/giga/get/DownloadRunnableFallback.java b/app/src/main/java/us/shandian/giga/get/DownloadRunnableFallback.java index 7fb1f0c77..b5937c577 100644 --- a/app/src/main/java/us/shandian/giga/get/DownloadRunnableFallback.java +++ b/app/src/main/java/us/shandian/giga/get/DownloadRunnableFallback.java @@ -10,9 +10,11 @@ import java.io.InputStream; import java.net.HttpURLConnection; import java.nio.channels.ClosedByInterruptException; +import us.shandian.giga.get.DownloadMission.HttpError; import us.shandian.giga.util.Utility; import static org.schabi.newpipe.BuildConfig.DEBUG; +import static us.shandian.giga.get.DownloadMission.ERROR_HTTP_FORBIDDEN; /** * Single-threaded fallback mode @@ -85,7 +87,7 @@ public class DownloadRunnableFallback extends Thread { mIs = mConn.getInputStream(); - byte[] buf = new byte[64 * 1024]; + byte[] buf = new byte[DownloadMission.BUFFER_SIZE]; int len = 0; while (mMission.running && (len = mIs.read(buf, 0, buf.length)) != -1) { @@ -103,6 +105,13 @@ public class DownloadRunnableFallback extends Thread { if (!mMission.running || e instanceof ClosedByInterruptException) return; + if (e instanceof HttpError && ((HttpError) e).statusCode == ERROR_HTTP_FORBIDDEN) { + // for youtube streams. The url has expired, recover + mMission.doRecover(e); + dispose(); + return; + } + if (mRetryCount++ >= mMission.maxRetry) { mMission.notifyError(e); return; diff --git a/app/src/main/java/us/shandian/giga/get/MissionRecoveryInfo.java b/app/src/main/java/us/shandian/giga/get/MissionRecoveryInfo.java new file mode 100644 index 000000000..553ba6d89 --- /dev/null +++ b/app/src/main/java/us/shandian/giga/get/MissionRecoveryInfo.java @@ -0,0 +1,79 @@ +package us.shandian.giga.get; + +import android.os.Parcel; +import android.os.Parcelable; +import android.support.annotation.NonNull; + +import org.schabi.newpipe.extractor.MediaFormat; +import org.schabi.newpipe.extractor.stream.AudioStream; +import org.schabi.newpipe.extractor.stream.Stream; +import org.schabi.newpipe.extractor.stream.SubtitlesStream; +import org.schabi.newpipe.extractor.stream.VideoStream; + +import java.io.Serializable; + +public class MissionRecoveryInfo implements Serializable, Parcelable { + private static final long serialVersionUID = 0L; + //public static final String DIRECT_SOURCE = "direct-source://"; + + public MediaFormat format; + String desired; + boolean desired2; + int desiredBitrate; + + transient int attempts = 0; + + String validateCondition = null; + + public MissionRecoveryInfo(@NonNull Stream stream) { + if (stream instanceof AudioStream) { + desiredBitrate = ((AudioStream) stream).average_bitrate; + desired2 = false; + } else if (stream instanceof VideoStream) { + desired = ((VideoStream) stream).getResolution(); + desired2 = ((VideoStream) stream).isVideoOnly(); + } else if (stream instanceof SubtitlesStream) { + desired = ((SubtitlesStream) stream).getLanguageTag(); + desired2 = ((SubtitlesStream) stream).isAutoGenerated(); + } else { + throw new RuntimeException("Unknown stream kind"); + } + + format = stream.getFormat(); + if (format == null) throw new NullPointerException("Stream format cannot be null"); + } + + @Override + public int describeContents() { + return 0; + } + + @Override + public void writeToParcel(Parcel parcel, int flags) { + parcel.writeInt(this.format.ordinal()); + parcel.writeString(this.desired); + parcel.writeInt(this.desired2 ? 0x01 : 0x00); + parcel.writeInt(this.desiredBitrate); + parcel.writeString(this.validateCondition); + } + + private MissionRecoveryInfo(Parcel parcel) { + this.format = MediaFormat.values()[parcel.readInt()]; + this.desired = parcel.readString(); + this.desired2 = parcel.readInt() != 0x00; + this.desiredBitrate = parcel.readInt(); + this.validateCondition = parcel.readString(); + } + + public static final Parcelable.Creator CREATOR = new Parcelable.Creator() { + @Override + public MissionRecoveryInfo createFromParcel(Parcel source) { + return new MissionRecoveryInfo(source); + } + + @Override + public MissionRecoveryInfo[] newArray(int size) { + return new MissionRecoveryInfo[size]; + } + }; +} diff --git a/app/src/main/java/us/shandian/giga/service/DownloadManager.java b/app/src/main/java/us/shandian/giga/service/DownloadManager.java index 3d34411b9..a859a87ca 100644 --- a/app/src/main/java/us/shandian/giga/service/DownloadManager.java +++ b/app/src/main/java/us/shandian/giga/service/DownloadManager.java @@ -177,7 +177,6 @@ public class DownloadManager { mis.psAlgorithm.setTemporalDir(pickAvailableTemporalDir(ctx)); } - mis.recovered = exists; mis.metadata = sub; mis.maxRetry = mPrefMaxRetry; mis.mHandler = mHandler; diff --git a/app/src/main/java/us/shandian/giga/service/DownloadManagerService.java b/app/src/main/java/us/shandian/giga/service/DownloadManagerService.java index 461787b62..ea9029c0b 100755 --- a/app/src/main/java/us/shandian/giga/service/DownloadManagerService.java +++ b/app/src/main/java/us/shandian/giga/service/DownloadManagerService.java @@ -23,6 +23,7 @@ import android.os.Handler; import android.os.Handler.Callback; import android.os.IBinder; import android.os.Message; +import android.os.Parcelable; import android.preference.PreferenceManager; import androidx.annotation.NonNull; import androidx.annotation.Nullable; @@ -40,8 +41,11 @@ import org.schabi.newpipe.player.helper.LockManager; import java.io.File; import java.io.IOException; import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; import us.shandian.giga.get.DownloadMission; +import us.shandian.giga.get.MissionRecoveryInfo; import us.shandian.giga.io.StoredDirectoryHelper; import us.shandian.giga.io.StoredFileHelper; import us.shandian.giga.postprocessing.Postprocessing; @@ -73,6 +77,7 @@ public class DownloadManagerService extends Service { private static final String EXTRA_PATH = "DownloadManagerService.extra.storagePath"; private static final String EXTRA_PARENT_PATH = "DownloadManagerService.extra.storageParentPath"; private static final String EXTRA_STORAGE_TAG = "DownloadManagerService.extra.storageTag"; + private static final String EXTRA_RECOVERY_INFO = "DownloadManagerService.extra.recoveryInfo"; private static final String ACTION_RESET_DOWNLOAD_FINISHED = APPLICATION_ID + ".reset_download_finished"; private static final String ACTION_OPEN_DOWNLOADS_FINISHED = APPLICATION_ID + ".open_downloads_finished"; @@ -364,18 +369,20 @@ public class DownloadManagerService extends Service { /** * Start a new download mission * - * @param context the activity context - * @param urls the list of urls to download - * @param storage where the file is saved - * @param kind type of file (a: audio v: video s: subtitle ?: file-extension defined) - * @param threads the number of threads maximal used to download chunks of the file. - * @param psName the name of the required post-processing algorithm, or {@code null} to ignore. - * @param source source url of the resource - * @param psArgs the arguments for the post-processing algorithm. - * @param nearLength the approximated final length of the file + * @param context the activity context + * @param urls array of urls to download + * @param storage where the file is saved + * @param kind type of file (a: audio v: video s: subtitle ?: file-extension defined) + * @param threads the number of threads maximal used to download chunks of the file. + * @param psName the name of the required post-processing algorithm, or {@code null} to ignore. + * @param source source url of the resource + * @param psArgs the arguments for the post-processing algorithm. + * @param nearLength the approximated final length of the file + * @param recoveryInfo array of MissionRecoveryInfo, in case is required recover the download */ - public static void startMission(Context context, String[] urls, StoredFileHelper storage, char kind, - int threads, String source, String psName, String[] psArgs, long nearLength) { + public static void startMission(Context context, String[] urls, StoredFileHelper storage, + char kind, int threads, String source, String psName, + String[] psArgs, long nearLength, MissionRecoveryInfo[] recoveryInfo) { Intent intent = new Intent(context, DownloadManagerService.class); intent.setAction(Intent.ACTION_RUN); intent.putExtra(EXTRA_URLS, urls); @@ -385,6 +392,7 @@ public class DownloadManagerService extends Service { intent.putExtra(EXTRA_POSTPROCESSING_NAME, psName); intent.putExtra(EXTRA_POSTPROCESSING_ARGS, psArgs); intent.putExtra(EXTRA_NEAR_LENGTH, nearLength); + intent.putExtra(EXTRA_RECOVERY_INFO, recoveryInfo); intent.putExtra(EXTRA_PARENT_PATH, storage.getParentUri()); intent.putExtra(EXTRA_PATH, storage.getUri()); @@ -404,6 +412,7 @@ public class DownloadManagerService extends Service { String source = intent.getStringExtra(EXTRA_SOURCE); long nearLength = intent.getLongExtra(EXTRA_NEAR_LENGTH, 0); String tag = intent.getStringExtra(EXTRA_STORAGE_TAG); + Parcelable[] parcelRecovery = intent.getParcelableArrayExtra(EXTRA_RECOVERY_INFO); StoredFileHelper storage; try { @@ -418,10 +427,15 @@ public class DownloadManagerService extends Service { else ps = Postprocessing.getAlgorithm(psName, psArgs); + MissionRecoveryInfo[] recovery = new MissionRecoveryInfo[parcelRecovery.length]; + for (int i = 0; i < parcelRecovery.length; i++) + recovery[i] = (MissionRecoveryInfo) parcelRecovery[i]; + final DownloadMission mission = new DownloadMission(urls, storage, kind, ps); mission.threadCount = threads; mission.source = source; mission.nearLength = nearLength; + mission.recoveryInfo = recovery; if (ps != null) ps.setTemporalDir(DownloadManager.pickAvailableTemporalDir(this)); diff --git a/app/src/main/java/us/shandian/giga/ui/adapter/MissionAdapter.java b/app/src/main/java/us/shandian/giga/ui/adapter/MissionAdapter.java index 6d1169031..6c6198750 100644 --- a/app/src/main/java/us/shandian/giga/ui/adapter/MissionAdapter.java +++ b/app/src/main/java/us/shandian/giga/ui/adapter/MissionAdapter.java @@ -62,7 +62,6 @@ import static android.content.Intent.FLAG_GRANT_READ_URI_PERMISSION; import static us.shandian.giga.get.DownloadMission.ERROR_CONNECT_HOST; import static us.shandian.giga.get.DownloadMission.ERROR_FILE_CREATION; import static us.shandian.giga.get.DownloadMission.ERROR_HTTP_NO_CONTENT; -import static us.shandian.giga.get.DownloadMission.ERROR_HTTP_UNSUPPORTED_RANGE; import static us.shandian.giga.get.DownloadMission.ERROR_INSUFFICIENT_STORAGE; import static us.shandian.giga.get.DownloadMission.ERROR_NOTHING; import static us.shandian.giga.get.DownloadMission.ERROR_PATH_CREATION; @@ -71,6 +70,7 @@ import static us.shandian.giga.get.DownloadMission.ERROR_POSTPROCESSING; import static us.shandian.giga.get.DownloadMission.ERROR_POSTPROCESSING_HOLD; import static us.shandian.giga.get.DownloadMission.ERROR_POSTPROCESSING_STOPPED; import static us.shandian.giga.get.DownloadMission.ERROR_PROGRESS_LOST; +import static us.shandian.giga.get.DownloadMission.ERROR_RESOURCE_GONE; import static us.shandian.giga.get.DownloadMission.ERROR_SSL_EXCEPTION; import static us.shandian.giga.get.DownloadMission.ERROR_TIMEOUT; import static us.shandian.giga.get.DownloadMission.ERROR_UNKNOWN_EXCEPTION; @@ -430,7 +430,7 @@ public class MissionAdapter extends Adapter implements Handler.Callb switch (mission.errCode) { case 416: - msg = R.string.error_http_requested_range_not_satisfiable; + msg = R.string.error_http_unsupported_range; break; case 404: msg = R.string.error_http_not_found; @@ -443,9 +443,6 @@ public class MissionAdapter extends Adapter implements Handler.Callb case ERROR_HTTP_NO_CONTENT: msg = R.string.error_http_no_content; break; - case ERROR_HTTP_UNSUPPORTED_RANGE: - msg = R.string.error_http_unsupported_range; - break; case ERROR_PATH_CREATION: msg = R.string.error_path_creation; break; @@ -480,6 +477,9 @@ public class MissionAdapter extends Adapter implements Handler.Callb case ERROR_TIMEOUT: msg = R.string.error_timeout; break; + case ERROR_RESOURCE_GONE: + msg = R.string.error_download_resource_gone; + break; default: if (mission.errCode >= 100 && mission.errCode < 600) { msgEx = "HTTP " + mission.errCode; @@ -859,7 +859,7 @@ public class MissionAdapter extends Adapter implements Handler.Callb delete.setVisible(true); - boolean flag = !mission.isPsFailed(); + boolean flag = !mission.isPsFailed() && mission.urls.length > 0; start.setVisible(flag); queue.setVisible(flag); } diff --git a/app/src/main/res/values-ar/strings.xml b/app/src/main/res/values-ar/strings.xml index 7156d08ba..43b45d15e 100644 --- a/app/src/main/res/values-ar/strings.xml +++ b/app/src/main/res/values-ar/strings.xml @@ -468,7 +468,6 @@ لا يمكن الاتصال بالخادم الخادم لايقوم بإرسال البيانات الخادم لا يقبل التنزيل المتعدد، إعادة المحاولة مع @string/msg_threads = 1 - عدم استيفاء النطاق المطلوب غير موجود فشلت المعالجة الاولية حذف التنزيلات المنتهية diff --git a/app/src/main/res/values-be/strings.xml b/app/src/main/res/values-be/strings.xml index 93307cbcf..3c79a96d3 100644 --- a/app/src/main/res/values-be/strings.xml +++ b/app/src/main/res/values-be/strings.xml @@ -455,7 +455,6 @@ Немагчыма злучыцца з серверам Не атрымалася атрымаць дадзеныя з сервера Сервер не падтрымлівае шматструменную загрузку, паспрабуйце з @string/msg_threads = 1 - Запытаны дыяпазон недапушчальны Не знойдзена Пасляапрацоўка не ўдалася Ачысціць завершаныя diff --git a/app/src/main/res/values-cmn/strings.xml b/app/src/main/res/values-cmn/strings.xml index 49801a190..bcb145c16 100644 --- a/app/src/main/res/values-cmn/strings.xml +++ b/app/src/main/res/values-cmn/strings.xml @@ -460,7 +460,6 @@ NewPipe 更新可用! 无法创建目标文件夹 服务器不接受多线程下载, 请使用 @string/msg_threads = 1重试 - 请求范围无法满足 继续进行%s个待下载转移 切换至移动数据时有用，尽管一些下载无法被暂停 显示评论 diff --git a/app/src/main/res/values-cs/strings.xml b/app/src/main/res/values-cs/strings.xml index d539923fe..b741e0d16 100644 --- a/app/src/main/res/values-cs/strings.xml +++ b/app/src/main/res/values-cs/strings.xml @@ -463,7 +463,6 @@ otevření ve vyskakovacím okně Nelze se připojit k serveru Server neposílá data Server neakceptuje vícevláknové stahování, opakujte akci s @string/msg_threads = 1 - Požadovaný rozsah nelze splnit Nenalezeno Post-processing selhal Vyčistit dokončená stahování diff --git a/app/src/main/res/values-da/strings.xml b/app/src/main/res/values-da/strings.xml index 42ffd474b..199c2f85d 100644 --- a/app/src/main/res/values-da/strings.xml +++ b/app/src/main/res/values-da/strings.xml @@ -380,7 +380,6 @@ Kan ikke forbinde til serveren Serveren sender ikke data Serveren accepterer ikke multitrådede downloads; prøv igen med @string/msg_threads = 1 - Det anmodede interval er ikke gyldigt Ikke fundet Efterbehandling fejlede Stop diff --git a/app/src/main/res/values-de/strings.xml b/app/src/main/res/values-de/strings.xml index 2d6b5b6d2..3279e919c 100644 --- a/app/src/main/res/values-de/strings.xml +++ b/app/src/main/res/values-de/strings.xml @@ -454,7 +454,6 @@ Kann nicht mit dem Server verbinden Der Server sendet keine Daten Der Server erlaubt kein mehrfädiges Herunterladen – wiederhole mit @string/msg_threads = 1 - Gewünschter Bereich ist nicht verfügbar Nicht gefunden Nachbearbeitung fehlgeschlagen Um fertige Downloads bereinigen diff --git a/app/src/main/res/values-el/strings.xml b/app/src/main/res/values-el/strings.xml index 4f3499cfd..372cbb1a2 100644 --- a/app/src/main/res/values-el/strings.xml +++ b/app/src/main/res/values-el/strings.xml @@ -456,7 +456,6 @@ Αδυναμία σύνδεσης με τον εξυπηρετητή Ο εξυπηρετητής δεν μπορεί να στείλει τα δεδομένα Ο εξυπηρετητής δέν υποστηρίζει πολυνηματικές λήψεις, ξαναπροσπαθήστε με @string/msg_threads = 1 - Το ζητούμενο εύρος δεν μπορεί να εξυπηρετηθεί Δεν βρέθηκε Μετεπεξεργασία απέτυχε Εκκαθάριση ολοκληρωμένων λήψεων diff --git a/app/src/main/res/values-es/strings.xml b/app/src/main/res/values-es/strings.xml index 3aa0bac66..2f69e62cb 100644 --- a/app/src/main/res/values-es/strings.xml +++ b/app/src/main/res/values-es/strings.xml @@ -351,8 +351,8 @@ \n3. Inicie sesión cuando se le pida \n4. Copie la URL del perfil a la que fue redireccionado. suID, soundcloud.com/suID - Observe que esta operación puede causar un uso intensivo de la red. -\n + Observe que esta operación puede causar un uso intensivo de la red. +\n \n¿Quiere continuar\? Cargar miniaturas Desactívela para evitar la carga de miniaturas y ahorrar datos y memoria. Se vaciará la antememoria de imágenes en la memoria volátil y en el disco. @@ -444,8 +444,8 @@ Fallo la conexión segura No se pudo encontrar el servidor No se puede conectar con el servidor - El servidor no está enviando datos - El servidor no acepta descargas multiproceso; intente de nuevo con @string/msg_threads = 1 + El servidor no devolvio datos + El servidor no acepta descargas multi-hilos, intente de nuevo con @string/msg_threads = 1 No se puede satisfacer el intervalo seleccionado No encontrado Falló el posprocesamiento @@ -453,6 +453,7 @@ No hay suficiente espacio disponible en el dispositivo Se perdió el progreso porque el archivo fue eliminado Tiempo de espera excedido + El recurso solicitado ya no esta disponible Preguntar dónde descargar Se preguntará dónde guardar cada descarga Se le preguntará dónde guardar cada descarga. diff --git a/app/src/main/res/values-et/strings.xml b/app/src/main/res/values-et/strings.xml index baad94b5d..4dfcc3d0e 100644 --- a/app/src/main/res/values-et/strings.xml +++ b/app/src/main/res/values-et/strings.xml @@ -457,7 +457,6 @@ Serveriga ei saadud ühendust Server ei saada andmeid Server ei toeta mitmelõimelisi allalaadimisi. Proovi uuesti kasutades @string/msg_threads = 1 - Taotletud vahemik ei ole rahuldatav Ei leitud Järeltöötlemine nurjus Eemalda lõpetatud allalaadimised diff --git a/app/src/main/res/values-eu/strings.xml b/app/src/main/res/values-eu/strings.xml index 7da39393e..7b636d383 100644 --- a/app/src/main/res/values-eu/strings.xml +++ b/app/src/main/res/values-eu/strings.xml @@ -456,7 +456,6 @@ Ezin da zerbitzariarekin konektatu Zerbitzariak ez du daturik bidaltzen Zerbitzariak ez ditu hainbat hariko deskargak onartzen, saiatu @string/msg_threads = 1 erabilita - Eskatutako barrutia ezin da bete Ez aurkitua Post-prozesuak huts egin du Garbitu amaitutako deskargak diff --git a/app/src/main/res/values-fr/strings.xml b/app/src/main/res/values-fr/strings.xml index 147502088..b4388e39f 100644 --- a/app/src/main/res/values-fr/strings.xml +++ b/app/src/main/res/values-fr/strings.xml @@ -467,7 +467,6 @@ Utilisation des onglets par défaut, erreur lors de la lecture des onglets enregistrés Le serveur n’accepte pas les téléchargements multi-fils, veuillez réessayer avec @string/msg_threads = 1 Continuer vos %s transferts en attente depuis Téléchargement - Le domaine désiré n\'est pas disponible Afficher les commentaires Désactiver pour ne pas afficher les commentaires Lecture automatique diff --git a/app/src/main/res/values-he/strings.xml b/app/src/main/res/values-he/strings.xml index b5a0778d4..5e340d8b3 100644 --- a/app/src/main/res/values-he/strings.xml +++ b/app/src/main/res/values-he/strings.xml @@ -461,7 +461,6 @@ לא ניתן להתחבר לשרת השרת לא שולח נתונים "השרת לא מקבל הורדות רב ערוציות, מוטב לנסות שוב עם ‎@string/msg_threads = 1 " - הטווח המבוקש לא מתאים לא נמצא העיבוד המאוחר נכשל פינוי ההורדות שהסתיימו diff --git a/app/src/main/res/values-hr/strings.xml b/app/src/main/res/values-hr/strings.xml index e85d5810e..aa4ff9113 100644 --- a/app/src/main/res/values-hr/strings.xml +++ b/app/src/main/res/values-hr/strings.xml @@ -454,7 +454,6 @@ Nije moguće povezati se s serverom Server ne šalje podatke Poslužitelj ne prihvaća preuzimanja s više niti, pokušaj ponovo s @string/msg_threads = 1 - Traženi raspon nije zadovoljavajući Nije pronađeno Naknadna obrada nije uspjela Obriši završena preuzimanja diff --git a/app/src/main/res/values-id/strings.xml b/app/src/main/res/values-id/strings.xml index db738d749..d52f5fafa 100644 --- a/app/src/main/res/values-id/strings.xml +++ b/app/src/main/res/values-id/strings.xml @@ -450,7 +450,6 @@ Tidak dapat terhubung ke server Server tidak mengirim data Server tidak menerima unduhan multi-utas, coba lagi dengan @string/msg_threads = 1 - Rentang yang diminta tidak memuaskan Tidak ditemukan Pengolahan-pasca gagal Hapus unduhan yang sudah selesai diff --git a/app/src/main/res/values-it/strings.xml b/app/src/main/res/values-it/strings.xml index 35fdebeda..c92292f99 100644 --- a/app/src/main/res/values-it/strings.xml +++ b/app/src/main/res/values-it/strings.xml @@ -454,7 +454,6 @@ Impossibile connettersi al server Il server non invia dati Il server non accetta download multipli, riprovare con @string/msg_threads = 1 - Intervallo richiesto non soddisfatto Non trovato Post-processing fallito Pulisci i download completati diff --git a/app/src/main/res/values-ja/strings.xml b/app/src/main/res/values-ja/strings.xml index b67da798c..58ca2ebff 100644 --- a/app/src/main/res/values-ja/strings.xml +++ b/app/src/main/res/values-ja/strings.xml @@ -440,7 +440,6 @@ サーバに接続できません サーバがデータを送信していません サーバが同時接続ダウンロードを受け付けません。再試行してください @string/msg_threads = 1 - 必要な範囲が満たされていません 見つかりません 保存処理に失敗しました 完了済みを一覧から削除します diff --git a/app/src/main/res/values-ko/strings.xml b/app/src/main/res/values-ko/strings.xml index 333891910..fdc76b04e 100644 --- a/app/src/main/res/values-ko/strings.xml +++ b/app/src/main/res/values-ko/strings.xml @@ -451,7 +451,6 @@ 서버에 접속할 수 없습니다 서버가 데이터를 전송하지 않고 있습니다 서버가 다중 스레드 다운로드를 받아들이지 않습니다, @string/msg_threads = 1 를 사용해 다시 시도해보세요 - 요청된 HTTP 범위가 충분하지 않습니다 HTTP 찾을 수 없습니다 후처리 작업이 실패하였습니다 완료된 다운로드 비우기 diff --git a/app/src/main/res/values-ms/strings.xml b/app/src/main/res/values-ms/strings.xml index c7fa5de92..daa120ea2 100644 --- a/app/src/main/res/values-ms/strings.xml +++ b/app/src/main/res/values-ms/strings.xml @@ -450,7 +450,6 @@ Tidak dapat menyambung ke server Server tidak menghantar data Server tidak menerima muat turun berbilang thread, cuba lagi dengan @string/msg_threads = 1 - Julat yang diminta tidak memuaskan Tidak ditemui Pemprosesan-pasca gagal Hapuskan senarai muat turun yang selesai diff --git a/app/src/main/res/values-nb-rNO/strings.xml b/app/src/main/res/values-nb-rNO/strings.xml index d26886844..6262480b0 100644 --- a/app/src/main/res/values-nb-rNO/strings.xml +++ b/app/src/main/res/values-nb-rNO/strings.xml @@ -496,7 +496,7 @@ Sett nedlastinger på pause Spør om hvor ting skal lastes ned til Du vil bli spurt om hvor hver nedlasting skal plasseres - Du vil bli spurt om hvor hver nedlasting skal plasseres. + Du vil bli spurt om hvor hver nedlasting skal plasseres. \nSkru på SAF hvis du vil laste ned til eksternt SD-kort Bruk SAF Lagringstilgangsrammeverk (SAF) tillater nedlastinger til eksternt SD-kort. diff --git a/app/src/main/res/values-nl-rBE/strings.xml b/app/src/main/res/values-nl-rBE/strings.xml index 94feb4915..f64ff6bf9 100644 --- a/app/src/main/res/values-nl-rBE/strings.xml +++ b/app/src/main/res/values-nl-rBE/strings.xml @@ -454,7 +454,6 @@ Kan geen verbinding maken met de server De server verzendt geen gegevens De server aanvaardt geen meerdradige downloads, probeert het opnieuw met @string/msg_threads = 1 - Gevraagd bereik niet beschikbaar Niet gevonden Nabewerking mislukt Voltooide downloads wissen diff --git a/app/src/main/res/values-nl/strings.xml b/app/src/main/res/values-nl/strings.xml index f7acba6ae..6aecc2cd1 100644 --- a/app/src/main/res/values-nl/strings.xml +++ b/app/src/main/res/values-nl/strings.xml @@ -454,7 +454,6 @@ Kan niet met de server verbinden De server verzendt geen gegevens De server accepteert geen multi-threaded downloads, probeer het opnieuw met @string/msg_threads = 1 - Gevraagde bereik niet beschikbaar Niet gevonden Nabewerking mislukt Voltooide downloads wissen diff --git a/app/src/main/res/values-pa/strings.xml b/app/src/main/res/values-pa/strings.xml index c31eb805d..b57564eba 100644 --- a/app/src/main/res/values-pa/strings.xml +++ b/app/src/main/res/values-pa/strings.xml @@ -450,7 +450,6 @@ ਸਰਵਰ ਨਾਲ ਜੁੜ ਨਹੀਂ ਸਕਦਾ ਸਰਵਰ ਨੇ ਡਾਟਾ ਨਹੀਂ ਭੇਜਿਆ ਸਰਵਰ ਮਲਟੀ-Threaded ਡਾਊਨਲੋਡਸ ਨੂੰ ਸਵੀਕਾਰ ਨਹੀਂ ਕਰਦਾ, ਇਸ ਨਾਲ ਦੁਬਾਰਾ ਕੋਸ਼ਿਸ਼ ਕਰੋ @string/msg_threads = 1 - ਬੇਨਤੀ ਕੀਤੀ ਸੀਮਾ ਤਸੱਲੀਬਖਸ਼ ਨਹੀਂ ਹੈ ਨਹੀਂ ਲਭਿਆ Post-processing ਫੇਲ੍ਹ ਮੁਕੰਮਲ ਹੋਈਆਂ ਡਾਊਨਲੋਡ ਸਾਫ਼ ਕਰੋ diff --git a/app/src/main/res/values-pl/strings.xml b/app/src/main/res/values-pl/strings.xml index d3c84aa22..ca1e52ff2 100644 --- a/app/src/main/res/values-pl/strings.xml +++ b/app/src/main/res/values-pl/strings.xml @@ -456,7 +456,6 @@ Nie można połączyć się z serwerem Serwer nie wysyła danych Serwer nie akceptuje pobierania wielowątkowego, spróbuj ponownie za pomocą @string/msg_threads = 1 - Niewłaściwy zakres Nie znaleziono Przetwarzanie końcowe nie powiodło się Wyczyść ukończone pobieranie diff --git a/app/src/main/res/values-pt-rBR/strings.xml b/app/src/main/res/values-pt-rBR/strings.xml index aaac4fd4c..0bdf4d006 100644 --- a/app/src/main/res/values-pt-rBR/strings.xml +++ b/app/src/main/res/values-pt-rBR/strings.xml @@ -463,7 +463,6 @@ abrir em modo popup Não foi possível conectar ao servidor O servidor não envia dados O servidor não aceita downloads em multi-thread, tente com @string/msg_threads = 1 - Intervalo solicitado não aceito Não encontrado Falha no pós processamento Limpar downloads finalizados diff --git a/app/src/main/res/values-pt/strings.xml b/app/src/main/res/values-pt/strings.xml index 5d7cd8146..6d55023d1 100644 --- a/app/src/main/res/values-pt/strings.xml +++ b/app/src/main/res/values-pt/strings.xml @@ -452,7 +452,6 @@ Não é possível ligar ao servidor O servidor não envia dados O servidor não aceita transferências de vários processos, tente novamente com @string/msg_threads = 1 - Intervalo solicitado não satisfatório Não encontrado Pós-processamento falhado Limpar transferências concluídas diff --git a/app/src/main/res/values-ru/strings.xml b/app/src/main/res/values-ru/strings.xml index 6f079a221..51771e1b1 100644 --- a/app/src/main/res/values-ru/strings.xml +++ b/app/src/main/res/values-ru/strings.xml @@ -454,7 +454,6 @@ Доступ запрещён системой Сервер не найден Сервер не принимает многопоточные загрузки, повторная попытка с @string/msg_threads = 1 - Запрашиваемый диапазон недопустим Не найдено Очистить завершённые Остановить diff --git a/app/src/main/res/values-sk/strings.xml b/app/src/main/res/values-sk/strings.xml index 09502f60a..36c0afd84 100644 --- a/app/src/main/res/values-sk/strings.xml +++ b/app/src/main/res/values-sk/strings.xml @@ -462,7 +462,6 @@ Nepodarilo sa pripojiť k serveru Server neposiela údaje Server neakceptuje preberanie viacerých vlákien, zopakujte s @string/msg_threads = 1 - Požadovaný rozsah nie je uspokojivý Nenájdené Post-spracovanie zlyhalo Vyčistiť dokončené sťahovania diff --git a/app/src/main/res/values-tr/strings.xml b/app/src/main/res/values-tr/strings.xml index c17b58f50..6c9c66f69 100644 --- a/app/src/main/res/values-tr/strings.xml +++ b/app/src/main/res/values-tr/strings.xml @@ -449,7 +449,6 @@ Sunucuya bağlanılamıyor Sunucu veri göndermiyor Sunucu, çok iş parçacıklı indirmeleri kabul etmez, @string/msg_threads = 1 ile yeniden deneyin - İstenen aralık karşılanamıyor Bulunamadı İşlem sonrası başarısız Tamamlanan indirmeleri temizle diff --git a/app/src/main/res/values-uk/strings.xml b/app/src/main/res/values-uk/strings.xml index 375557b04..fcce99e89 100644 --- a/app/src/main/res/values-uk/strings.xml +++ b/app/src/main/res/values-uk/strings.xml @@ -471,7 +471,6 @@ Помилка зчитування збережених вкладок. Використовую типові вкладки. Вкладки, що відображаються на головній сторінці Показувати сповіщення з пропозицією оновити застосунок за наявності нової версії - Запитуваний діапазон неприпустимий Продовжити ваші %s відкладених переміщень із Завантажень Корисно під час переходу на мобільні дані, хоча деякі завантаження не можуть бути призупинені Показувати коментарі diff --git a/app/src/main/res/values-vi/strings.xml b/app/src/main/res/values-vi/strings.xml index 74b8b395c..f8860acfd 100644 --- a/app/src/main/res/values-vi/strings.xml +++ b/app/src/main/res/values-vi/strings.xml @@ -449,7 +449,6 @@ Không thế kết nối với máy chủ Máy chủ không gửi dữ liệu về Máy chủ không chấp nhận tải đa luồng, thử lại với số luồng = 1 - (HTTP) Không thể đáp ứng khoảng dữ liệu đã yêu cầu Không tìm thấy Xử lý thất bại Dọn các tải về đã hoàn thành diff --git a/app/src/main/res/values-zh-rTW/strings.xml b/app/src/main/res/values-zh-rTW/strings.xml index fe4c1b00a..310bae3a3 100644 --- a/app/src/main/res/values-zh-rTW/strings.xml +++ b/app/src/main/res/values-zh-rTW/strings.xml @@ -447,7 +447,6 @@ 無法連線到伺服器 伺服器沒有傳送資料 伺服器不接受多執行緒下載，請以 @string/msg_threads = 1 重試 - 請求範圍無法滿足 找不到 後處理失敗 清除已結束的下載 diff --git a/app/src/main/res/values/strings.xml b/app/src/main/res/values/strings.xml index a34b00ea9..2917fb9fd 100644 --- a/app/src/main/res/values/strings.xml +++ b/app/src/main/res/values/strings.xml @@ -551,13 +551,13 @@ Can not connect to the server The server does not send data The server does not accept multi-threaded downloads, retry with @string/msg_threads = 1 - Requested range not satisfiable Not found Post-processing failed NewPipe was closed while working on the file No space left on device Progress lost, because the file was deleted Connection timeout + The solicited resource is not available anymore Clear finished downloads Are you sure? Continue your %s pending transfers from Downloads From 570738190d3afb354445517e86781e2fe91f3459 Mon Sep 17 00:00:00 2001 From: kapodamy Date: Sun, 29 Sep 2019 01:44:13 -0300 Subject: [PATCH 17/28] Mp4FromDashWriter fixes * correct calculation of "co64" box and usage of 64bits offsets * generate one chunk for audio streams like ffmpeg does, attempt to fix cut-off audio * misc. cleanup --- .../newpipe/streams/Mp4FromDashWriter.java | 79 ++++++++++++------- 1 file changed, 50 insertions(+), 29 deletions(-) diff --git a/app/src/main/java/org/schabi/newpipe/streams/Mp4FromDashWriter.java b/app/src/main/java/org/schabi/newpipe/streams/Mp4FromDashWriter.java index 03aab447c..420f77955 100644 --- a/app/src/main/java/org/schabi/newpipe/streams/Mp4FromDashWriter.java +++ b/app/src/main/java/org/schabi/newpipe/streams/Mp4FromDashWriter.java @@ -6,6 +6,7 @@ import org.schabi.newpipe.streams.Mp4DashReader.Mp4DashChunk; import org.schabi.newpipe.streams.Mp4DashReader.Mp4DashSample; import org.schabi.newpipe.streams.Mp4DashReader.Mp4Track; import org.schabi.newpipe.streams.Mp4DashReader.TrunEntry; +import org.schabi.newpipe.streams.Mp4DashReader.TrackKind; import org.schabi.newpipe.streams.io.SharpStream; import java.io.IOException; @@ -22,6 +23,7 @@ public class Mp4FromDashWriter { private final static byte SAMPLES_PER_CHUNK = 6;// ffmpeg uses 2, basic uses 1 (with 60fps uses 21 or 22). NewPipe will use 6 private final static long THRESHOLD_FOR_CO64 = 0xFFFEFFFFL;// near 3.999 GiB private final static int THRESHOLD_MOOV_LENGTH = (256 * 1024) + (2048 * 1024); // 2.2 MiB enough for: 1080p 60fps 00h35m00s + private final static short SINGLE_CHUNK_SAMPLE_BUFFER = 256; private final long time; @@ -145,7 +147,7 @@ public class Mp4FromDashWriter { // not allowed for very short tracks (less than 0.5 seconds) // outStream = output; - int read = 8;// mdat box header size + long read = 8;// mdat box header size long totalSampleSize = 0; int[] sampleExtra = new int[readers.length]; int[] defaultMediaTime = new int[readers.length]; @@ -157,6 +159,8 @@ public class Mp4FromDashWriter { tablesInfo[i] = new TablesInfo(); } + boolean singleChunk = tracks.length == 1 && tracks[0].kind == TrackKind.Audio; + // for (int i = 0; i < readers.length; i++) { int samplesSize = 0; @@ -210,14 +214,21 @@ public class Mp4FromDashWriter { tablesInfo[i].stco = (tmp / SAMPLES_PER_CHUNK) + 1;// +1 for samples in first chunk tmp = tmp % SAMPLES_PER_CHUNK; - if (tmp == 0) { + if (singleChunk) { + // avoid split audio streams in chunks + tablesInfo[i].stsc = 1; + tablesInfo[i].stsc_bEntries = new int[]{ + 1, tablesInfo[i].stsz, 1 + }; + tablesInfo[i].stco = 1; + } else if (tmp == 0) { tablesInfo[i].stsc = 2;// first chunk (init) and succesive chunks tablesInfo[i].stsc_bEntries = new int[]{ 1, SAMPLES_PER_CHUNK_INIT, 1, 2, SAMPLES_PER_CHUNK, 1 }; } else { - tablesInfo[i].stsc = 3;// first chunk (init) and succesive chunks and remain chunk + tablesInfo[i].stsc = 3;// first chunk (init) and successive chunks and remain chunk tablesInfo[i].stsc_bEntries = new int[]{ 1, SAMPLES_PER_CHUNK_INIT, 1, 2, SAMPLES_PER_CHUNK, 1, @@ -268,10 +279,10 @@ public class Mp4FromDashWriter { } else {*/ if (auxSize > 0) { int length = auxSize; - byte[] buffer = new byte[8 * 1024];// 8 KiB + byte[] buffer = new byte[64 * 1024];// 64 KiB while (length > 0) { int count = Math.min(length, buffer.length); - outWrite(buffer, 0, count); + outWrite(buffer, count); length -= count; } } @@ -280,7 +291,7 @@ public class Mp4FromDashWriter { outSeek(ftyp_size); } - // tablesInfo contais row counts + // tablesInfo contains row counts // and after returning from make_moov() will contain table offsets make_moov(defaultMediaTime, tablesInfo, is64); @@ -291,7 +302,7 @@ public class Mp4FromDashWriter { writeEntryArray(tablesInfo[i].stsc, tablesInfo[i].stsc_bEntries.length, tablesInfo[i].stsc_bEntries); tablesInfo[i].stsc_bEntries = null; if (tablesInfo[i].ctts > 0) { - sampleCount[i] = 1;// index is not base zero + sampleCount[i] = 1;// the index is not base zero sampleExtra[i] = -1; } } @@ -303,8 +314,8 @@ public class Mp4FromDashWriter { outWrite(make_mdat(totalSampleSize, is64)); int[] sampleIndex = new int[readers.length]; - int[] sizes = new int[SAMPLES_PER_CHUNK]; - int[] sync = new int[SAMPLES_PER_CHUNK]; + int[] sizes = new int[singleChunk ? SINGLE_CHUNK_SAMPLE_BUFFER : SAMPLES_PER_CHUNK]; + int[] sync = new int[singleChunk ? SINGLE_CHUNK_SAMPLE_BUFFER : SAMPLES_PER_CHUNK]; int written = readers.length; while (written > 0) { @@ -317,7 +328,12 @@ public class Mp4FromDashWriter { long chunkOffset = writeOffset; int syncCount = 0; - int limit = sampleIndex[i] == 0 ? SAMPLES_PER_CHUNK_INIT : SAMPLES_PER_CHUNK; + int limit; + if (singleChunk) { + limit = SINGLE_CHUNK_SAMPLE_BUFFER; + } else { + limit = sampleIndex[i] == 0 ? SAMPLES_PER_CHUNK_INIT : SAMPLES_PER_CHUNK; + } int j = 0; for (; j < limit; j++) { @@ -354,7 +370,7 @@ public class Mp4FromDashWriter { sizes[j] = sample.data.length; } - outWrite(sample.data, 0, sample.data.length); + outWrite(sample.data, sample.data.length); } if (j > 0) { @@ -368,10 +384,16 @@ public class Mp4FromDashWriter { tablesInfo[i].stss = writeEntryArray(tablesInfo[i].stss, syncCount, sync); } - if (is64) { - tablesInfo[i].stco = writeEntry64(tablesInfo[i].stco, chunkOffset); - } else { - tablesInfo[i].stco = writeEntryArray(tablesInfo[i].stco, 1, (int) chunkOffset); + if (tablesInfo[i].stco > 0) { + if (is64) { + tablesInfo[i].stco = writeEntry64(tablesInfo[i].stco, chunkOffset); + } else { + tablesInfo[i].stco = writeEntryArray(tablesInfo[i].stco, 1, (int) chunkOffset); + } + + if (singleChunk) { + tablesInfo[i].stco = -1; + } } outRestore(); @@ -451,12 +473,12 @@ public class Mp4FromDashWriter { // private void outWrite(byte[] buffer) throws IOException { - outWrite(buffer, 0, buffer.length); + outWrite(buffer, buffer.length); } - private void outWrite(byte[] buffer, int offset, int count) throws IOException { + private void outWrite(byte[] buffer, int count) throws IOException { writeOffset += count; - outStream.write(buffer, offset, count); + outStream.write(buffer, 0, count); } private void outSeek(long offset) throws IOException { @@ -509,7 +531,6 @@ public class Mp4FromDashWriter { ); if (extra >= 0) { - //size += 4;// commented for auxiliar buffer !!! offset += 4; auxWrite(extra); } @@ -531,7 +552,7 @@ public class Mp4FromDashWriter { if (moovSimulation) { writeOffset += buffer.length; } else if (auxBuffer == null) { - outWrite(buffer, 0, buffer.length); + outWrite(buffer, buffer.length); } else { auxBuffer.put(buffer); } @@ -703,7 +724,7 @@ public class Mp4FromDashWriter { int mediaTime; if (tracks[index].trak.edst_elst == null) { - // is a audio track ¿is edst/elst opcional for audio tracks? + // is a audio track ¿is edst/elst optional for audio tracks? mediaTime = 0x00;// ffmpeg set this value as zero, instead of defaultMediaTime bMediaRate = 0x00010000; } else { @@ -798,13 +819,13 @@ public class Mp4FromDashWriter { class TablesInfo { - public int stts; - public int stsc; - public int[] stsc_bEntries; - public int ctts; - public int stsz; - public int stsz_default; - public int stss; - public int stco; + int stts; + int stsc; + int[] stsc_bEntries; + int ctts; + int stsz; + int stsz_default; + int stss; + int stco; } } From 4292ca94ff6d36602bd3834f9ebc544a61c19272 Mon Sep 17 00:00:00 2001 From: kapodamy Date: Mon, 30 Sep 2019 23:52:49 -0300 Subject: [PATCH 18/28] misc changes * OggFromWebMWriter: rewrite (again), reduce iterations over the input. Works as-is (video streams are not supported) * WebMReader: use int for SimpleBlock.dataSize instead of long * Download Recovery: allow recovering uninitialized downloads * check range-requests using HEAD method instead of GET * DownloadRunnableFallback: add workaround for 32kB/s issue, unknown issue origin, wont fix * reporting downloads errors now include the source url with the selected quality and format --- .../newpipe/streams/OggFromWebMWriter.java | 216 +++++------------- .../schabi/newpipe/streams/WebMReader.java | 4 +- .../giga/get/DownloadInitializer.java | 35 +-- .../us/shandian/giga/get/DownloadMission.java | 36 +-- .../giga/get/DownloadMissionRecover.java | 146 +++++++++--- .../shandian/giga/get/DownloadRunnable.java | 2 +- .../giga/get/DownloadRunnableFallback.java | 20 +- .../giga/get/MissionRecoveryInfo.java | 43 +++- .../giga/ui/adapter/MissionAdapter.java | 36 ++- app/src/main/res/values-es/strings.xml | 2 +- app/src/main/res/values/strings.xml | 2 +- 11 files changed, 294 insertions(+), 248 deletions(-) diff --git a/app/src/main/java/org/schabi/newpipe/streams/OggFromWebMWriter.java b/app/src/main/java/org/schabi/newpipe/streams/OggFromWebMWriter.java index 091ae6d2a..e6363e423 100644 --- a/app/src/main/java/org/schabi/newpipe/streams/OggFromWebMWriter.java +++ b/app/src/main/java/org/schabi/newpipe/streams/OggFromWebMWriter.java @@ -12,8 +12,6 @@ import java.io.Closeable; import java.io.IOException; import java.nio.ByteBuffer; import java.nio.ByteOrder; -import java.util.ArrayList; -import java.util.Random; import javax.annotation.Nullable; @@ -23,15 +21,13 @@ import javax.annotation.Nullable; public class OggFromWebMWriter implements Closeable { private static final byte FLAG_UNSET = 0x00; - private static final byte FLAG_CONTINUED = 0x01; + //private static final byte FLAG_CONTINUED = 0x01; private static final byte FLAG_FIRST = 0x02; private static final byte FLAG_LAST = 0x04; private final static byte HEADER_CHECKSUM_OFFSET = 22; private final static byte HEADER_SIZE = 27; - private final static short BUFFER_SIZE = 8 * 1024;// 8KiB - private final static int TIME_SCALE_NS = 1000000000; private boolean done = false; @@ -43,7 +39,6 @@ public class OggFromWebMWriter implements Closeable { private int sequence_count = 0; private final int STREAM_ID; private byte packet_flag = FLAG_FIRST; - private int track_index = 0; private WebMReader webm = null; private WebMTrack webm_track = null; @@ -71,7 +66,7 @@ public class OggFromWebMWriter implements Closeable { this.source = source; this.output = target; - this.STREAM_ID = (new Random(System.currentTimeMillis())).nextInt(); + this.STREAM_ID = (int) System.currentTimeMillis(); populate_crc32_table(); } @@ -130,7 +125,6 @@ public class OggFromWebMWriter implements Closeable { try { webm_track = webm.selectTrack(trackIndex); - track_index = trackIndex; } finally { parsed = true; } @@ -154,8 +148,11 @@ public class OggFromWebMWriter implements Closeable { public void build() throws IOException { float resolution; - int read; - byte[] buffer; + SimpleBlock bloq; + ByteBuffer header = ByteBuffer.allocate(27 + (255 * 255)); + ByteBuffer page = ByteBuffer.allocate(64 * 1024); + + header.order(ByteOrder.LITTLE_ENDIAN); /* step 1: get the amount of frames per seconds */ switch (webm_track.kind) { @@ -176,57 +173,32 @@ public class OggFromWebMWriter implements Closeable { throw new RuntimeException("not implemented"); } - /* step 2a: create packet with code init data */ - ArrayList data_extra = new ArrayList<>(4); - + /* step 2: create packet with code init data */ if (webm_track.codecPrivate != null) { addPacketSegment(webm_track.codecPrivate.length); - ByteBuffer buff = byte_buffer(HEADER_SIZE + segment_table_size + webm_track.codecPrivate.length); - - make_packetHeader(0x00, buff, webm_track.codecPrivate); - data_extra.add(buff.array()); + make_packetHeader(0x00, header, webm_track.codecPrivate); + write(header); + output.write(webm_track.codecPrivate); } - /* step 2b: create packet with metadata */ - buffer = make_metadata(); + /* step 3: create packet with metadata */ + byte[] buffer = make_metadata(); if (buffer != null) { addPacketSegment(buffer.length); - ByteBuffer buff = byte_buffer(HEADER_SIZE + segment_table_size + buffer.length); - - make_packetHeader(0x00, buff, buffer); - data_extra.add(buff.array()); + make_packetHeader(0x00, header, buffer); + write(header); + output.write(buffer); } - - /* step 3: calculate amount of packets */ - SimpleBlock bloq; - int reserve_header = 0; - int headers_amount = 0; - + /* step 4: calculate amount of packets */ while (webm_segment != null) { bloq = getNextBlock(); - if (addPacketSegment(bloq)) { - continue; - } - - reserve_header += HEADER_SIZE + segment_table_size;// header size - clearSegmentTable(); - webm_block = bloq; - headers_amount++; - } - - /* step 4: create packet headers */ - rewind_source(); - - ByteBuffer headers = byte_buffer(reserve_header); - short[] headers_size = new short[headers_amount]; - int header_index = 0; - - while (webm_segment != null) { - bloq = getNextBlock(); - - if (addPacketSegment(bloq)) { + if (bloq != null && addPacketSegment(bloq)) { + int pos = page.position(); + //noinspection ResultOfMethodCallIgnored + bloq.data.read(page.array(), pos, bloq.dataSize); + page.position(pos + bloq.dataSize); continue; } @@ -251,75 +223,21 @@ public class OggFromWebMWriter implements Closeable { elapsed_ns = elapsed_ns / TIME_SCALE_NS; elapsed_ns = Math.ceil(elapsed_ns * resolution); - // create header - headers_size[header_index++] = make_packetHeader((long) elapsed_ns, headers, null); + // create header and calculate page checksum + int checksum = make_packetHeader((long) elapsed_ns, header, null); + checksum = calc_crc32(checksum, page.array(), page.position()); + + header.putInt(HEADER_CHECKSUM_OFFSET, checksum); + + // dump data + write(header); + write(page); + webm_block = bloq; } - - - /* step 5: calculate checksums */ - rewind_source(); - - int offset = 0; - buffer = new byte[BUFFER_SIZE]; - - for (header_index = 0; header_index < headers_size.length; header_index++) { - int checksum_offset = offset + HEADER_CHECKSUM_OFFSET; - int checksum = headers.getInt(checksum_offset); - - while (webm_segment != null) { - bloq = getNextBlock(); - - if (!addPacketSegment(bloq)) { - clearSegmentTable(); - webm_block = bloq; - break; - } - - // calculate page checksum - while ((read = bloq.data.read(buffer)) > 0) { - checksum = calc_crc32(checksum, buffer, 0, read); - } - } - - headers.putInt(checksum_offset, checksum); - offset += headers_size[header_index]; - } - - /* step 6: write extra headers */ - rewind_source(); - - for (byte[] buff : data_extra) { - output.write(buff); - } - - /* step 7: write stream packets */ - byte[] headers_buffers = headers.array(); - offset = 0; - buffer = new byte[BUFFER_SIZE]; - - for (header_index = 0; header_index < headers_size.length; header_index++) { - output.write(headers_buffers, offset, headers_size[header_index]); - offset += headers_size[header_index]; - - while (webm_segment != null) { - bloq = getNextBlock(); - - if (addPacketSegment(bloq)) { - while ((read = bloq.data.read(buffer)) > 0) { - output.write(buffer, 0, read); - } - } else { - clearSegmentTable(); - webm_block = bloq; - break; - } - } - } } - private short make_packetHeader(long gran_pos, ByteBuffer buffer, byte[] immediate_page) { - int offset = buffer.position(); + private int make_packetHeader(long gran_pos, @NonNull ByteBuffer buffer, byte[] immediate_page) { short length = HEADER_SIZE; buffer.putInt(0x5367674f);// "OggS" binary string in little-endian @@ -340,17 +258,15 @@ public class OggFromWebMWriter implements Closeable { clearSegmentTable();// clear segment table for next header - int checksum_crc32 = calc_crc32(0x00, buffer.array(), offset, length); + int checksum_crc32 = calc_crc32(0x00, buffer.array(), length); if (immediate_page != null) { - checksum_crc32 = calc_crc32(checksum_crc32, immediate_page, 0, immediate_page.length); - System.arraycopy(immediate_page, 0, buffer.array(), length, immediate_page.length); + checksum_crc32 = calc_crc32(checksum_crc32, immediate_page, immediate_page.length); + buffer.putInt(HEADER_CHECKSUM_OFFSET, checksum_crc32); segment_table_next_timestamp -= TIME_SCALE_NS; } - buffer.putInt(offset + HEADER_CHECKSUM_OFFSET, checksum_crc32); - - return length; + return checksum_crc32; } @Nullable @@ -358,7 +274,7 @@ public class OggFromWebMWriter implements Closeable { if ("A_OPUS".equals(webm_track.codecId)) { return new byte[]{ 0x4F, 0x70, 0x75, 0x73, 0x54, 0x61, 0x67, 0x73,// "OpusTags" binary string - 0x07, 0x00, 0x00, 0x00,// writing application string size + 0x07, 0x00, 0x00, 0x00,// writting application string size 0x4E, 0x65, 0x77, 0x50, 0x69, 0x70, 0x65,// "NewPipe" binary string 0x00, 0x00, 0x00, 0x00// additional tags count (zero means no tags) }; @@ -366,7 +282,7 @@ public class OggFromWebMWriter implements Closeable { return new byte[]{ 0x03,// ???????? 0x76, 0x6f, 0x72, 0x62, 0x69, 0x73,// "vorbis" binary string - 0x07, 0x00, 0x00, 0x00,// writing application string size + 0x07, 0x00, 0x00, 0x00,// writting application string size 0x4E, 0x65, 0x77, 0x50, 0x69, 0x70, 0x65,// "NewPipe" binary string 0x01, 0x00, 0x00, 0x00,// additional tags count (zero means no tags) @@ -387,22 +303,9 @@ public class OggFromWebMWriter implements Closeable { return null; } - private void rewind_source() throws IOException { - source.rewind(); - - webm = new WebMReader(source); - webm.parse(); - webm_track = webm.selectTrack(track_index); - webm_segment = webm.getNextSegment(); - webm_cluster = null; - webm_block = null; - webm_block_last_timecode = 0L; - - segment_table_next_timestamp = TIME_SCALE_NS; - } - - private ByteBuffer byte_buffer(int size) { - return ByteBuffer.allocate(size).order(ByteOrder.LITTLE_ENDIAN); + private void write(ByteBuffer buffer) throws IOException { + output.write(buffer.array(), 0, buffer.position()); + buffer.position(0); } // @@ -460,41 +363,32 @@ public class OggFromWebMWriter implements Closeable { // private void clearSegmentTable() { - if (packet_flag != FLAG_CONTINUED) { - segment_table_next_timestamp += TIME_SCALE_NS; - packet_flag = FLAG_UNSET; - } + segment_table_next_timestamp += TIME_SCALE_NS; + packet_flag = FLAG_UNSET; segment_table_size = 0; } private boolean addPacketSegment(SimpleBlock block) { - if (block == null) { - return false; - } - long timestamp = block.absoluteTimeCodeNs + webm_track.codecDelay; if (timestamp >= segment_table_next_timestamp) { return false; } - boolean result = addPacketSegment((int) block.dataSize); - - if (!result && segment_table_next_timestamp < timestamp) { - // WARNING: ¡¡¡¡ not implemented (lack of documentation) !!!! - packet_flag = FLAG_CONTINUED; - } - - return result; + return addPacketSegment(block.dataSize); } private boolean addPacketSegment(int size) { + if (size > 65025) { + throw new UnsupportedOperationException("page size cannot be larger than 65025"); + } + int available = (segment_table.length - segment_table_size) * 255; - boolean extra = size == 255; + boolean extra = (size % 255) == 0; if (extra) { // add a zero byte entry in the table - // required to indicate the sample size is exactly 255 + // required to indicate the sample size is multiple of 255 available -= 255; } @@ -528,12 +422,10 @@ public class OggFromWebMWriter implements Closeable { } } - private int calc_crc32(int initial_crc, byte[] buffer, int offset, int size) { - size += offset; - - for (; offset < size; offset++) { + private int calc_crc32(int initial_crc, byte[] buffer, int size) { + for (int i = 0; i < size; i++) { int reg = (initial_crc >>> 24) & 0xff; - initial_crc = (initial_crc << 8) ^ crc32_table[reg ^ (buffer[offset] & 0xff)]; + initial_crc = (initial_crc << 8) ^ crc32_table[reg ^ (buffer[i] & 0xff)]; } return initial_crc; diff --git a/app/src/main/java/org/schabi/newpipe/streams/WebMReader.java b/app/src/main/java/org/schabi/newpipe/streams/WebMReader.java index 13c15370d..4cb96d901 100644 --- a/app/src/main/java/org/schabi/newpipe/streams/WebMReader.java +++ b/app/src/main/java/org/schabi/newpipe/streams/WebMReader.java @@ -368,7 +368,7 @@ public class WebMReader { obj.trackNumber = readEncodedNumber(); obj.relativeTimeCode = stream.readShort(); obj.flags = (byte) stream.read(); - obj.dataSize = (ref.offset + ref.size) - stream.position(); + obj.dataSize = (int) ((ref.offset + ref.size) - stream.position()); obj.createdFromBlock = ref.type == ID_Block; // NOTE: lacing is not implemented, and will be mixed with the stream data @@ -465,7 +465,7 @@ public class WebMReader { public short relativeTimeCode; public long absoluteTimeCodeNs; public byte flags; - public long dataSize; + public int dataSize; private final Element ref; public boolean isKeyframe() { diff --git a/app/src/main/java/us/shandian/giga/get/DownloadInitializer.java b/app/src/main/java/us/shandian/giga/get/DownloadInitializer.java index 593feafa7..17a2a7403 100644 --- a/app/src/main/java/us/shandian/giga/get/DownloadInitializer.java +++ b/app/src/main/java/us/shandian/giga/get/DownloadInitializer.java @@ -14,6 +14,7 @@ import java.nio.channels.ClosedByInterruptException; import us.shandian.giga.util.Utility; import static org.schabi.newpipe.BuildConfig.DEBUG; +import static us.shandian.giga.get.DownloadMission.ERROR_HTTP_FORBIDDEN; public class DownloadInitializer extends Thread { private final static String TAG = "DownloadInitializer"; @@ -29,9 +30,9 @@ public class DownloadInitializer extends Thread { mConn = null; } - private static void safeClose(HttpURLConnection con) { + private void dispose() { try { - con.getInputStream().close(); + mConn.getInputStream().close(); } catch (Exception e) { // nothing to do } @@ -52,9 +53,9 @@ public class DownloadInitializer extends Thread { long lowestSize = Long.MAX_VALUE; for (int i = 0; i < mMission.urls.length && mMission.running; i++) { - mConn = mMission.openConnection(mMission.urls[i], mId, -1, -1); + mConn = mMission.openConnection(mMission.urls[i], true, -1, -1); mMission.establishConnection(mId, mConn); - safeClose(mConn); + dispose(); if (Thread.interrupted()) return; long length = Utility.getContentLength(mConn); @@ -82,9 +83,9 @@ public class DownloadInitializer extends Thread { } } else { // ask for the current resource length - mConn = mMission.openConnection(mId, -1, -1); + mConn = mMission.openConnection(true, -1, -1); mMission.establishConnection(mId, mConn); - safeClose(mConn); + dispose(); if (!mMission.running || Thread.interrupted()) return; @@ -108,9 +109,9 @@ public class DownloadInitializer extends Thread { } } else { // Open again - mConn = mMission.openConnection(mId, mMission.length - 10, mMission.length); + mConn = mMission.openConnection(true, mMission.length - 10, mMission.length); mMission.establishConnection(mId, mConn); - safeClose(mConn); + dispose(); if (!mMission.running || Thread.interrupted()) return; @@ -171,7 +172,14 @@ public class DownloadInitializer extends Thread { } catch (InterruptedIOException | ClosedByInterruptException e) { return; } catch (Exception e) { - if (!mMission.running) return; + if (!mMission.running || super.isInterrupted()) return; + + if (e instanceof DownloadMission.HttpError && ((DownloadMission.HttpError) e).statusCode == ERROR_HTTP_FORBIDDEN) { + // for youtube streams. The url has expired + interrupt(); + mMission.doRecover(e); + return; + } if (e instanceof IOException && e.getMessage().contains("Permission denied")) { mMission.notifyError(DownloadMission.ERROR_PERMISSION_DENIED, e); @@ -194,13 +202,6 @@ public class DownloadInitializer extends Thread { @Override public void interrupt() { super.interrupt(); - - if (mConn != null) { - try { - mConn.disconnect(); - } catch (Exception e) { - // nothing to do - } - } + if (mConn != null) dispose(); } } diff --git a/app/src/main/java/us/shandian/giga/get/DownloadMission.java b/app/src/main/java/us/shandian/giga/get/DownloadMission.java index 77b417118..918d6dbea 100644 --- a/app/src/main/java/us/shandian/giga/get/DownloadMission.java +++ b/app/src/main/java/us/shandian/giga/get/DownloadMission.java @@ -204,22 +204,24 @@ public class DownloadMission extends Mission { /** * Opens a connection * - * @param threadId id of the calling thread, used only for debugging - * @param rangeStart range start - * @param rangeEnd range end + * @param headRequest {@code true} for use {@code HEAD} request method, otherwise, {@code GET} is used + * @param rangeStart range start + * @param rangeEnd range end * @return a {@link java.net.URLConnection URLConnection} linking to the URL. * @throws IOException if an I/O exception occurs. */ - HttpURLConnection openConnection(int threadId, long rangeStart, long rangeEnd) throws IOException { - return openConnection(urls[current], threadId, rangeStart, rangeEnd); + HttpURLConnection openConnection(boolean headRequest, long rangeStart, long rangeEnd) throws IOException { + return openConnection(urls[current], headRequest, rangeStart, rangeEnd); } - HttpURLConnection openConnection(String url, int threadId, long rangeStart, long rangeEnd) throws IOException { + HttpURLConnection openConnection(String url, boolean headRequest, long rangeStart, long rangeEnd) throws IOException { HttpURLConnection conn = (HttpURLConnection) new URL(url).openConnection(); conn.setInstanceFollowRedirects(true); conn.setRequestProperty("User-Agent", DownloaderImpl.USER_AGENT); conn.setRequestProperty("Accept", "*/*"); + if (headRequest) conn.setRequestMethod("HEAD"); + // BUG workaround: switching between networks can freeze the download forever conn.setConnectTimeout(30000); conn.setReadTimeout(10000); @@ -229,10 +231,6 @@ public class DownloadMission extends Mission { if (rangeEnd > 0) req += rangeEnd; conn.setRequestProperty("Range", req); - - if (DEBUG) { - Log.d(TAG, threadId + ":" + conn.getRequestProperty("Range")); - } } return conn; @@ -245,13 +243,14 @@ public class DownloadMission extends Mission { * @throws HttpError if the HTTP Status-Code is not satisfiable */ void establishConnection(int threadId, HttpURLConnection conn) throws IOException, HttpError { - conn.connect(); int statusCode = conn.getResponseCode(); if (DEBUG) { + Log.d(TAG, threadId + ":Range=" + conn.getRequestProperty("Range")); Log.d(TAG, threadId + ":Content-Length=" + conn.getContentLength() + " Code:" + statusCode); } + switch (statusCode) { case 204: case 205: @@ -676,6 +675,15 @@ public class DownloadMission extends Mission { return (isPsFailed() || errCode == ERROR_POSTPROCESSING_HOLD) || isFinished(); } + /** + * Indicates if mission urls has expired and there an attempt to renovate them + * + * @return {@code true} if the mission is running a recovery procedure, otherwise, {@code false} + */ + public boolean isRecovering() { + return threads != null && threads.length > 0 && threads[0] instanceof DownloadRunnable && threads[0].isAlive(); + } + private boolean doPostprocessing() { if (psAlgorithm == null || psState == 2) return true; @@ -742,10 +750,8 @@ public class DownloadMission extends Mission { } } - // set the current download url to null in case if the recovery - // process is canceled. Next time start() method is called the - // recovery will be executed, saving time - urls[current] = null; + errCode = ERROR_NOTHING; + errObject = null; if (recoveryInfo[current].attempts >= maxRetry) { recoveryInfo[current].attempts = 0; diff --git a/app/src/main/java/us/shandian/giga/get/DownloadMissionRecover.java b/app/src/main/java/us/shandian/giga/get/DownloadMissionRecover.java index 9abd93717..5efbd1153 100644 --- a/app/src/main/java/us/shandian/giga/get/DownloadMissionRecover.java +++ b/app/src/main/java/us/shandian/giga/get/DownloadMissionRecover.java @@ -10,10 +10,12 @@ import org.schabi.newpipe.extractor.stream.SubtitlesStream; import org.schabi.newpipe.extractor.stream.VideoStream; import java.io.IOException; +import java.io.InterruptedIOException; import java.net.HttpURLConnection; import java.nio.channels.ClosedByInterruptException; import java.util.List; +import static us.shandian.giga.get.DownloadMission.ERROR_NOTHING; import static us.shandian.giga.get.DownloadMission.ERROR_RESOURCE_GONE; public class DownloadMissionRecover extends Thread { @@ -21,14 +23,17 @@ public class DownloadMissionRecover extends Thread { static final int mID = -3; private final DownloadMission mMission; - private final MissionRecoveryInfo mRecovery; private final Exception mFromError; + private final boolean notInitialized; + private HttpURLConnection mConn; + private MissionRecoveryInfo mRecovery; + private StreamExtractor mExtractor; DownloadMissionRecover(DownloadMission mission, Exception originError) { mMission = mission; mFromError = originError; - mRecovery = mission.recoveryInfo[mission.current]; + notInitialized = mission.blocks == null && mission.current == 0; } @Override @@ -38,28 +43,78 @@ public class DownloadMissionRecover extends Thread { return; } + /*if (mMission.source.startsWith(MissionRecoveryInfo.DIRECT_SOURCE)) { + resolve(mMission.source.substring(MissionRecoveryInfo.DIRECT_SOURCE.length())); + return; + }*/ + try { - /*if (mMission.source.startsWith(MissionRecoveryInfo.DIRECT_SOURCE)) { - resolve(mMission.source.substring(MissionRecoveryInfo.DIRECT_SOURCE.length())); - return; - }*/ - StreamingService svr = NewPipe.getServiceByUrl(mMission.source); - - if (svr == null) { - throw new RuntimeException("Unknown source service"); - } - - StreamExtractor extractor = svr.getStreamExtractor(mMission.source); - extractor.fetchPage(); - + mExtractor = svr.getStreamExtractor(mMission.source); + mExtractor.fetchPage(); + } catch (InterruptedIOException | ClosedByInterruptException e) { + return; + } catch (Exception e) { if (!mMission.running || super.isInterrupted()) return; + mMission.notifyError(e); + return; + } + // maybe the following check is redundant + if (!mMission.running || super.isInterrupted()) return; + + if (!notInitialized) { + // set the current download url to null in case if the recovery + // process is canceled. Next time start() method is called the + // recovery will be executed, saving time + mMission.urls[mMission.current] = null; + + mRecovery = mMission.recoveryInfo[mMission.current]; + resolveStream(); + return; + } + + Log.w(TAG, "mission is not fully initialized, this will take a while"); + + try { + for (; mMission.current < mMission.urls.length; mMission.current++) { + mRecovery = mMission.recoveryInfo[mMission.current]; + + if (test()) continue; + if (!mMission.running) return; + + resolveStream(); + if (!mMission.running) return; + + // before continue, check if the current stream was resolved + if (mMission.urls[mMission.current] == null || mMission.errCode != ERROR_NOTHING) { + break; + } + } + } finally { + mMission.current = 0; + } + + mMission.writeThisToFile(); + + if (!mMission.running || super.isInterrupted()) return; + + mMission.running = false; + mMission.start(); + } + + private void resolveStream() { + if (mExtractor.getErrorMessage() != null) { + mMission.notifyError(mFromError); + return; + } + + try { String url = null; - switch (mMission.kind) { + switch (mRecovery.kind) { case 'a': - for (AudioStream audio : extractor.getAudioStreams()) { + for (AudioStream audio : mExtractor.getAudioStreams()) { if (audio.average_bitrate == mRecovery.desiredBitrate && audio.getFormat() == mRecovery.format) { url = audio.getUrl(); break; @@ -69,9 +124,9 @@ public class DownloadMissionRecover extends Thread { case 'v': List videoStreams; if (mRecovery.desired2) - videoStreams = extractor.getVideoOnlyStreams(); + videoStreams = mExtractor.getVideoOnlyStreams(); else - videoStreams = extractor.getVideoStreams(); + videoStreams = mExtractor.getVideoStreams(); for (VideoStream video : videoStreams) { if (video.resolution.equals(mRecovery.desired) && video.getFormat() == mRecovery.format) { url = video.getUrl(); @@ -80,7 +135,7 @@ public class DownloadMissionRecover extends Thread { } break; case 's': - for (SubtitlesStream subtitles : extractor.getSubtitles(mRecovery.format)) { + for (SubtitlesStream subtitles : mExtractor.getSubtitles(mRecovery.format)) { String tag = subtitles.getLanguageTag(); if (tag.equals(mRecovery.desired) && subtitles.isAutoGenerated() == mRecovery.desired2) { url = subtitles.getURL(); @@ -114,7 +169,7 @@ public class DownloadMissionRecover extends Thread { ////// Validate the http resource doing a range request ///////////////////// try { - mConn = mMission.openConnection(url, mID, mMission.length - 10, mMission.length); + mConn = mMission.openConnection(url, true, mMission.length - 10, mMission.length); mConn.setRequestProperty("If-Range", mRecovery.validateCondition); mMission.establishConnection(mID, mConn); @@ -140,22 +195,24 @@ public class DownloadMissionRecover extends Thread { if (!mMission.running || e instanceof ClosedByInterruptException) return; throw e; } finally { - this.interrupt(); + disconnect(); } } private void recover(String url, boolean stale) { Log.i(TAG, - String.format("download recovered name=%s isStale=%s url=%s", mMission.storage.getName(), stale, url) + String.format("recover() name=%s isStale=%s url=%s", mMission.storage.getName(), stale, url) ); + mMission.urls[mMission.current] = url; + mRecovery.attempts = 0; + if (url == null) { mMission.notifyError(ERROR_RESOURCE_GONE, null); return; } - mMission.urls[mMission.current] = url; - mRecovery.attempts = 0; + if (notInitialized) return; if (stale) { mMission.resetState(false, false, DownloadMission.ERROR_NOTHING); @@ -208,15 +265,40 @@ public class DownloadMissionRecover extends Thread { return range; } + private boolean test() { + if (mMission.urls[mMission.current] == null) return false; + + try { + mConn = mMission.openConnection(mMission.urls[mMission.current], true, -1, -1); + mMission.establishConnection(mID, mConn); + + if (mConn.getResponseCode() == 200) return true; + } catch (Exception e) { + // nothing to do + } finally { + disconnect(); + } + + return false; + } + + private void disconnect() { + try { + try { + mConn.getInputStream().close(); + } finally { + mConn.disconnect(); + } + } catch (Exception e) { + // nothing to do + } finally { + mConn = null; + } + } + @Override public void interrupt() { super.interrupt(); - if (mConn != null) { - try { - mConn.disconnect(); - } catch (Exception e) { - // nothing to do - } - } + if (mConn != null) disconnect(); } } diff --git a/app/src/main/java/us/shandian/giga/get/DownloadRunnable.java b/app/src/main/java/us/shandian/giga/get/DownloadRunnable.java index 1d2a4eee7..b0dc793bc 100644 --- a/app/src/main/java/us/shandian/giga/get/DownloadRunnable.java +++ b/app/src/main/java/us/shandian/giga/get/DownloadRunnable.java @@ -80,7 +80,7 @@ public class DownloadRunnable extends Thread { } try { - mConn = mMission.openConnection(mId, start, end); + mConn = mMission.openConnection(false, start, end); mMission.establishConnection(mId, mConn); // check if the download can be resumed diff --git a/app/src/main/java/us/shandian/giga/get/DownloadRunnableFallback.java b/app/src/main/java/us/shandian/giga/get/DownloadRunnableFallback.java index b5937c577..e64322b48 100644 --- a/app/src/main/java/us/shandian/giga/get/DownloadRunnableFallback.java +++ b/app/src/main/java/us/shandian/giga/get/DownloadRunnableFallback.java @@ -35,7 +35,11 @@ public class DownloadRunnableFallback extends Thread { private void dispose() { try { - if (mIs != null) mIs.close(); + try { + if (mIs != null) mIs.close(); + } finally { + mConn.disconnect(); + } } catch (IOException e) { // nothing to do } @@ -68,7 +72,13 @@ public class DownloadRunnableFallback extends Thread { long rangeStart = (mMission.unknownLength || start < 1) ? -1 : start; int mId = 1; - mConn = mMission.openConnection(mId, rangeStart, -1); + mConn = mMission.openConnection(false, rangeStart, -1); + + if (mRetryCount == 0 && rangeStart == -1) { + // workaround: bypass android connection pool + mConn.setRequestProperty("Range", "bytes=0-"); + } + mMission.establishConnection(mId, mConn); // check if the download can be resumed @@ -96,6 +106,8 @@ public class DownloadRunnableFallback extends Thread { mMission.notifyProgress(len); } + dispose(); + // if thread goes interrupted check if the last part is written. This avoid re-download the whole file done = len == -1; } catch (Exception e) { @@ -107,8 +119,8 @@ public class DownloadRunnableFallback extends Thread { if (e instanceof HttpError && ((HttpError) e).statusCode == ERROR_HTTP_FORBIDDEN) { // for youtube streams. The url has expired, recover - mMission.doRecover(e); dispose(); + mMission.doRecover(e); return; } @@ -125,8 +137,6 @@ public class DownloadRunnableFallback extends Thread { return; } - dispose(); - if (done) { mMission.notifyFinished(); } else { diff --git a/app/src/main/java/us/shandian/giga/get/MissionRecoveryInfo.java b/app/src/main/java/us/shandian/giga/get/MissionRecoveryInfo.java index 553ba6d89..bd1d9bc49 100644 --- a/app/src/main/java/us/shandian/giga/get/MissionRecoveryInfo.java +++ b/app/src/main/java/us/shandian/giga/get/MissionRecoveryInfo.java @@ -16,25 +16,28 @@ public class MissionRecoveryInfo implements Serializable, Parcelable { private static final long serialVersionUID = 0L; //public static final String DIRECT_SOURCE = "direct-source://"; - public MediaFormat format; + MediaFormat format; String desired; boolean desired2; int desiredBitrate; + byte kind; + String validateCondition = null; transient int attempts = 0; - String validateCondition = null; - public MissionRecoveryInfo(@NonNull Stream stream) { if (stream instanceof AudioStream) { desiredBitrate = ((AudioStream) stream).average_bitrate; desired2 = false; + kind = 'a'; } else if (stream instanceof VideoStream) { desired = ((VideoStream) stream).getResolution(); desired2 = ((VideoStream) stream).isVideoOnly(); + kind = 'v'; } else if (stream instanceof SubtitlesStream) { desired = ((SubtitlesStream) stream).getLanguageTag(); desired2 = ((SubtitlesStream) stream).isAutoGenerated(); + kind = 's'; } else { throw new RuntimeException("Unknown stream kind"); } @@ -43,6 +46,38 @@ public class MissionRecoveryInfo implements Serializable, Parcelable { if (format == null) throw new NullPointerException("Stream format cannot be null"); } + @NonNull + @Override + public String toString() { + String info; + StringBuilder str = new StringBuilder(); + str.append("type="); + switch (kind) { + case 'a': + str.append("audio"); + info = "bitrate=" + desiredBitrate; + break; + case 'v': + str.append("video"); + info = "quality=" + desired + " videoOnly=" + desired2; + break; + case 's': + str.append("subtitles"); + info = "language=" + desired + " autoGenerated=" + desired2; + break; + default: + info = ""; + str.append("other"); + } + + str.append(" format=") + .append(format.getName()) + .append(' ') + .append(info); + + return str.toString(); + } + @Override public int describeContents() { return 0; @@ -54,6 +89,7 @@ public class MissionRecoveryInfo implements Serializable, Parcelable { parcel.writeString(this.desired); parcel.writeInt(this.desired2 ? 0x01 : 0x00); parcel.writeInt(this.desiredBitrate); + parcel.writeByte(this.kind); parcel.writeString(this.validateCondition); } @@ -62,6 +98,7 @@ public class MissionRecoveryInfo implements Serializable, Parcelable { this.desired = parcel.readString(); this.desired2 = parcel.readInt() != 0x00; this.desiredBitrate = parcel.readInt(); + this.kind = parcel.readByte(); this.validateCondition = parcel.readString(); } diff --git a/app/src/main/java/us/shandian/giga/ui/adapter/MissionAdapter.java b/app/src/main/java/us/shandian/giga/ui/adapter/MissionAdapter.java index 6c6198750..78fd7ea9d 100644 --- a/app/src/main/java/us/shandian/giga/ui/adapter/MissionAdapter.java +++ b/app/src/main/java/us/shandian/giga/ui/adapter/MissionAdapter.java @@ -36,6 +36,7 @@ import android.widget.Toast; import org.schabi.newpipe.BuildConfig; import org.schabi.newpipe.R; +import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.report.ErrorActivity; import org.schabi.newpipe.report.UserAction; import org.schabi.newpipe.util.NavigationHelper; @@ -44,11 +45,11 @@ import java.io.File; import java.lang.ref.WeakReference; import java.net.URI; import java.util.ArrayList; -import java.util.Collections; import us.shandian.giga.get.DownloadMission; import us.shandian.giga.get.FinishedMission; import us.shandian.giga.get.Mission; +import us.shandian.giga.get.MissionRecoveryInfo; import us.shandian.giga.io.StoredFileHelper; import us.shandian.giga.service.DownloadManager; import us.shandian.giga.service.DownloadManagerService; @@ -234,7 +235,7 @@ public class MissionAdapter extends Adapter implements Handler.Callb // hide on error // show if current resource length is not fetched // show if length is unknown - h.progress.setMarquee(!hasError && (!mission.isInitialized() || mission.unknownLength)); + h.progress.setMarquee(mission.isRecovering() || !hasError && (!mission.isInitialized() || mission.unknownLength)); float progress; if (mission.unknownLength) { @@ -463,13 +464,13 @@ public class MissionAdapter extends Adapter implements Handler.Callb break; case ERROR_POSTPROCESSING: case ERROR_POSTPROCESSING_HOLD: - showError(mission.errObject, UserAction.DOWNLOAD_POSTPROCESSING, R.string.error_postprocessing_failed); + showError(mission, UserAction.DOWNLOAD_POSTPROCESSING, R.string.error_postprocessing_failed); return; case ERROR_INSUFFICIENT_STORAGE: msg = R.string.error_insufficient_storage; break; case ERROR_UNKNOWN_EXCEPTION: - showError(mission.errObject, UserAction.DOWNLOAD_FAILED, R.string.general_error); + showError(mission, UserAction.DOWNLOAD_FAILED, R.string.general_error); return; case ERROR_PROGRESS_LOST: msg = R.string.error_progress_lost; @@ -486,7 +487,7 @@ public class MissionAdapter extends Adapter implements Handler.Callb } else if (mission.errObject == null) { msgEx = "(not_decelerated_error_code)"; } else { - showError(mission.errObject, UserAction.DOWNLOAD_FAILED, msg); + showError(mission, UserAction.DOWNLOAD_FAILED, msg); return; } break; @@ -503,7 +504,7 @@ public class MissionAdapter extends Adapter implements Handler.Callb if (mission.errObject != null && (mission.errCode < 100 || mission.errCode >= 600)) { @StringRes final int mMsg = msg; builder.setPositiveButton(R.string.error_report_title, (dialog, which) -> - showError(mission.errObject, UserAction.DOWNLOAD_FAILED, mMsg) + showError(mission, UserAction.DOWNLOAD_FAILED, mMsg) ); } @@ -513,13 +514,30 @@ public class MissionAdapter extends Adapter implements Handler.Callb .show(); } - private void showError(Exception exception, UserAction action, @StringRes int reason) { + private void showError(DownloadMission mission, UserAction action, @StringRes int reason) { + StringBuilder request = new StringBuilder(256); + request.append(mission.source); + + request.append(" ["); + if (mission.recoveryInfo != null) { + for (MissionRecoveryInfo recovery : mission.recoveryInfo) + request.append(" {").append(recovery.toString()).append("} "); + } + request.append("]"); + + String service; + try { + service = NewPipe.getServiceByUrl(mission.source).getServiceInfo().getName(); + } catch (Exception e) { + service = "-"; + } + ErrorActivity.reportError( mContext, - Collections.singletonList(exception), + mission.errObject, null, null, - ErrorActivity.ErrorInfo.make(action, "-", "-", reason) + ErrorActivity.ErrorInfo.make(action, service, request.toString(), reason) ); } diff --git a/app/src/main/res/values-es/strings.xml b/app/src/main/res/values-es/strings.xml index 2f69e62cb..b14aab94b 100644 --- a/app/src/main/res/values-es/strings.xml +++ b/app/src/main/res/values-es/strings.xml @@ -453,7 +453,7 @@ No hay suficiente espacio disponible en el dispositivo Se perdió el progreso porque el archivo fue eliminado Tiempo de espera excedido - El recurso solicitado ya no esta disponible + No se puede recuperar esta descarga Preguntar dónde descargar Se preguntará dónde guardar cada descarga Se le preguntará dónde guardar cada descarga. diff --git a/app/src/main/res/values/strings.xml b/app/src/main/res/values/strings.xml index 2917fb9fd..f929e0d2b 100644 --- a/app/src/main/res/values/strings.xml +++ b/app/src/main/res/values/strings.xml @@ -557,7 +557,7 @@ No space left on device Progress lost, because the file was deleted Connection timeout - The solicited resource is not available anymore + Cannot recover this download Clear finished downloads Are you sure? Continue your %s pending transfers from Downloads From 60d4c8a55df3e28e749e28c578372811bfa5ed77 Mon Sep 17 00:00:00 2001 From: kapodamy Date: Tue, 1 Oct 2019 13:00:16 -0300 Subject: [PATCH 19/28] fallback for pending downloads directory --- .../giga/service/DownloadManager.java | 31 +++++++++++++------ 1 file changed, 21 insertions(+), 10 deletions(-) diff --git a/app/src/main/java/us/shandian/giga/service/DownloadManager.java b/app/src/main/java/us/shandian/giga/service/DownloadManager.java index a859a87ca..89c44638d 100644 --- a/app/src/main/java/us/shandian/giga/service/DownloadManager.java +++ b/app/src/main/java/us/shandian/giga/service/DownloadManager.java @@ -37,6 +37,7 @@ public class DownloadManager { public static final String TAG_AUDIO = "audio"; public static final String TAG_VIDEO = "video"; + private static final String DOWNLOADS_METADATA_FOLDER = "pending_downloads"; private final FinishedMissionStore mFinishedMissionStore; @@ -75,24 +76,33 @@ public class DownloadManager { mPendingMissionsDir = getPendingDir(context); if (!Utility.mkdir(mPendingMissionsDir, false)) { - throw new RuntimeException("failed to create pending_downloads in data directory"); + throw new RuntimeException("failed to create " + DOWNLOADS_METADATA_FOLDER + " directory"); } loadPendingMissions(context); } private static File getPendingDir(@NonNull Context context) { - //File dir = new File(ContextCompat.getDataDir(context), "pending_downloads"); - File dir = context.getExternalFilesDir("pending_downloads"); + File dir = context.getExternalFilesDir(DOWNLOADS_METADATA_FOLDER); + if (testDir(dir)) return dir; - if (dir == null) { - // One of the following paths are not accessible ¿unmounted internal memory? - // /storage/emulated/0/Android/data/org.schabi.newpipe[.debug]/pending_downloads - // /sdcard/Android/data/org.schabi.newpipe[.debug]/pending_downloads - Log.w(TAG, "path to pending downloads are not accessible"); + dir = new File(context.getFilesDir(), DOWNLOADS_METADATA_FOLDER); + if (testDir(dir)) return dir; + + throw new RuntimeException("path to pending downloads are not accessible"); + } + + private static boolean testDir(@Nullable File dir) { + if (dir == null) return false; + + try { + File tmp = new File(dir, ".tmp"); + if (!tmp.createNewFile()) return false; + return tmp.delete();// if the file was created, SHOULD BE deleted too + } catch (Exception e) { + Log.e(TAG, "testDir() failed: " + dir.getAbsolutePath(), e); + return false; } - - return dir; } /** @@ -132,6 +142,7 @@ public class DownloadManager { for (File sub : subs) { if (!sub.isFile()) continue; + if (sub.getName().equals(".tmp")) continue; DownloadMission mis = Utility.readFromFile(sub); if (mis == null || mis.isFinished()) { From da052df106a165aa40774a00a4bfc008a1bd055c Mon Sep 17 00:00:00 2001 From: kapodamy Date: Tue, 1 Oct 2019 15:01:17 -0300 Subject: [PATCH 20/28] update DownloadManager.java * check if the directory pending_downloads was created --- .../java/us/shandian/giga/service/DownloadManager.java | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/app/src/main/java/us/shandian/giga/service/DownloadManager.java b/app/src/main/java/us/shandian/giga/service/DownloadManager.java index 89c44638d..2d1e9cd00 100644 --- a/app/src/main/java/us/shandian/giga/service/DownloadManager.java +++ b/app/src/main/java/us/shandian/giga/service/DownloadManager.java @@ -75,10 +75,6 @@ public class DownloadManager { mMissionsFinished = loadFinishedMissions(); mPendingMissionsDir = getPendingDir(context); - if (!Utility.mkdir(mPendingMissionsDir, false)) { - throw new RuntimeException("failed to create " + DOWNLOADS_METADATA_FOLDER + " directory"); - } - loadPendingMissions(context); } @@ -96,6 +92,11 @@ public class DownloadManager { if (dir == null) return false; try { + if (!Utility.mkdir(dir, false)) { + Log.e(TAG, "testDir() cannot create the directory in path: " + dir.getAbsolutePath()); + return false; + } + File tmp = new File(dir, ".tmp"); if (!tmp.createNewFile()) return false; return tmp.delete();// if the file was created, SHOULD BE deleted too From 8a992d4c47180a2fa2f4911990374c94acc9505f Mon Sep 17 00:00:00 2001 From: kapodamy Date: Tue, 1 Oct 2019 16:28:45 -0300 Subject: [PATCH 21/28] update WebMWriter.java fix wrong cue generation --- app/src/main/java/org/schabi/newpipe/streams/WebMWriter.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/src/main/java/org/schabi/newpipe/streams/WebMWriter.java b/app/src/main/java/org/schabi/newpipe/streams/WebMWriter.java index 1bf994b1e..8525fabd2 100644 --- a/app/src/main/java/org/schabi/newpipe/streams/WebMWriter.java +++ b/app/src/main/java/org/schabi/newpipe/streams/WebMWriter.java @@ -249,7 +249,7 @@ public class WebMWriter implements Closeable { nextCueTime += DEFAULT_CUES_EACH_MS; } keyFrames.add( - new KeyFrame(baseSegmentOffset, currentClusterOffset - 7, written, bTimecode.length, bloq.absoluteTimecode) + new KeyFrame(baseSegmentOffset, currentClusterOffset - 8, written, bTimecode.length, bloq.absoluteTimecode) ); } } From 763995d4c98fe43b3b037e75b4b0ec5c17286ad0 Mon Sep 17 00:00:00 2001 From: kapodamy Date: Wed, 2 Oct 2019 13:31:45 -0300 Subject: [PATCH 22/28] update DownloadDialog.java keep *.opus extension --- .../main/java/org/schabi/newpipe/download/DownloadDialog.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/src/main/java/org/schabi/newpipe/download/DownloadDialog.java b/app/src/main/java/org/schabi/newpipe/download/DownloadDialog.java index 0006b3c12..60b6192be 100644 --- a/app/src/main/java/org/schabi/newpipe/download/DownloadDialog.java +++ b/app/src/main/java/org/schabi/newpipe/download/DownloadDialog.java @@ -562,7 +562,7 @@ public class DownloadDialog extends DialogFragment implements RadioGroup.OnCheck mainStorage = mainStorageVideo; format = videoStreamsAdapter.getItem(selectedVideoIndex).getFormat(); mime = format.mimeType; - filename += format == MediaFormat.OPUS ? "ogg" : format.suffix; + filename += format.suffix; break; case R.id.subtitle_button: mainStorage = mainStorageVideo;// subtitle & video files go together From e6d9d8e26d0660e4e889fd733e2011302360ca6b Mon Sep 17 00:00:00 2001 From: kapodamy Date: Wed, 9 Oct 2019 23:49:23 -0300 Subject: [PATCH 23/28] code cleanup * migrate few annotations to androidx * mission recovery: better error handling (except StreamExtractor.getErrorMessage() method always returns an error) * post-processing: more detailed progress [file specific changes] DownloadMission.java * remove redundant/boilerplate code (again) * make few variables volatile * better file "length" approximation * use "done" variable to count the amount of bytes downloaded (simplify percent calc in UI code) Postprocessing.java * if case of error use "ERROR_POSTPROCESSING" instead of "ERROR_UNKNOWN_EXCEPTION" * simplify source stream init DownloadManager.java * move all "service message sending" code to DownloadMission * remove not implemented method "notifyUserPendingDownloads()" also his unused strings DownloadManagerService.java * use START_STICKY instead of START_NOT_STICKY * simplify addMissionEventListener()/removeMissionEventListener() methods (always are called from the main thread) Deleter.java * better method definition MissionAdapter.java * better method definition * code cleanup * the UI is now refreshed every 750ms * simplify download progress calculation * indicates if the download is actually recovering * smooth download speed measure * show estimated remain time MainFragment.java: * check if viewPager is null (issued by "Apply changes" feature of Android Studio) --- .../newpipe/fragments/MainFragment.java | 9 + .../newpipe/streams/OggFromWebMWriter.java | 2 +- .../giga/get/DownloadInitializer.java | 5 +- .../us/shandian/giga/get/DownloadMission.java | 287 ++++++++--------- .../giga/get/DownloadMissionRecover.java | 160 +++++----- .../shandian/giga/get/DownloadRunnable.java | 5 +- .../giga/get/DownloadRunnableFallback.java | 29 +- .../us/shandian/giga/get/FinishedMission.java | 6 +- .../giga/get/MissionRecoveryInfo.java | 10 +- .../giga/io/ChunkFileInputStream.java | 19 +- .../shandian/giga/io/CircularFileWriter.java | 30 +- .../us/shandian/giga/io/ProgressReport.java | 11 + .../postprocessing/OggFromWebmDemuxer.java | 2 +- .../giga/postprocessing/Postprocessing.java | 54 ++-- .../giga/service/DownloadManager.java | 57 +--- .../giga/service/DownloadManagerService.java | 45 ++- .../giga/ui/adapter/MissionAdapter.java | 302 +++++++++--------- .../us/shandian/giga/ui/common/Deleter.java | 9 +- .../giga/ui/common/ProgressDrawable.java | 5 +- .../giga/ui/fragment/MissionsFragment.java | 45 ++- .../java/us/shandian/giga/util/Utility.java | 50 ++- app/src/main/res/values-ar/strings.xml | 1 - app/src/main/res/values-be/strings.xml | 1 - app/src/main/res/values-cmn/strings.xml | 1 - app/src/main/res/values-cs/strings.xml | 1 - app/src/main/res/values-da/strings.xml | 1 - app/src/main/res/values-de/strings.xml | 1 - app/src/main/res/values-el/strings.xml | 1 - app/src/main/res/values-es/strings.xml | 2 +- app/src/main/res/values-et/strings.xml | 1 - app/src/main/res/values-eu/strings.xml | 1 - app/src/main/res/values-fr/strings.xml | 1 - app/src/main/res/values-he/strings.xml | 1 - app/src/main/res/values-hr/strings.xml | 1 - app/src/main/res/values-id/strings.xml | 1 - app/src/main/res/values-it/strings.xml | 1 - app/src/main/res/values-ja/strings.xml | 1 - app/src/main/res/values-ko/strings.xml | 1 - app/src/main/res/values-ms/strings.xml | 1 - app/src/main/res/values-nb-rNO/strings.xml | 1 - app/src/main/res/values-nl-rBE/strings.xml | 1 - app/src/main/res/values-nl/strings.xml | 1 - app/src/main/res/values-pa/strings.xml | 1 - app/src/main/res/values-pl/strings.xml | 1 - app/src/main/res/values-pt-rBR/strings.xml | 1 - app/src/main/res/values-pt/strings.xml | 1 - app/src/main/res/values-ru/strings.xml | 1 - app/src/main/res/values-sk/strings.xml | 1 - app/src/main/res/values-tr/strings.xml | 1 - app/src/main/res/values-uk/strings.xml | 1 - app/src/main/res/values-vi/strings.xml | 1 - app/src/main/res/values-zh-rTW/strings.xml | 1 - app/src/main/res/values/strings.xml | 2 +- 53 files changed, 554 insertions(+), 622 deletions(-) create mode 100644 app/src/main/java/us/shandian/giga/io/ProgressReport.java diff --git a/app/src/main/java/org/schabi/newpipe/fragments/MainFragment.java b/app/src/main/java/org/schabi/newpipe/fragments/MainFragment.java index 720e0f216..70e0d9fb1 100644 --- a/app/src/main/java/org/schabi/newpipe/fragments/MainFragment.java +++ b/app/src/main/java/org/schabi/newpipe/fragments/MainFragment.java @@ -2,6 +2,15 @@ package org.schabi.newpipe.fragments; import android.content.Context; import android.os.Bundle; +import androidx.annotation.NonNull; +import androidx.annotation.Nullable; +import com.google.android.material.tabs.TabLayout; +import androidx.fragment.app.Fragment; +import androidx.fragment.app.FragmentManager; +import androidx.fragment.app.FragmentPagerAdapter; +import androidx.viewpager.widget.ViewPager; +import androidx.appcompat.app.ActionBar; +import androidx.appcompat.app.AppCompatActivity; import android.util.Log; import android.view.LayoutInflater; import android.view.Menu; diff --git a/app/src/main/java/org/schabi/newpipe/streams/OggFromWebMWriter.java b/app/src/main/java/org/schabi/newpipe/streams/OggFromWebMWriter.java index e6363e423..37bf9c6d7 100644 --- a/app/src/main/java/org/schabi/newpipe/streams/OggFromWebMWriter.java +++ b/app/src/main/java/org/schabi/newpipe/streams/OggFromWebMWriter.java @@ -1,6 +1,6 @@ package org.schabi.newpipe.streams; -import android.support.annotation.NonNull; +import androidx.annotation.NonNull; import org.schabi.newpipe.streams.WebMReader.Cluster; import org.schabi.newpipe.streams.WebMReader.Segment; diff --git a/app/src/main/java/us/shandian/giga/get/DownloadInitializer.java b/app/src/main/java/us/shandian/giga/get/DownloadInitializer.java index 17a2a7403..618200f27 100644 --- a/app/src/main/java/us/shandian/giga/get/DownloadInitializer.java +++ b/app/src/main/java/us/shandian/giga/get/DownloadInitializer.java @@ -1,9 +1,10 @@ package us.shandian.giga.get; -import androidx.annotation.NonNull; import android.text.TextUtils; import android.util.Log; +import androidx.annotation.NonNull; + import org.schabi.newpipe.streams.io.SharpStream; import java.io.IOException; @@ -177,7 +178,7 @@ public class DownloadInitializer extends Thread { if (e instanceof DownloadMission.HttpError && ((DownloadMission.HttpError) e).statusCode == ERROR_HTTP_FORBIDDEN) { // for youtube streams. The url has expired interrupt(); - mMission.doRecover(e); + mMission.doRecover(ERROR_HTTP_FORBIDDEN); return; } diff --git a/app/src/main/java/us/shandian/giga/get/DownloadMission.java b/app/src/main/java/us/shandian/giga/get/DownloadMission.java index 918d6dbea..5ef72162c 100644 --- a/app/src/main/java/us/shandian/giga/get/DownloadMission.java +++ b/app/src/main/java/us/shandian/giga/get/DownloadMission.java @@ -4,18 +4,21 @@ import android.os.Handler; import android.util.Log; import androidx.annotation.Nullable; +import androidx.annotation.NonNull; import org.schabi.newpipe.DownloaderImpl; import java.io.File; import java.io.FileNotFoundException; import java.io.IOException; +import java.io.InterruptedIOException; import java.io.Serializable; import java.net.ConnectException; import java.net.HttpURLConnection; import java.net.SocketTimeoutException; import java.net.URL; import java.net.UnknownHostException; +import java.nio.channels.ClosedByInterruptException; import javax.net.ssl.SSLException; @@ -27,7 +30,7 @@ import us.shandian.giga.util.Utility; import static org.schabi.newpipe.BuildConfig.DEBUG; public class DownloadMission extends Mission { - private static final long serialVersionUID = 6L;// last bump: 28 september 2019 + private static final long serialVersionUID = 6L;// last bump: 07 october 2019 static final int BUFFER_SIZE = 64 * 1024; static final int BLOCK_SIZE = 512 * 1024; @@ -61,9 +64,9 @@ public class DownloadMission extends Mission { public String[] urls; /** - * Number of bytes downloaded + * Number of bytes downloaded and written */ - public long done; + public volatile long done; /** * Indicates a file generated dynamically on the web server @@ -119,7 +122,7 @@ public class DownloadMission extends Mission { /** * Download/File resume offset in fallback mode (if applicable) {@link DownloadRunnableFallback} */ - long fallbackResumeOffset; + volatile long fallbackResumeOffset; /** * Maximum of download threads running, chosen by the user @@ -132,22 +135,23 @@ public class DownloadMission extends Mission { public MissionRecoveryInfo[] recoveryInfo; private transient int finishCount; - public transient boolean running; + public transient volatile boolean running; public boolean enqueued; public int errCode = ERROR_NOTHING; public Exception errObject = null; public transient Handler mHandler; - private transient boolean mWritingToFile; private transient boolean[] blockAcquired; + private transient long writingToFileNext; + private transient volatile boolean writingToFile; + final Object LOCK = new Lock(); - private transient boolean deleted; - - public transient volatile Thread[] threads = new Thread[0]; - private transient Thread init = null; + @NonNull + public transient Thread[] threads = new Thread[0]; + public transient Thread init = null; public DownloadMission(String[] urls, StoredFileHelper storage, char kind, Postprocessing psInstance) { if (urls == null) throw new NullPointerException("urls is null"); @@ -246,8 +250,10 @@ public class DownloadMission extends Mission { int statusCode = conn.getResponseCode(); if (DEBUG) { - Log.d(TAG, threadId + ":Range=" + conn.getRequestProperty("Range")); - Log.d(TAG, threadId + ":Content-Length=" + conn.getContentLength() + " Code:" + statusCode); + Log.d(TAG, threadId + ":[request] Range=" + conn.getRequestProperty("Range")); + Log.d(TAG, threadId + ":[response] Code=" + statusCode); + Log.d(TAG, threadId + ":[response] Content-Length=" + conn.getContentLength()); + Log.d(TAG, threadId + ":[response] Content-Range=" + conn.getHeaderField("Content-Range")); } @@ -272,24 +278,19 @@ public class DownloadMission extends Mission { } synchronized void notifyProgress(long deltaLen) { - if (!running) return; - if (unknownLength) { length += deltaLen;// Update length before proceeding } done += deltaLen; - if (done > length) { - done = length; - } + if (metadata == null) return; - if (done != length && !deleted && !mWritingToFile) { - mWritingToFile = true; - runAsync(-2, this::writeThisToFile); + if (!writingToFile && (done > writingToFileNext || deltaLen < 0)) { + writingToFile = true; + writingToFileNext = done + BLOCK_SIZE; + writeThisToFileAsync(); } - - notify(DownloadManagerService.MESSAGE_PROGRESS); } synchronized void notifyError(Exception err) { @@ -342,43 +343,42 @@ public class DownloadMission extends Mission { notify(DownloadManagerService.MESSAGE_ERROR); - if (running) { - running = false; - if (threads != null) selfPause(); - } + if (running) pauseThreads(); } synchronized void notifyFinished() { - if (errCode > ERROR_NOTHING) return; - - finishCount++; - - if (blocks.length < 1 || threads == null || finishCount == threads.length) { - if (errCode != ERROR_NOTHING) return; + if (current < urls.length) { + if (++finishCount < threads.length) return; if (DEBUG) { - Log.d(TAG, "onFinish: " + (current + 1) + "/" + urls.length); - } - - if ((current + 1) < urls.length) { - // prepare next sub-mission - long current_offset = offsets[current++]; - offsets[current] = current_offset + length; - initializer(); - return; + Log.d(TAG, "onFinish: downloaded " + (current + 1) + "/" + urls.length); } current++; - unknownLength = false; - - if (!doPostprocessing()) return; - - enqueued = false; - running = false; - deleteThisFromFile(); - - notify(DownloadManagerService.MESSAGE_FINISHED); + if (current < urls.length) { + // prepare next sub-mission + offsets[current] = offsets[current - 1] + length; + initializer(); + return; + } } + + if (psAlgorithm != null && psState == 0) { + threads = new Thread[]{ + runAsync(1, this::doPostprocessing) + }; + return; + } + + + // this mission is fully finished + + unknownLength = false; + enqueued = false; + running = false; + + deleteThisFromFile(); + notify(DownloadManagerService.MESSAGE_FINISHED); } private void notifyPostProcessing(int state) { @@ -396,10 +396,15 @@ public class DownloadMission extends Mission { Log.d(TAG, action + " postprocessing on " + storage.getName()); + if (state == 2) { + psState = state; + return; + } + synchronized (LOCK) { // don't return without fully write the current state psState = state; - Utility.writeToFile(metadata, DownloadMission.this); + writeThisToFile(); } } @@ -411,12 +416,7 @@ public class DownloadMission extends Mission { if (running || isFinished() || urls.length < 1) return; // ensure that the previous state is completely paused. - int maxWait = 10000;// 10 seconds - joinForThread(init, maxWait); - if (threads != null) { - for (Thread thread : threads) joinForThread(thread, maxWait); - threads = null; - } + joinForThreads(10000); running = true; errCode = ERROR_NOTHING; @@ -427,12 +427,14 @@ public class DownloadMission extends Mission { } if (current >= urls.length) { - runAsync(1, this::notifyFinished); + notifyFinished(); return; } + notify(DownloadManagerService.MESSAGE_RUNNING); + if (urls[current] == null) { - doRecover(null); + doRecover(ERROR_RESOURCE_GONE); return; } @@ -446,18 +448,13 @@ public class DownloadMission extends Mission { blockAcquired = new boolean[blocks.length]; if (blocks.length < 1) { - if (unknownLength) { - done = 0; - length = 0; - } - threads = new Thread[]{runAsync(1, new DownloadRunnableFallback(this))}; } else { int remainingBlocks = 0; for (int block : blocks) if (block >= 0) remainingBlocks++; if (remainingBlocks < 1) { - runAsync(1, this::notifyFinished); + notifyFinished(); return; } @@ -483,6 +480,7 @@ public class DownloadMission extends Mission { } running = false; + notify(DownloadManagerService.MESSAGE_PAUSED); if (init != null && init.isAlive()) { // NOTE: if start() method is running ¡will no have effect! @@ -497,29 +495,14 @@ public class DownloadMission extends Mission { Log.w(TAG, "pausing a download that can not be resumed (range requests not allowed by the server)."); } - // check if the calling thread (alias UI thread) is interrupted - if (Thread.currentThread().isInterrupted()) { - writeThisToFile(); - return; - } - - // wait for all threads are suspended before save the state - if (threads != null) runAsync(-1, this::selfPause); + init = null; + pauseThreads(); } - private void selfPause() { - try { - for (Thread thread : threads) { - if (thread.isAlive()) { - thread.interrupt(); - thread.join(5000); - } - } - } catch (Exception e) { - // nothing to do - } finally { - writeThisToFile(); - } + private void pauseThreads() { + running = false; + joinForThreads(-1); + writeThisToFile(); } /** @@ -527,9 +510,10 @@ public class DownloadMission extends Mission { */ @Override public boolean delete() { - deleted = true; if (psAlgorithm != null) psAlgorithm.cleanupTemporalDir(); + notify(DownloadManagerService.MESSAGE_DELETED); + boolean res = deleteThisFromFile(); if (!super.delete()) return false; @@ -544,35 +528,37 @@ public class DownloadMission extends Mission { * @param persistChanges {@code true} to commit changes to the metadata file, otherwise, {@code false} */ public void resetState(boolean rollback, boolean persistChanges, int errorCode) { - done = 0; + length = 0; errCode = errorCode; errObject = null; unknownLength = false; - threads = null; + threads = new Thread[0]; fallbackResumeOffset = 0; blocks = null; blockAcquired = null; if (rollback) current = 0; - - if (persistChanges) - Utility.writeToFile(metadata, DownloadMission.this); + if (persistChanges) writeThisToFile(); } private void initializer() { init = runAsync(DownloadInitializer.mId, new DownloadInitializer(this)); } + private void writeThisToFileAsync() { + runAsync(-2, this::writeThisToFile); + } + /** * Write this {@link DownloadMission} to the meta file asynchronously * if no thread is already running. */ void writeThisToFile() { synchronized (LOCK) { - if (deleted) return; - Utility.writeToFile(metadata, DownloadMission.this); + if (metadata == null) return; + Utility.writeToFile(metadata, this); + writingToFile = false; } - mWritingToFile = false; } /** @@ -625,11 +611,10 @@ public class DownloadMission extends Mission { public long getLength() { long calculated; if (psState == 1 || psState == 3) { - calculated = length; - } else { - calculated = offsets[current < offsets.length ? current : (offsets.length - 1)] + length; + return length; } + calculated = offsets[current < offsets.length ? current : (offsets.length - 1)] + length; calculated -= offsets[0];// don't count reserved space return calculated > nearLength ? calculated : nearLength; @@ -642,7 +627,7 @@ public class DownloadMission extends Mission { */ public void setEnqueued(boolean queue) { enqueued = queue; - runAsync(-2, this::writeThisToFile); + writeThisToFileAsync(); } /** @@ -681,24 +666,19 @@ public class DownloadMission extends Mission { * @return {@code true} if the mission is running a recovery procedure, otherwise, {@code false} */ public boolean isRecovering() { - return threads != null && threads.length > 0 && threads[0] instanceof DownloadRunnable && threads[0].isAlive(); + return threads.length > 0 && threads[0] instanceof DownloadMissionRecover && threads[0].isAlive(); } - private boolean doPostprocessing() { - if (psAlgorithm == null || psState == 2) return true; - + private void doPostprocessing() { + errCode = ERROR_NOTHING; errObject = null; + Thread thread = Thread.currentThread(); notifyPostProcessing(1); - notifyProgress(0); - if (DEBUG) - Thread.currentThread().setName("[" + TAG + "] ps = " + - psAlgorithm.getClass().getSimpleName() + - " filename = " + storage.getName() - ); - - threads = new Thread[]{Thread.currentThread()}; + if (DEBUG) { + thread.setName("[" + TAG + "] ps = " + psAlgorithm + " filename = " + storage.getName()); + } Exception exception = null; @@ -707,6 +687,11 @@ public class DownloadMission extends Mission { } catch (Exception err) { Log.e(TAG, "Post-processing failed. " + psAlgorithm.toString(), err); + if (err instanceof InterruptedIOException || err instanceof ClosedByInterruptException || thread.isInterrupted()) { + notifyError(DownloadMission.ERROR_POSTPROCESSING_STOPPED, null); + return; + } + if (errCode == ERROR_NOTHING) errCode = ERROR_POSTPROCESSING; exception = err; @@ -717,56 +702,38 @@ public class DownloadMission extends Mission { if (errCode != ERROR_NOTHING) { if (exception == null) exception = errObject; notifyError(ERROR_POSTPROCESSING, exception); - - return false; + return; } - return true; + notifyFinished(); } /** * Attempts to recover the download * - * @param fromError exception which require update the url from the source + * @param errorCode error code which trigger the recovery procedure */ - void doRecover(Exception fromError) { + void doRecover(int errorCode) { Log.i(TAG, "Attempting to recover the mission: " + storage.getName()); if (recoveryInfo == null) { - if (fromError == null) - notifyError(ERROR_RESOURCE_GONE, null); - else - notifyError(fromError); - + notifyError(errorCode, null); urls = new String[0];// mark this mission as dead return; } - if (threads != null) { - for (Thread thread : threads) { - if (thread == Thread.currentThread()) continue; - thread.interrupt(); - joinForThread(thread, 0); - } - } - - errCode = ERROR_NOTHING; - errObject = null; - - if (recoveryInfo[current].attempts >= maxRetry) { - recoveryInfo[current].attempts = 0; - notifyError(fromError); - return; - } + joinForThreads(0); threads = new Thread[]{ - runAsync(DownloadMissionRecover.mID, new DownloadMissionRecover(this, fromError)) + runAsync(DownloadMissionRecover.mID, new DownloadMissionRecover(this, errorCode)) }; } private boolean deleteThisFromFile() { synchronized (LOCK) { - return metadata.delete(); + boolean res = metadata.delete(); + metadata = null; + return res; } } @@ -776,8 +743,8 @@ public class DownloadMission extends Mission { * @param id id of new thread (used for debugging only) * @param who the Runnable whose {@code run} method is invoked. */ - private void runAsync(int id, Runnable who) { - runAsync(id, new Thread(who)); + private Thread runAsync(int id, Runnable who) { + return runAsync(id, new Thread(who)); } /** @@ -806,28 +773,44 @@ public class DownloadMission extends Mission { /** * Waits at most {@code millis} milliseconds for the thread to die * - * @param thread the desired thread * @param millis the time to wait in milliseconds */ - private void joinForThread(Thread thread, int millis) { - if (thread == null || !thread.isAlive()) return; - if (thread == Thread.currentThread()) return; + private void joinForThreads(int millis) { + final Thread currentThread = Thread.currentThread(); - if (DEBUG) { - Log.w(TAG, "a thread is !still alive!: " + thread.getName()); + if (init != null && init != currentThread && init.isAlive()) { + init.interrupt(); + + if (millis > 0) { + try { + init.join(millis); + } catch (InterruptedException e) { + Log.w(TAG, "Initializer thread is still running", e); + return; + } + } } - // still alive, this should not happen. - // Possible reasons: + // if a thread is still alive, possible reasons: // slow device // the user is spamming start/pause buttons // start() method called quickly after pause() + for (Thread thread : threads) { + if (!thread.isAlive() || thread == Thread.currentThread()) continue; + thread.interrupt(); + } + try { - thread.join(millis); + for (Thread thread : threads) { + if (!thread.isAlive()) continue; + if (DEBUG) { + Log.w(TAG, "thread alive: " + thread.getName()); + } + if (millis > 0) thread.join(millis); + } } catch (InterruptedException e) { - Log.d(TAG, "timeout on join : " + thread.getName()); - throw new RuntimeException("A thread is still running:\n" + thread.getName()); + throw new RuntimeException("A download thread is still running", e); } } diff --git a/app/src/main/java/us/shandian/giga/get/DownloadMissionRecover.java b/app/src/main/java/us/shandian/giga/get/DownloadMissionRecover.java index 5efbd1153..eb660e564 100644 --- a/app/src/main/java/us/shandian/giga/get/DownloadMissionRecover.java +++ b/app/src/main/java/us/shandian/giga/get/DownloadMissionRecover.java @@ -4,6 +4,7 @@ import android.util.Log; import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.StreamingService; +import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.stream.AudioStream; import org.schabi.newpipe.extractor.stream.StreamExtractor; import org.schabi.newpipe.extractor.stream.SubtitlesStream; @@ -15,7 +16,8 @@ import java.net.HttpURLConnection; import java.nio.channels.ClosedByInterruptException; import java.util.List; -import static us.shandian.giga.get.DownloadMission.ERROR_NOTHING; +import us.shandian.giga.get.DownloadMission.HttpError; + import static us.shandian.giga.get.DownloadMission.ERROR_RESOURCE_GONE; public class DownloadMissionRecover extends Thread { @@ -23,47 +25,67 @@ public class DownloadMissionRecover extends Thread { static final int mID = -3; private final DownloadMission mMission; - private final Exception mFromError; - private final boolean notInitialized; + private final boolean mNotInitialized; + + private final int mErrCode; private HttpURLConnection mConn; private MissionRecoveryInfo mRecovery; private StreamExtractor mExtractor; - DownloadMissionRecover(DownloadMission mission, Exception originError) { + DownloadMissionRecover(DownloadMission mission, int errCode) { mMission = mission; - mFromError = originError; - notInitialized = mission.blocks == null && mission.current == 0; + mNotInitialized = mission.blocks == null && mission.current == 0; + mErrCode = errCode; } @Override public void run() { if (mMission.source == null) { - mMission.notifyError(mFromError); + mMission.notifyError(mErrCode, null); return; } + Exception err = null; + int attempt = 0; + + while (attempt++ < mMission.maxRetry) { + try { + tryRecover(); + return; + } catch (InterruptedIOException | ClosedByInterruptException e) { + return; + } catch (Exception e) { + if (!mMission.running || super.isInterrupted()) return; + err = e; + } + } + + // give up + mMission.notifyError(mErrCode, err); + } + + private void tryRecover() throws ExtractionException, IOException, HttpError { /*if (mMission.source.startsWith(MissionRecoveryInfo.DIRECT_SOURCE)) { resolve(mMission.source.substring(MissionRecoveryInfo.DIRECT_SOURCE.length())); return; }*/ - try { - StreamingService svr = NewPipe.getServiceByUrl(mMission.source); - mExtractor = svr.getStreamExtractor(mMission.source); - mExtractor.fetchPage(); - } catch (InterruptedIOException | ClosedByInterruptException e) { - return; - } catch (Exception e) { - if (!mMission.running || super.isInterrupted()) return; - mMission.notifyError(e); - return; + if (mExtractor == null) { + try { + StreamingService svr = NewPipe.getServiceByUrl(mMission.source); + mExtractor = svr.getStreamExtractor(mMission.source); + mExtractor.fetchPage(); + } catch (ExtractionException e) { + mExtractor = null; + throw e; + } } // maybe the following check is redundant if (!mMission.running || super.isInterrupted()) return; - if (!notInitialized) { + if (!mNotInitialized) { // set the current download url to null in case if the recovery // process is canceled. Next time start() method is called the // recovery will be executed, saving time @@ -87,7 +109,7 @@ public class DownloadMissionRecover extends Thread { if (!mMission.running) return; // before continue, check if the current stream was resolved - if (mMission.urls[mMission.current] == null || mMission.errCode != ERROR_NOTHING) { + if (mMission.urls[mMission.current] == null) { break; } } @@ -103,59 +125,54 @@ public class DownloadMissionRecover extends Thread { mMission.start(); } - private void resolveStream() { - if (mExtractor.getErrorMessage() != null) { - mMission.notifyError(mFromError); + private void resolveStream() throws IOException, ExtractionException, HttpError { + // FIXME: this getErrorMessage() always returns "video is unavailable" + /*if (mExtractor.getErrorMessage() != null) { + mMission.notifyError(mErrCode, new ExtractionException(mExtractor.getErrorMessage())); return; + }*/ + + String url = null; + + switch (mRecovery.kind) { + case 'a': + for (AudioStream audio : mExtractor.getAudioStreams()) { + if (audio.average_bitrate == mRecovery.desiredBitrate && audio.getFormat() == mRecovery.format) { + url = audio.getUrl(); + break; + } + } + break; + case 'v': + List videoStreams; + if (mRecovery.desired2) + videoStreams = mExtractor.getVideoOnlyStreams(); + else + videoStreams = mExtractor.getVideoStreams(); + for (VideoStream video : videoStreams) { + if (video.resolution.equals(mRecovery.desired) && video.getFormat() == mRecovery.format) { + url = video.getUrl(); + break; + } + } + break; + case 's': + for (SubtitlesStream subtitles : mExtractor.getSubtitles(mRecovery.format)) { + String tag = subtitles.getLanguageTag(); + if (tag.equals(mRecovery.desired) && subtitles.isAutoGenerated() == mRecovery.desired2) { + url = subtitles.getURL(); + break; + } + } + break; + default: + throw new RuntimeException("Unknown stream type"); } - try { - String url = null; - - switch (mRecovery.kind) { - case 'a': - for (AudioStream audio : mExtractor.getAudioStreams()) { - if (audio.average_bitrate == mRecovery.desiredBitrate && audio.getFormat() == mRecovery.format) { - url = audio.getUrl(); - break; - } - } - break; - case 'v': - List videoStreams; - if (mRecovery.desired2) - videoStreams = mExtractor.getVideoOnlyStreams(); - else - videoStreams = mExtractor.getVideoStreams(); - for (VideoStream video : videoStreams) { - if (video.resolution.equals(mRecovery.desired) && video.getFormat() == mRecovery.format) { - url = video.getUrl(); - break; - } - } - break; - case 's': - for (SubtitlesStream subtitles : mExtractor.getSubtitles(mRecovery.format)) { - String tag = subtitles.getLanguageTag(); - if (tag.equals(mRecovery.desired) && subtitles.isAutoGenerated() == mRecovery.desired2) { - url = subtitles.getURL(); - break; - } - } - break; - default: - throw new RuntimeException("Unknown stream type"); - } - - resolve(url); - } catch (Exception e) { - if (!mMission.running || e instanceof ClosedByInterruptException) return; - mRecovery.attempts++; - mMission.notifyError(e); - } + resolve(url); } - private void resolve(String url) throws IOException, DownloadMission.HttpError { + private void resolve(String url) throws IOException, HttpError { if (mRecovery.validateCondition == null) { Log.w(TAG, "validation condition not defined, the resource can be stale"); } @@ -190,10 +207,7 @@ public class DownloadMissionRecover extends Thread { return; } - throw new DownloadMission.HttpError(code); - } catch (Exception e) { - if (!mMission.running || e instanceof ClosedByInterruptException) return; - throw e; + throw new HttpError(code); } finally { disconnect(); } @@ -205,14 +219,14 @@ public class DownloadMissionRecover extends Thread { ); mMission.urls[mMission.current] = url; - mRecovery.attempts = 0; if (url == null) { + mMission.urls = new String[0]; mMission.notifyError(ERROR_RESOURCE_GONE, null); return; } - if (notInitialized) return; + if (mNotInitialized) return; if (stale) { mMission.resetState(false, false, DownloadMission.ERROR_NOTHING); diff --git a/app/src/main/java/us/shandian/giga/get/DownloadRunnable.java b/app/src/main/java/us/shandian/giga/get/DownloadRunnable.java index b0dc793bc..4aa6e912e 100644 --- a/app/src/main/java/us/shandian/giga/get/DownloadRunnable.java +++ b/app/src/main/java/us/shandian/giga/get/DownloadRunnable.java @@ -87,6 +87,7 @@ public class DownloadRunnable extends Thread { if (mConn.getResponseCode() == 416) { if (block.done > 0) { // try again from the start (of the block) + mMission.notifyProgress(-block.done); block.done = 0; retry = true; mConn.disconnect(); @@ -114,7 +115,7 @@ public class DownloadRunnable extends Thread { int len; // use always start <= end - // fixes a deadlock in DownloadRunnable because youtube is sending one byte alone after downloading 26MiB exactly + // fixes a deadlock because in some videos, youtube is sending one byte alone while (start <= end && mMission.running && (len = is.read(buf, 0, buf.length)) != -1) { f.write(buf, 0, len); start += len; @@ -135,7 +136,7 @@ public class DownloadRunnable extends Thread { if (mId == 1) { // only the first thread will execute the recovery procedure - mMission.doRecover(e); + mMission.doRecover(ERROR_HTTP_FORBIDDEN); } return; } diff --git a/app/src/main/java/us/shandian/giga/get/DownloadRunnableFallback.java b/app/src/main/java/us/shandian/giga/get/DownloadRunnableFallback.java index e64322b48..9cb40cb32 100644 --- a/app/src/main/java/us/shandian/giga/get/DownloadRunnableFallback.java +++ b/app/src/main/java/us/shandian/giga/get/DownloadRunnableFallback.java @@ -1,8 +1,9 @@ package us.shandian.giga.get; -import androidx.annotation.NonNull; import android.util.Log; +import androidx.annotation.NonNull; + import org.schabi.newpipe.streams.io.SharpStream; import java.io.IOException; @@ -47,22 +48,10 @@ public class DownloadRunnableFallback extends Thread { if (mF != null) mF.close(); } - private long loadPosition() { - synchronized (mMission.LOCK) { - return mMission.fallbackResumeOffset; - } - } - - private void savePosition(long position) { - synchronized (mMission.LOCK) { - mMission.fallbackResumeOffset = position; - } - } - @Override public void run() { boolean done; - long start = loadPosition(); + long start = mMission.fallbackResumeOffset; if (DEBUG && !mMission.unknownLength && start > 0) { Log.i(TAG, "Resuming a single-thread download at " + start); @@ -83,6 +72,7 @@ public class DownloadRunnableFallback extends Thread { // check if the download can be resumed if (mConn.getResponseCode() == 416 && start > 0) { + mMission.notifyProgress(-start); start = 0; mRetryCount--; throw new DownloadMission.HttpError(416); @@ -92,6 +82,11 @@ public class DownloadRunnableFallback extends Thread { if (!mMission.unknownLength) mMission.unknownLength = Utility.getContentLength(mConn) == -1; + if (mMission.unknownLength || mConn.getResponseCode() == 200) { + // restart amount of bytes downloaded + mMission.done = mMission.offsets[mMission.current] - mMission.offsets[0]; + } + mF = mMission.storage.getStream(); mF.seek(mMission.offsets[mMission.current] + start); @@ -113,14 +108,14 @@ public class DownloadRunnableFallback extends Thread { } catch (Exception e) { dispose(); - savePosition(start); + mMission.fallbackResumeOffset = start; if (!mMission.running || e instanceof ClosedByInterruptException) return; if (e instanceof HttpError && ((HttpError) e).statusCode == ERROR_HTTP_FORBIDDEN) { // for youtube streams. The url has expired, recover dispose(); - mMission.doRecover(e); + mMission.doRecover(ERROR_HTTP_FORBIDDEN); return; } @@ -140,7 +135,7 @@ public class DownloadRunnableFallback extends Thread { if (done) { mMission.notifyFinished(); } else { - savePosition(start); + mMission.fallbackResumeOffset = start; } } diff --git a/app/src/main/java/us/shandian/giga/get/FinishedMission.java b/app/src/main/java/us/shandian/giga/get/FinishedMission.java index b468f3c76..6bc5423b8 100644 --- a/app/src/main/java/us/shandian/giga/get/FinishedMission.java +++ b/app/src/main/java/us/shandian/giga/get/FinishedMission.java @@ -2,17 +2,17 @@ package us.shandian.giga.get; import androidx.annotation.NonNull; -public class FinishedMission extends Mission { +public class FinishedMission extends Mission { public FinishedMission() { } public FinishedMission(@NonNull DownloadMission mission) { source = mission.source; - length = mission.length;// ¿or mission.done? + length = mission.length; timestamp = mission.timestamp; kind = mission.kind; storage = mission.storage; - } + } diff --git a/app/src/main/java/us/shandian/giga/get/MissionRecoveryInfo.java b/app/src/main/java/us/shandian/giga/get/MissionRecoveryInfo.java index bd1d9bc49..f6a3a3984 100644 --- a/app/src/main/java/us/shandian/giga/get/MissionRecoveryInfo.java +++ b/app/src/main/java/us/shandian/giga/get/MissionRecoveryInfo.java @@ -2,7 +2,8 @@ package us.shandian.giga.get; import android.os.Parcel; import android.os.Parcelable; -import android.support.annotation.NonNull; + +import androidx.annotation.NonNull; import org.schabi.newpipe.extractor.MediaFormat; import org.schabi.newpipe.extractor.stream.AudioStream; @@ -23,8 +24,6 @@ public class MissionRecoveryInfo implements Serializable, Parcelable { byte kind; String validateCondition = null; - transient int attempts = 0; - public MissionRecoveryInfo(@NonNull Stream stream) { if (stream instanceof AudioStream) { desiredBitrate = ((AudioStream) stream).average_bitrate; @@ -51,7 +50,7 @@ public class MissionRecoveryInfo implements Serializable, Parcelable { public String toString() { String info; StringBuilder str = new StringBuilder(); - str.append("type="); + str.append("{type="); switch (kind) { case 'a': str.append("audio"); @@ -73,7 +72,8 @@ public class MissionRecoveryInfo implements Serializable, Parcelable { str.append(" format=") .append(format.getName()) .append(' ') - .append(info); + .append(info) + .append('}'); return str.toString(); } diff --git a/app/src/main/java/us/shandian/giga/io/ChunkFileInputStream.java b/app/src/main/java/us/shandian/giga/io/ChunkFileInputStream.java index 16a90fcee..98015e37e 100644 --- a/app/src/main/java/us/shandian/giga/io/ChunkFileInputStream.java +++ b/app/src/main/java/us/shandian/giga/io/ChunkFileInputStream.java @@ -5,21 +5,23 @@ import org.schabi.newpipe.streams.io.SharpStream; import java.io.IOException; public class ChunkFileInputStream extends SharpStream { + private static final int REPORT_INTERVAL = 256 * 1024; private SharpStream source; private final long offset; private final long length; private long position; - public ChunkFileInputStream(SharpStream target, long start) throws IOException { - this(target, start, target.length()); - } + private long progressReport; + private final ProgressReport onProgress; - public ChunkFileInputStream(SharpStream target, long start, long end) throws IOException { + public ChunkFileInputStream(SharpStream target, long start, long end, ProgressReport callback) throws IOException { source = target; offset = start; length = end - start; position = 0; + onProgress = callback; + progressReport = REPORT_INTERVAL; if (length < 1) { source.close(); @@ -60,12 +62,12 @@ public class ChunkFileInputStream extends SharpStream { } @Override - public int read(byte b[]) throws IOException { + public int read(byte[] b) throws IOException { return read(b, 0, b.length); } @Override - public int read(byte b[], int off, int len) throws IOException { + public int read(byte[] b, int off, int len) throws IOException { if ((position + len) > length) { len = (int) (length - position); } @@ -76,6 +78,11 @@ public class ChunkFileInputStream extends SharpStream { int res = source.read(b, off, len); position += res; + if (onProgress != null && position > progressReport) { + onProgress.report(position); + progressReport = position + REPORT_INTERVAL; + } + return res; } diff --git a/app/src/main/java/us/shandian/giga/io/CircularFileWriter.java b/app/src/main/java/us/shandian/giga/io/CircularFileWriter.java index e2afb9202..102580570 100644 --- a/app/src/main/java/us/shandian/giga/io/CircularFileWriter.java +++ b/app/src/main/java/us/shandian/giga/io/CircularFileWriter.java @@ -174,12 +174,12 @@ public class CircularFileWriter extends SharpStream { } @Override - public void write(byte b[]) throws IOException { + public void write(byte[] b) throws IOException { write(b, 0, b.length); } @Override - public void write(byte b[], int off, int len) throws IOException { + public void write(byte[] b, int off, int len) throws IOException { if (len == 0) { return; } @@ -261,7 +261,7 @@ public class CircularFileWriter extends SharpStream { @Override public void rewind() throws IOException { if (onProgress != null) { - onProgress.report(-out.length - aux.length);// rollback the whole progress + onProgress.report(0);// rollback the whole progress } seek(0); @@ -357,16 +357,6 @@ public class CircularFileWriter extends SharpStream { long check(); } - public interface ProgressReport { - - /** - * Report the size of the new file - * - * @param progress the new size - */ - void report(long progress); - } - public interface WriteErrorHandle { /** @@ -381,10 +371,10 @@ public class CircularFileWriter extends SharpStream { class BufferedFile { - protected final SharpStream target; + final SharpStream target; private long offset; - protected long length; + long length; private byte[] queue = new byte[QUEUE_BUFFER_SIZE]; private int queueSize; @@ -397,16 +387,16 @@ public class CircularFileWriter extends SharpStream { this.target = target; } - protected long getOffset() { + long getOffset() { return offset + queueSize;// absolute offset in the file } - protected void close() { + void close() { queue = null; target.close(); } - protected void write(byte b[], int off, int len) throws IOException { + void write(byte[] b, int off, int len) throws IOException { while (len > 0) { // if the queue is full, the method available() will flush the queue int read = Math.min(available(), len); @@ -436,7 +426,7 @@ public class CircularFileWriter extends SharpStream { target.seek(0); } - protected int available() throws IOException { + int available() throws IOException { if (queueSize >= queue.length) { flush(); return queue.length; @@ -451,7 +441,7 @@ public class CircularFileWriter extends SharpStream { target.seek(0); } - protected void seek(long absoluteOffset) throws IOException { + void seek(long absoluteOffset) throws IOException { if (absoluteOffset == offset) { return;// nothing to do } diff --git a/app/src/main/java/us/shandian/giga/io/ProgressReport.java b/app/src/main/java/us/shandian/giga/io/ProgressReport.java new file mode 100644 index 000000000..14ae9ded9 --- /dev/null +++ b/app/src/main/java/us/shandian/giga/io/ProgressReport.java @@ -0,0 +1,11 @@ +package us.shandian.giga.io; + +public interface ProgressReport { + + /** + * Report the size of the new file + * + * @param progress the new size + */ + void report(long progress); +} \ No newline at end of file diff --git a/app/src/main/java/us/shandian/giga/postprocessing/OggFromWebmDemuxer.java b/app/src/main/java/us/shandian/giga/postprocessing/OggFromWebmDemuxer.java index 605c0a88b..04958c495 100644 --- a/app/src/main/java/us/shandian/giga/postprocessing/OggFromWebmDemuxer.java +++ b/app/src/main/java/us/shandian/giga/postprocessing/OggFromWebmDemuxer.java @@ -1,6 +1,6 @@ package us.shandian.giga.postprocessing; -import android.support.annotation.NonNull; +import androidx.annotation.NonNull; import org.schabi.newpipe.streams.OggFromWebMWriter; import org.schabi.newpipe.streams.io.SharpStream; diff --git a/app/src/main/java/us/shandian/giga/postprocessing/Postprocessing.java b/app/src/main/java/us/shandian/giga/postprocessing/Postprocessing.java index 92510c3df..773ff92d1 100644 --- a/app/src/main/java/us/shandian/giga/postprocessing/Postprocessing.java +++ b/app/src/main/java/us/shandian/giga/postprocessing/Postprocessing.java @@ -1,9 +1,9 @@ package us.shandian.giga.postprocessing; -import android.os.Message; -import androidx.annotation.NonNull; import android.util.Log; +import androidx.annotation.NonNull; + import org.schabi.newpipe.streams.io.SharpStream; import java.io.File; @@ -14,11 +14,11 @@ import us.shandian.giga.get.DownloadMission; import us.shandian.giga.io.ChunkFileInputStream; import us.shandian.giga.io.CircularFileWriter; import us.shandian.giga.io.CircularFileWriter.OffsetChecker; -import us.shandian.giga.service.DownloadManagerService; +import us.shandian.giga.io.ProgressReport; import static us.shandian.giga.get.DownloadMission.ERROR_NOTHING; +import static us.shandian.giga.get.DownloadMission.ERROR_POSTPROCESSING; import static us.shandian.giga.get.DownloadMission.ERROR_POSTPROCESSING_HOLD; -import static us.shandian.giga.get.DownloadMission.ERROR_UNKNOWN_EXCEPTION; public abstract class Postprocessing implements Serializable { @@ -63,22 +63,22 @@ public abstract class Postprocessing implements Serializable { * Get a boolean value that indicate if the given algorithm work on the same * file */ - public final boolean worksOnSameFile; + public boolean worksOnSameFile; /** * Indicates whether the selected algorithm needs space reserved at the beginning of the file */ - public final boolean reserveSpace; + public boolean reserveSpace; /** * Gets the given algorithm short name */ - private final String name; + private String name; private String[] args; - protected transient DownloadMission mission; + private transient DownloadMission mission; private File tempFile; @@ -109,16 +109,24 @@ public abstract class Postprocessing implements Serializable { long finalLength = -1; mission.done = 0; - mission.length = mission.storage.length(); + + long length = mission.storage.length() - mission.offsets[0]; + mission.length = length > mission.nearLength ? length : mission.nearLength; + + final ProgressReport readProgress = (long position) -> { + position -= mission.offsets[0]; + if (position > mission.done) mission.done = position; + }; if (worksOnSameFile) { ChunkFileInputStream[] sources = new ChunkFileInputStream[mission.urls.length]; try { - int i = 0; - for (; i < sources.length - 1; i++) { - sources[i] = new ChunkFileInputStream(mission.storage.getStream(), mission.offsets[i], mission.offsets[i + 1]); + for (int i = 0, j = 1; i < sources.length; i++, j++) { + SharpStream source = mission.storage.getStream(); + long end = j < sources.length ? mission.offsets[j] : source.length(); + + sources[i] = new ChunkFileInputStream(source, mission.offsets[i], end, readProgress); } - sources[i] = new ChunkFileInputStream(mission.storage.getStream(), mission.offsets[i]); if (test(sources)) { for (SharpStream source : sources) source.rewind(); @@ -140,7 +148,7 @@ public abstract class Postprocessing implements Serializable { }; out = new CircularFileWriter(mission.storage.getStream(), tempFile, checker); - out.onProgress = this::progressReport; + out.onProgress = (long position) -> mission.done = position; out.onWriteError = (err) -> { mission.psState = 3; @@ -187,11 +195,10 @@ public abstract class Postprocessing implements Serializable { if (result == OK_RESULT) { if (finalLength != -1) { - mission.done = finalLength; mission.length = finalLength; } } else { - mission.errCode = ERROR_UNKNOWN_EXCEPTION; + mission.errCode = ERROR_POSTPROCESSING; mission.errObject = new RuntimeException("post-processing algorithm returned " + result); } @@ -229,23 +236,12 @@ public abstract class Postprocessing implements Serializable { return args[index]; } - private void progressReport(long done) { - mission.done = done; - if (mission.length < mission.done) mission.length = mission.done; - - Message m = new Message(); - m.what = DownloadManagerService.MESSAGE_PROGRESS; - m.obj = mission; - - mission.mHandler.sendMessage(m); - } - @NonNull @Override public String toString() { StringBuilder str = new StringBuilder(); - str.append("name=").append(name).append('['); + str.append("{ name=").append(name).append('['); if (args != null) { for (String arg : args) { @@ -255,6 +251,6 @@ public abstract class Postprocessing implements Serializable { str.delete(0, 1); } - return str.append(']').toString(); + return str.append("] }").toString(); } } diff --git a/app/src/main/java/us/shandian/giga/service/DownloadManager.java b/app/src/main/java/us/shandian/giga/service/DownloadManager.java index 2d1e9cd00..e8bc468e9 100644 --- a/app/src/main/java/us/shandian/giga/service/DownloadManager.java +++ b/app/src/main/java/us/shandian/giga/service/DownloadManager.java @@ -2,13 +2,11 @@ package us.shandian.giga.service; import android.content.Context; import android.os.Handler; +import android.util.Log; + import androidx.annotation.NonNull; import androidx.annotation.Nullable; import androidx.recyclerview.widget.DiffUtil; -import android.util.Log; -import android.widget.Toast; - -import org.schabi.newpipe.R; import java.io.File; import java.io.IOException; @@ -152,6 +150,8 @@ public class DownloadManager { continue; } + mis.threads = new Thread[0]; + boolean exists; try { mis.storage = StoredFileHelper.deserialize(mis.storage, ctx); @@ -170,8 +170,6 @@ public class DownloadManager { // is Java IO (avoid showing the "Save as..." dialog) if (exists && mis.storage.isDirect() && !mis.storage.delete()) Log.w(TAG, "Unable to delete incomplete download file: " + sub.getPath()); - - exists = true; } mis.psState = 0; @@ -243,7 +241,6 @@ public class DownloadManager { boolean start = !mPrefQueueLimit || getRunningMissionsCount() < 1; if (canDownloadInCurrentNetwork() && start) { - mHandler.sendEmptyMessage(DownloadManagerService.MESSAGE_PROGRESS); mission.start(); } } @@ -252,7 +249,6 @@ public class DownloadManager { public void resumeMission(DownloadMission mission) { if (!mission.running) { - mHandler.sendEmptyMessage(DownloadManagerService.MESSAGE_PROGRESS); mission.start(); } } @@ -261,7 +257,6 @@ public class DownloadManager { if (mission.running) { mission.setEnqueued(false); mission.pause(); - mHandler.sendEmptyMessage(DownloadManagerService.MESSAGE_PAUSED); } } @@ -274,7 +269,6 @@ public class DownloadManager { mFinishedMissionStore.deleteMission(mission); } - mHandler.sendEmptyMessage(DownloadManagerService.MESSAGE_DELETED); mission.delete(); } } @@ -291,7 +285,6 @@ public class DownloadManager { mFinishedMissionStore.deleteMission(mission); } - mHandler.sendEmptyMessage(DownloadManagerService.MESSAGE_DELETED); mission.storage = null; mission.delete(); } @@ -374,35 +367,29 @@ public class DownloadManager { } public void pauseAllMissions(boolean force) { - boolean flag = false; - synchronized (this) { for (DownloadMission mission : mMissionsPending) { if (!mission.running || mission.isPsRunning() || mission.isFinished()) continue; - if (force) mission.threads = null;// avoid waiting for threads + if (force) { + // avoid waiting for threads + mission.init = null; + mission.threads = new Thread[0]; + } mission.pause(); - flag = true; } } - - if (flag) mHandler.sendEmptyMessage(DownloadManagerService.MESSAGE_PAUSED); } public void startAllMissions() { - boolean flag = false; - synchronized (this) { for (DownloadMission mission : mMissionsPending) { if (mission.running || mission.isCorrupt()) continue; - flag = true; mission.start(); } } - - if (flag) mHandler.sendEmptyMessage(DownloadManagerService.MESSAGE_PROGRESS); } /** @@ -483,28 +470,18 @@ public class DownloadManager { boolean isMetered = mPrefMeteredDownloads && mLastNetworkStatus == NetworkState.MeteredOperating; - int running = 0; - int paused = 0; synchronized (this) { for (DownloadMission mission : mMissionsPending) { if (mission.isCorrupt() || mission.isPsRunning()) continue; if (mission.running && isMetered) { - paused++; mission.pause(); } else if (!mission.running && !isMetered && mission.enqueued) { - running++; mission.start(); if (mPrefQueueLimit) break; } } } - - if (running > 0) { - mHandler.sendEmptyMessage(DownloadManagerService.MESSAGE_PROGRESS); - return; - } - if (paused > 0) mHandler.sendEmptyMessage(DownloadManagerService.MESSAGE_PAUSED); } void updateMaximumAttempts() { @@ -513,22 +490,6 @@ public class DownloadManager { } } - /** - * Fast check for pending downloads. If exists, the user will be notified - * TODO: call this method in somewhere - * - * @param context the application context - */ - public static void notifyUserPendingDownloads(Context context) { - int pending = getPendingDir(context).list().length; - if (pending < 1) return; - - Toast.makeText(context, context.getString( - R.string.msg_pending_downloads, - String.valueOf(pending) - ), Toast.LENGTH_LONG).show(); - } - public MissionState checkForExistingMission(StoredFileHelper storage) { synchronized (this) { DownloadMission pending = getPendingMission(storage); diff --git a/app/src/main/java/us/shandian/giga/service/DownloadManagerService.java b/app/src/main/java/us/shandian/giga/service/DownloadManagerService.java index ea9029c0b..3da0e75b8 100755 --- a/app/src/main/java/us/shandian/giga/service/DownloadManagerService.java +++ b/app/src/main/java/us/shandian/giga/service/DownloadManagerService.java @@ -25,14 +25,15 @@ import android.os.IBinder; import android.os.Message; import android.os.Parcelable; import android.preference.PreferenceManager; +import android.util.Log; +import android.util.SparseArray; +import android.widget.Toast; + import androidx.annotation.NonNull; import androidx.annotation.Nullable; import androidx.annotation.StringRes; import androidx.core.app.NotificationCompat; import androidx.core.app.NotificationCompat.Builder; -import android.util.Log; -import android.util.SparseArray; -import android.widget.Toast; import org.schabi.newpipe.R; import org.schabi.newpipe.download.DownloadActivity; @@ -41,8 +42,6 @@ import org.schabi.newpipe.player.helper.LockManager; import java.io.File; import java.io.IOException; import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; import us.shandian.giga.get.DownloadMission; import us.shandian.giga.get.MissionRecoveryInfo; @@ -58,11 +57,11 @@ public class DownloadManagerService extends Service { private static final String TAG = "DownloadManagerService"; + public static final int MESSAGE_RUNNING = 0; public static final int MESSAGE_PAUSED = 1; public static final int MESSAGE_FINISHED = 2; - public static final int MESSAGE_PROGRESS = 3; - public static final int MESSAGE_ERROR = 4; - public static final int MESSAGE_DELETED = 5; + public static final int MESSAGE_ERROR = 3; + public static final int MESSAGE_DELETED = 4; private static final int FOREGROUND_NOTIFICATION_ID = 1000; private static final int DOWNLOADS_NOTIFICATION_ID = 1001; @@ -217,9 +216,11 @@ public class DownloadManagerService extends Service { .addFlags(Intent.FLAG_ACTIVITY_NEW_TASK) ); } + return START_NOT_STICKY; } } - return START_NOT_STICKY; + + return START_STICKY; } @Override @@ -250,6 +251,7 @@ public class DownloadManagerService extends Service { if (icDownloadFailed != null) icDownloadFailed.recycle(); if (icLauncher != null) icLauncher.recycle(); + mHandler = null; mManager.pauseAllMissions(true); } @@ -274,6 +276,8 @@ public class DownloadManagerService extends Service { } private boolean handleMessage(@NonNull Message msg) { + if (mHandler == null) return true; + DownloadMission mission = (DownloadMission) msg.obj; switch (msg.what) { @@ -284,7 +288,7 @@ public class DownloadManagerService extends Service { handleConnectivityState(false); updateForegroundState(mManager.runMissions()); break; - case MESSAGE_PROGRESS: + case MESSAGE_RUNNING: updateForegroundState(true); break; case MESSAGE_ERROR: @@ -300,11 +304,8 @@ public class DownloadManagerService extends Service { if (msg.what != MESSAGE_ERROR) mFailedDownloads.delete(mFailedDownloads.indexOfValue(mission)); - synchronized (mEchoObservers) { - for (Callback observer : mEchoObservers) { - observer.handleMessage(msg); - } - } + for (Callback observer : mEchoObservers) + observer.handleMessage(msg); return true; } @@ -523,16 +524,6 @@ public class DownloadManagerService extends Service { return PendingIntent.getService(this, intent.hashCode(), intent, PendingIntent.FLAG_UPDATE_CURRENT); } - private void manageObservers(Callback handler, boolean add) { - synchronized (mEchoObservers) { - if (add) { - mEchoObservers.add(handler); - } else { - mEchoObservers.remove(handler); - } - } - } - private void manageLock(boolean acquire) { if (acquire == mLockAcquired) return; @@ -605,11 +596,11 @@ public class DownloadManagerService extends Service { } public void addMissionEventListener(Callback handler) { - manageObservers(handler, true); + mEchoObservers.add(handler); } public void removeMissionEventListener(Callback handler) { - manageObservers(handler, false); + mEchoObservers.remove(handler); } public void clearDownloadNotifications() { diff --git a/app/src/main/java/us/shandian/giga/ui/adapter/MissionAdapter.java b/app/src/main/java/us/shandian/giga/ui/adapter/MissionAdapter.java index 78fd7ea9d..e3a7f112a 100644 --- a/app/src/main/java/us/shandian/giga/ui/adapter/MissionAdapter.java +++ b/app/src/main/java/us/shandian/giga/ui/adapter/MissionAdapter.java @@ -10,16 +10,6 @@ import android.os.AsyncTask; import android.os.Build; import android.os.Handler; import android.os.Message; -import androidx.annotation.NonNull; -import androidx.annotation.Nullable; -import androidx.annotation.StringRes; -import androidx.core.content.FileProvider; -import androidx.core.view.ViewCompat; -import androidx.appcompat.app.AlertDialog; -import androidx.recyclerview.widget.DiffUtil; -import androidx.recyclerview.widget.RecyclerView; -import androidx.recyclerview.widget.RecyclerView.Adapter; -import androidx.recyclerview.widget.RecyclerView.ViewHolder; import android.util.Log; import android.util.SparseArray; import android.view.HapticFeedbackConstants; @@ -34,6 +24,17 @@ import android.widget.PopupMenu; import android.widget.TextView; import android.widget.Toast; +import androidx.annotation.NonNull; +import androidx.annotation.Nullable; +import androidx.annotation.StringRes; +import androidx.appcompat.app.AlertDialog; +import androidx.core.content.FileProvider; +import androidx.core.view.ViewCompat; +import androidx.recyclerview.widget.DiffUtil; +import androidx.recyclerview.widget.RecyclerView; +import androidx.recyclerview.widget.RecyclerView.Adapter; +import androidx.recyclerview.widget.RecyclerView.ViewHolder; + import org.schabi.newpipe.BuildConfig; import org.schabi.newpipe.R; import org.schabi.newpipe.extractor.NewPipe; @@ -82,6 +83,7 @@ public class MissionAdapter extends Adapter implements Handler.Callb private static final String TAG = "MissionAdapter"; private static final String UNDEFINED_PROGRESS = "--.-%"; private static final String DEFAULT_MIME_TYPE = "*/*"; + private static final String UNDEFINED_ETA = "--:--"; static { @@ -103,10 +105,11 @@ public class MissionAdapter extends Adapter implements Handler.Callb private View mEmptyMessage; private RecoverHelper mRecover; - public MissionAdapter(Context context, @NonNull DownloadManager downloadManager, View emptyMessage) { + private final Runnable rUpdater = this::updater; + + public MissionAdapter(Context context, @NonNull DownloadManager downloadManager, View emptyMessage, View root) { mContext = context; mDownloadManager = downloadManager; - mDeleter = null; mInflater = (LayoutInflater) mContext.getSystemService(Context.LAYOUT_INFLATER_SERVICE); mLayout = R.layout.mission_item; @@ -117,7 +120,10 @@ public class MissionAdapter extends Adapter implements Handler.Callb mIterator = downloadManager.getIterator(); + mDeleter = new Deleter(root, mContext, this, mDownloadManager, mIterator, mHandler); + checkEmptyMessageVisibility(); + onResume(); } @Override @@ -142,17 +148,13 @@ public class MissionAdapter extends Adapter implements Handler.Callb if (h.item.mission instanceof DownloadMission) { mPendingDownloadsItems.remove(h); if (mPendingDownloadsItems.size() < 1) { - setAutoRefresh(false); checkMasterButtonsVisibility(); } } h.popupMenu.dismiss(); h.item = null; - h.lastTimeStamp = -1; - h.lastDone = -1; - h.lastCurrent = -1; - h.state = 0; + h.resetSpeedMeasure(); } @Override @@ -191,7 +193,6 @@ public class MissionAdapter extends Adapter implements Handler.Callb h.size.setText(length); h.pause.setTitle(mission.unknownLength ? R.string.stop : R.string.pause); - h.lastCurrent = mission.current; updateProgress(h); mPendingDownloadsItems.add(h); } else { @@ -216,20 +217,10 @@ public class MissionAdapter extends Adapter implements Handler.Callb private void updateProgress(ViewHolderItem h) { if (h == null || h.item == null || h.item.mission instanceof FinishedMission) return; - long now = System.currentTimeMillis(); DownloadMission mission = (DownloadMission) h.item.mission; - - if (h.lastCurrent != mission.current) { - h.lastCurrent = mission.current; - h.lastTimeStamp = now; - h.lastDone = 0; - } else { - if (h.lastTimeStamp == -1) h.lastTimeStamp = now; - if (h.lastDone == -1) h.lastDone = mission.done; - } - - long deltaTime = now - h.lastTimeStamp; - long deltaDone = mission.done - h.lastDone; + double done = mission.done; + long length = mission.getLength(); + long now = System.currentTimeMillis(); boolean hasError = mission.errCode != ERROR_NOTHING; // hide on error @@ -237,19 +228,16 @@ public class MissionAdapter extends Adapter implements Handler.Callb // show if length is unknown h.progress.setMarquee(mission.isRecovering() || !hasError && (!mission.isInitialized() || mission.unknownLength)); - float progress; + double progress; if (mission.unknownLength) { - progress = Float.NaN; + progress = Double.NaN; h.progress.setProgress(0f); } else { - progress = (float) ((double) mission.done / mission.length); - if (mission.urls.length > 1 && mission.current < mission.urls.length) { - progress = (progress / mission.urls.length) + ((float) mission.current / mission.urls.length); - } + progress = done / length; } if (hasError) { - h.progress.setProgress(isNotFinite(progress) ? 1f : progress); + h.progress.setProgress(isNotFinite(progress) ? 1d : progress); h.status.setText(R.string.msg_error); } else if (isNotFinite(progress)) { h.status.setText(UNDEFINED_PROGRESS); @@ -258,59 +246,78 @@ public class MissionAdapter extends Adapter implements Handler.Callb h.progress.setProgress(progress); } - long length = mission.getLength(); + @StringRes int state; + String sizeStr = Utility.formatBytes(length).concat(" "); - int state; if (mission.isPsFailed() || mission.errCode == ERROR_POSTPROCESSING_HOLD) { - state = 0; + h.size.setText(sizeStr); + return; } else if (!mission.running) { - state = mission.enqueued ? 1 : 2; + state = mission.enqueued ? R.string.queued : R.string.paused; } else if (mission.isPsRunning()) { - state = 3; + state = R.string.post_processing; + } else if (mission.isRecovering()) { + state = R.string.recovering; } else { state = 0; } if (state != 0) { // update state without download speed - if (h.state != state) { - String statusStr; - h.state = state; + h.size.setText(sizeStr.concat("(").concat(mContext.getString(state)).concat(")")); + h.resetSpeedMeasure(); + return; + } - switch (state) { - case 1: - statusStr = mContext.getString(R.string.queued); - break; - case 2: - statusStr = mContext.getString(R.string.paused); - break; - case 3: - statusStr = mContext.getString(R.string.post_processing); - break; - default: - statusStr = "?"; - break; - } + if (h.lastTimestamp < 0) { + h.size.setText(sizeStr); + h.lastTimestamp = now; + h.lastDone = done; + return; + } - h.size.setText(Utility.formatBytes(length).concat(" (").concat(statusStr).concat(")")); - } else if (deltaDone > 0) { - h.lastTimeStamp = now; - h.lastDone = mission.done; - } + long deltaTime = now - h.lastTimestamp; + double deltaDone = done - h.lastDone; + if (h.lastDone > done) { + h.lastDone = done; + h.size.setText(sizeStr); return; } if (deltaDone > 0 && deltaTime > 0) { - float speed = (deltaDone * 1000f) / deltaTime; + float speed = (float) ((deltaDone * 1000d) / deltaTime); + float averageSpeed = speed; - String speedStr = Utility.formatSpeed(speed); - String sizeStr = Utility.formatBytes(length); + if (h.lastSpeedIdx < 0) { + for (int i = 0; i < h.lastSpeed.length; i++) { + h.lastSpeed[i] = speed; + } + h.lastSpeedIdx = 0; + } else { + for (int i = 0; i < h.lastSpeed.length; i++) { + averageSpeed += h.lastSpeed[i]; + } + averageSpeed /= h.lastSpeed.length + 1f; + } - h.size.setText(sizeStr.concat(" ").concat(speedStr)); + String speedStr = Utility.formatSpeed(averageSpeed); + String etaStr; - h.lastTimeStamp = now; - h.lastDone = mission.done; + if (mission.unknownLength) { + etaStr = ""; + } else { + long eta = (long) Math.ceil((length - done) / averageSpeed); + etaStr = " @ ".concat(Utility.stringifySeconds(eta)); + } + + h.size.setText(sizeStr.concat(speedStr).concat(etaStr)); + + h.lastTimestamp = now; + h.lastDone = done; + h.lastSpeed[h.lastSpeedIdx++] = speed; + + if (h.lastSpeedIdx >= h.lastSpeed.length) h.lastSpeedIdx = 0; } } @@ -389,6 +396,13 @@ public class MissionAdapter extends Adapter implements Handler.Callb return true; } + private ViewHolderItem getViewHolder(Object mission) { + for (ViewHolderItem h : mPendingDownloadsItems) { + if (h.item.mission == mission) return h; + } + return null; + } + @Override public boolean handleMessage(@NonNull Message msg) { if (mStartButton != null && mPauseButton != null) { @@ -396,33 +410,28 @@ public class MissionAdapter extends Adapter implements Handler.Callb } switch (msg.what) { - case DownloadManagerService.MESSAGE_PROGRESS: case DownloadManagerService.MESSAGE_ERROR: case DownloadManagerService.MESSAGE_FINISHED: + case DownloadManagerService.MESSAGE_DELETED: + case DownloadManagerService.MESSAGE_PAUSED: break; default: return false; } - if (msg.what == DownloadManagerService.MESSAGE_PROGRESS) { - setAutoRefresh(true); - return true; - } + ViewHolderItem h = getViewHolder(msg.obj); + if (h == null) return false; - for (ViewHolderItem h : mPendingDownloadsItems) { - if (h.item.mission != msg.obj) continue; - - if (msg.what == DownloadManagerService.MESSAGE_FINISHED) { + switch (msg.what) { + case DownloadManagerService.MESSAGE_FINISHED: + case DownloadManagerService.MESSAGE_DELETED: // DownloadManager should mark the download as finished applyChanges(); return true; - } - - updateProgress(h); - return true; } - return false; + updateProgress(h); + return true; } private void showError(@NonNull DownloadMission mission) { @@ -470,8 +479,13 @@ public class MissionAdapter extends Adapter implements Handler.Callb msg = R.string.error_insufficient_storage; break; case ERROR_UNKNOWN_EXCEPTION: - showError(mission, UserAction.DOWNLOAD_FAILED, R.string.general_error); - return; + if (mission.errObject != null) { + showError(mission, UserAction.DOWNLOAD_FAILED, R.string.general_error); + return; + } else { + msg = R.string.msg_error; + break; + } case ERROR_PROGRESS_LOST: msg = R.string.error_progress_lost; break; @@ -521,7 +535,9 @@ public class MissionAdapter extends Adapter implements Handler.Callb request.append(" ["); if (mission.recoveryInfo != null) { for (MissionRecoveryInfo recovery : mission.recoveryInfo) - request.append(" {").append(recovery.toString()).append("} "); + request.append(' ') + .append(recovery.toString()) + .append(' '); } request.append("]"); @@ -556,16 +572,10 @@ public class MissionAdapter extends Adapter implements Handler.Callb switch (id) { case R.id.start: h.status.setText(UNDEFINED_PROGRESS); - h.state = -1; - h.size.setText(Utility.formatBytes(mission.getLength())); mDownloadManager.resumeMission(mission); return true; case R.id.pause: - h.state = -1; mDownloadManager.pauseMission(mission); - updateProgress(h); - h.lastTimeStamp = -1; - h.lastDone = -1; return true; case R.id.error_message_view: showError(mission); @@ -598,12 +608,9 @@ public class MissionAdapter extends Adapter implements Handler.Callb shareFile(h.item.mission); return true; case R.id.delete: - if (mDeleter == null) { - mDownloadManager.deleteMission(h.item.mission); - } else { - mDeleter.append(h.item.mission); - } + mDeleter.append(h.item.mission); applyChanges(); + checkMasterButtonsVisibility(); return true; case R.id.md5: case R.id.sha1: @@ -639,7 +646,7 @@ public class MissionAdapter extends Adapter implements Handler.Callb mIterator.end(); for (ViewHolderItem item : mPendingDownloadsItems) { - item.lastTimeStamp = -1; + item.resetSpeedMeasure(); } notifyDataSetChanged(); @@ -672,6 +679,7 @@ public class MissionAdapter extends Adapter implements Handler.Callb public void checkMasterButtonsVisibility() { boolean[] state = mIterator.hasValidPendingMissions(); + Log.d(TAG, "checkMasterButtonsVisibility() running=" + state[0] + " paused=" + state[1]); setButtonVisible(mPauseButton, state[0]); setButtonVisible(mStartButton, state[1]); } @@ -681,86 +689,57 @@ public class MissionAdapter extends Adapter implements Handler.Callb button.setVisible(visible); } - public void ensurePausedMissions() { + public void refreshMissionItems() { for (ViewHolderItem h : mPendingDownloadsItems) { if (((DownloadMission) h.item.mission).running) continue; updateProgress(h); - h.lastTimeStamp = -1; - h.lastDone = -1; + h.resetSpeedMeasure(); } } - public void deleterDispose(boolean commitChanges) { - if (mDeleter != null) mDeleter.dispose(commitChanges); + public void onDestroy() { + mDeleter.dispose(); } - public void deleterLoad(View view) { - if (mDeleter == null) - mDeleter = new Deleter(view, mContext, this, mDownloadManager, mIterator, mHandler); + public void onResume() { + mDeleter.resume(); + mHandler.post(rUpdater); } - public void deleterResume() { - if (mDeleter != null) mDeleter.resume(); - } - - public void recoverMission(DownloadMission mission) { - for (ViewHolderItem h : mPendingDownloadsItems) { - if (mission != h.item.mission) continue; - - mission.errObject = null; - mission.resetState(true, false, DownloadMission.ERROR_NOTHING); - - h.status.setText(UNDEFINED_PROGRESS); - h.state = -1; - h.size.setText(Utility.formatBytes(mission.getLength())); - h.progress.setMarquee(true); - - mDownloadManager.resumeMission(mission); - return; - } - - } - - - private boolean mUpdaterRunning = false; - private final Runnable rUpdater = this::updater; - public void onPaused() { - setAutoRefresh(false); + mDeleter.pause(); + mHandler.removeCallbacks(rUpdater); } - private void setAutoRefresh(boolean enabled) { - if (enabled && !mUpdaterRunning) { - mUpdaterRunning = true; - updater(); - } else if (!enabled && mUpdaterRunning) { - mUpdaterRunning = false; - mHandler.removeCallbacks(rUpdater); - } + + public void recoverMission(DownloadMission mission) { + ViewHolderItem h = getViewHolder(mission); + if (h == null) return; + + mission.errObject = null; + mission.resetState(true, false, DownloadMission.ERROR_NOTHING); + + h.status.setText(UNDEFINED_PROGRESS); + h.size.setText(Utility.formatBytes(mission.getLength())); + h.progress.setMarquee(true); + + mDownloadManager.resumeMission(mission); } private void updater() { - if (!mUpdaterRunning) return; - - boolean running = false; for (ViewHolderItem h : mPendingDownloadsItems) { // check if the mission is running first if (!((DownloadMission) h.item.mission).running) continue; updateProgress(h); - running = true; } - if (running) { - mHandler.postDelayed(rUpdater, 1000); - } else { - mUpdaterRunning = false; - } + mHandler.postDelayed(rUpdater, 1000); } - private boolean isNotFinite(Float value) { - return Float.isNaN(value) || Float.isInfinite(value); + private boolean isNotFinite(double value) { + return Double.isNaN(value) || Double.isInfinite(value); } public void setRecover(@NonNull RecoverHelper callback) { @@ -789,10 +768,11 @@ public class MissionAdapter extends Adapter implements Handler.Callb MenuItem source; MenuItem checksum; - long lastTimeStamp = -1; - long lastDone = -1; - int lastCurrent = -1; - int state = 0; + long lastTimestamp = -1; + double lastDone; + int lastSpeedIdx; + float[] lastSpeed = new float[3]; + String estimatedTimeArrival = UNDEFINED_ETA; ViewHolderItem(View view) { super(view); @@ -902,6 +882,12 @@ public class MissionAdapter extends Adapter implements Handler.Callb return popup; } + + private void resetSpeedMeasure() { + estimatedTimeArrival = UNDEFINED_ETA; + lastTimestamp = -1; + lastSpeedIdx = -1; + } } class ViewHolderHeader extends RecyclerView.ViewHolder { diff --git a/app/src/main/java/us/shandian/giga/ui/common/Deleter.java b/app/src/main/java/us/shandian/giga/ui/common/Deleter.java index 81b4e33e8..a0828c23d 100644 --- a/app/src/main/java/us/shandian/giga/ui/common/Deleter.java +++ b/app/src/main/java/us/shandian/giga/ui/common/Deleter.java @@ -4,9 +4,10 @@ import android.content.Context; import android.content.Intent; import android.graphics.Color; import android.os.Handler; -import com.google.android.material.snackbar.Snackbar; import android.view.View; +import com.google.android.material.snackbar.Snackbar; + import org.schabi.newpipe.R; import java.util.ArrayList; @@ -113,7 +114,7 @@ public class Deleter { show(); } - private void pause() { + public void pause() { running = false; mHandler.removeCallbacks(rNext); mHandler.removeCallbacks(rShow); @@ -126,13 +127,11 @@ public class Deleter { mHandler.postDelayed(rShow, DELAY_RESUME); } - public void dispose(boolean commitChanges) { + public void dispose() { if (items.size() < 1) return; pause(); - if (!commitChanges) return; - for (Mission mission : items) mDownloadManager.deleteMission(mission); items = null; } diff --git a/app/src/main/java/us/shandian/giga/ui/common/ProgressDrawable.java b/app/src/main/java/us/shandian/giga/ui/common/ProgressDrawable.java index a0ff24aaa..3f638d418 100644 --- a/app/src/main/java/us/shandian/giga/ui/common/ProgressDrawable.java +++ b/app/src/main/java/us/shandian/giga/ui/common/ProgressDrawable.java @@ -9,6 +9,7 @@ import android.graphics.Rect; import android.graphics.drawable.Drawable; import android.os.Handler; import android.os.Looper; + import androidx.annotation.ColorInt; import androidx.annotation.NonNull; @@ -35,8 +36,8 @@ public class ProgressDrawable extends Drawable { mForegroundColor = foreground; } - public void setProgress(float progress) { - mProgress = progress; + public void setProgress(double progress) { + mProgress = (float) progress; invalidateSelf(); } diff --git a/app/src/main/java/us/shandian/giga/ui/fragment/MissionsFragment.java b/app/src/main/java/us/shandian/giga/ui/fragment/MissionsFragment.java index 26da47b1f..921eaff5c 100644 --- a/app/src/main/java/us/shandian/giga/ui/fragment/MissionsFragment.java +++ b/app/src/main/java/us/shandian/giga/ui/fragment/MissionsFragment.java @@ -12,11 +12,6 @@ import android.os.Bundle; import android.os.Environment; import android.os.IBinder; import android.preference.PreferenceManager; -import androidx.annotation.NonNull; -import androidx.fragment.app.Fragment; -import androidx.recyclerview.widget.GridLayoutManager; -import androidx.recyclerview.widget.LinearLayoutManager; -import androidx.recyclerview.widget.RecyclerView; import android.view.LayoutInflater; import android.view.Menu; import android.view.MenuItem; @@ -24,6 +19,12 @@ import android.view.View; import android.view.ViewGroup; import android.widget.Toast; +import androidx.annotation.NonNull; +import androidx.fragment.app.Fragment; +import androidx.recyclerview.widget.GridLayoutManager; +import androidx.recyclerview.widget.LinearLayoutManager; +import androidx.recyclerview.widget.RecyclerView; + import com.nononsenseapps.filepicker.Utils; import org.schabi.newpipe.R; @@ -72,8 +73,7 @@ public class MissionsFragment extends Fragment { mBinder = (DownloadManagerBinder) binder; mBinder.clearDownloadNotifications(); - mAdapter = new MissionAdapter(mContext, mBinder.getDownloadManager(), mEmpty); - mAdapter.deleterLoad(getView()); + mAdapter = new MissionAdapter(mContext, mBinder.getDownloadManager(), mEmpty, getView()); mAdapter.setRecover(MissionsFragment.this::recoverMission); @@ -132,7 +132,7 @@ public class MissionsFragment extends Fragment { * Added in API level 23. */ @Override - public void onAttach(Context context) { + public void onAttach(@NonNull Context context) { super.onAttach(context); // Bug: in api< 23 this is never called @@ -147,7 +147,7 @@ public class MissionsFragment extends Fragment { */ @SuppressWarnings("deprecation") @Override - public void onAttach(Activity activity) { + public void onAttach(@NonNull Activity activity) { super.onAttach(activity); mContext = activity; @@ -162,7 +162,7 @@ public class MissionsFragment extends Fragment { mBinder.removeMissionEventListener(mAdapter); mBinder.enableNotifications(true); mContext.unbindService(mConnection); - mAdapter.deleterDispose(true); + mAdapter.onDestroy(); mBinder = null; mAdapter = null; @@ -196,13 +196,11 @@ public class MissionsFragment extends Fragment { prompt.create().show(); return true; case R.id.start_downloads: - item.setVisible(false); mBinder.getDownloadManager().startAllMissions(); return true; case R.id.pause_downloads: - item.setVisible(false); mBinder.getDownloadManager().pauseAllMissions(false); - mAdapter.ensurePausedMissions();// update items view + mAdapter.refreshMissionItems();// update items view default: return super.onOptionsItemSelected(item); } @@ -271,23 +269,12 @@ public class MissionsFragment extends Fragment { } } - @Override - public void onSaveInstanceState(@NonNull Bundle outState) { - super.onSaveInstanceState(outState); - - if (mAdapter != null) { - mAdapter.deleterDispose(false); - mForceUpdate = true; - mBinder.removeMissionEventListener(mAdapter); - } - } - @Override public void onResume() { super.onResume(); if (mAdapter != null) { - mAdapter.deleterResume(); + mAdapter.onResume(); if (mForceUpdate) { mForceUpdate = false; @@ -303,7 +290,13 @@ public class MissionsFragment extends Fragment { @Override public void onPause() { super.onPause(); - if (mAdapter != null) mAdapter.onPaused(); + + if (mAdapter != null) { + mForceUpdate = true; + mBinder.removeMissionEventListener(mAdapter); + mAdapter.onPaused(); + } + if (mBinder != null) mBinder.enableNotifications(true); } diff --git a/app/src/main/java/us/shandian/giga/util/Utility.java b/app/src/main/java/us/shandian/giga/util/Utility.java index 21fdd72ad..46207777a 100644 --- a/app/src/main/java/us/shandian/giga/util/Utility.java +++ b/app/src/main/java/us/shandian/giga/util/Utility.java @@ -4,13 +4,14 @@ import android.content.ClipData; import android.content.ClipboardManager; import android.content.Context; import android.os.Build; +import android.util.Log; +import android.widget.Toast; + import androidx.annotation.ColorInt; import androidx.annotation.DrawableRes; import androidx.annotation.NonNull; import androidx.annotation.Nullable; import androidx.core.content.ContextCompat; -import android.util.Log; -import android.widget.Toast; import org.schabi.newpipe.R; import org.schabi.newpipe.streams.io.SharpStream; @@ -26,6 +27,7 @@ import java.io.Serializable; import java.net.HttpURLConnection; import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; +import java.util.Locale; import us.shandian.giga.io.StoredFileHelper; @@ -39,26 +41,28 @@ public class Utility { } public static String formatBytes(long bytes) { + Locale locale = Locale.getDefault(); if (bytes < 1024) { - return String.format("%d B", bytes); + return String.format(locale, "%d B", bytes); } else if (bytes < 1024 * 1024) { - return String.format("%.2f kB", bytes / 1024d); + return String.format(locale, "%.2f kB", bytes / 1024d); } else if (bytes < 1024 * 1024 * 1024) { - return String.format("%.2f MB", bytes / 1024d / 1024d); + return String.format(locale, "%.2f MB", bytes / 1024d / 1024d); } else { - return String.format("%.2f GB", bytes / 1024d / 1024d / 1024d); + return String.format(locale, "%.2f GB", bytes / 1024d / 1024d / 1024d); } } - public static String formatSpeed(float speed) { + public static String formatSpeed(double speed) { + Locale locale = Locale.getDefault(); if (speed < 1024) { - return String.format("%.2f B/s", speed); + return String.format(locale, "%.2f B/s", speed); } else if (speed < 1024 * 1024) { - return String.format("%.2f kB/s", speed / 1024); + return String.format(locale, "%.2f kB/s", speed / 1024); } else if (speed < 1024 * 1024 * 1024) { - return String.format("%.2f MB/s", speed / 1024 / 1024); + return String.format(locale, "%.2f MB/s", speed / 1024 / 1024); } else { - return String.format("%.2f GB/s", speed / 1024 / 1024 / 1024); + return String.format(locale, "%.2f GB/s", speed / 1024 / 1024 / 1024); } } @@ -188,12 +192,11 @@ public class Utility { switch (type) { case MUSIC: return R.drawable.music; + default: case VIDEO: return R.drawable.video; case SUBTITLE: return R.drawable.subtitle; - default: - return R.drawable.video; } } @@ -274,4 +277,25 @@ public class Utility { return -1; } + + private static String pad(int number) { + return number < 10 ? ("0" + number) : String.valueOf(number); + } + + public static String stringifySeconds(double seconds) { + int h = (int) Math.floor(seconds / 3600); + int m = (int) Math.floor((seconds - (h * 3600)) / 60); + int s = (int) (seconds - (h * 3600) - (m * 60)); + + String str = ""; + + if (h < 1 && m < 1) { + str = "00:"; + } else { + if (h > 0) str = pad(h) + ":"; + if (m > 0) str += pad(m) + ":"; + } + + return str + pad(s); + } } diff --git a/app/src/main/res/values-ar/strings.xml b/app/src/main/res/values-ar/strings.xml index 43b45d15e..86cbbb59a 100644 --- a/app/src/main/res/values-ar/strings.xml +++ b/app/src/main/res/values-ar/strings.xml @@ -471,7 +471,6 @@ غير موجود فشلت المعالجة الاولية حذف التنزيلات المنتهية - "قم بإستكمال %s حيثما يتم التحويل من التنزيلات" توقف أقصى عدد للمحاولات الحد الأقصى لعدد محاولات قبل إلغاء التحميل diff --git a/app/src/main/res/values-be/strings.xml b/app/src/main/res/values-be/strings.xml index 3c79a96d3..1cf3abd7e 100644 --- a/app/src/main/res/values-be/strings.xml +++ b/app/src/main/res/values-be/strings.xml @@ -458,7 +458,6 @@ Не знойдзена Пасляапрацоўка не ўдалася Ачысціць завершаныя - Аднавіць прыпыненыя загрузкі (%s) Спыніць Максімум спробаў Колькасць спробаў перад адменай загрузкі diff --git a/app/src/main/res/values-cmn/strings.xml b/app/src/main/res/values-cmn/strings.xml index bcb145c16..3ff479bfd 100644 --- a/app/src/main/res/values-cmn/strings.xml +++ b/app/src/main/res/values-cmn/strings.xml @@ -460,7 +460,6 @@ NewPipe 更新可用! 无法创建目标文件夹 服务器不接受多线程下载, 请使用 @string/msg_threads = 1重试 - 继续进行%s个待下载转移 切换至移动数据时有用，尽管一些下载无法被暂停 显示评论 禁用停止显示评论 diff --git a/app/src/main/res/values-cs/strings.xml b/app/src/main/res/values-cs/strings.xml index b741e0d16..9a9cc8654 100644 --- a/app/src/main/res/values-cs/strings.xml +++ b/app/src/main/res/values-cs/strings.xml @@ -466,7 +466,6 @@ otevření ve vyskakovacím okně Nenalezeno Post-processing selhal Vyčistit dokončená stahování - Pokračovat ve stahování %s souborů, čekajících na stažení Zastavit Maximální počet pokusů o opakování Maximální počet pokusů před zrušením stahování diff --git a/app/src/main/res/values-da/strings.xml b/app/src/main/res/values-da/strings.xml index 199c2f85d..5e44aab61 100644 --- a/app/src/main/res/values-da/strings.xml +++ b/app/src/main/res/values-da/strings.xml @@ -447,7 +447,6 @@ sat på pause sat i kø Ryd færdige downloads - Fortsæt dine %s ventende overførsler fra Downloads Maksimalt antal genforsøg Maksimalt antal forsøg før downloaden opgives Sæt på pause ved skift til mobildata diff --git a/app/src/main/res/values-de/strings.xml b/app/src/main/res/values-de/strings.xml index 3279e919c..0dc0de8b4 100644 --- a/app/src/main/res/values-de/strings.xml +++ b/app/src/main/res/values-de/strings.xml @@ -457,7 +457,6 @@ Nicht gefunden Nachbearbeitung fehlgeschlagen Um fertige Downloads bereinigen - Setze deine %s ausstehenden Übertragungen von Downloads fort Anhalten Maximale Wiederholungen Maximalanzahl der Versuche, bevor der Download abgebrochen wird diff --git a/app/src/main/res/values-el/strings.xml b/app/src/main/res/values-el/strings.xml index 372cbb1a2..115b8d0b3 100644 --- a/app/src/main/res/values-el/strings.xml +++ b/app/src/main/res/values-el/strings.xml @@ -459,7 +459,6 @@ Δεν βρέθηκε Μετεπεξεργασία απέτυχε Εκκαθάριση ολοκληρωμένων λήψεων - Συνέχιση των %s εκκρεμών σας λήψεων Διακοπή Μέγιστες επαναπροσπάθειες Μέγιστος αριθμός προσπαθειών προτού γίνει ακύρωση της λήψης diff --git a/app/src/main/res/values-es/strings.xml b/app/src/main/res/values-es/strings.xml index b14aab94b..6fcbc9fa7 100644 --- a/app/src/main/res/values-es/strings.xml +++ b/app/src/main/res/values-es/strings.xml @@ -406,6 +406,7 @@ pausado en cola posprocesamiento + recuperando Añadir a cola Acción denegada por el sistema Se eliminó el archivo @@ -424,7 +425,6 @@ Mostrar como grilla Mostrar como lista Limpiar descargas finalizadas - Tienes %s descargas pendientes, ve a Descargas para continuarlas ¿Lo confirma\? Detener Intentos máximos diff --git a/app/src/main/res/values-et/strings.xml b/app/src/main/res/values-et/strings.xml index 4dfcc3d0e..99dc6cc80 100644 --- a/app/src/main/res/values-et/strings.xml +++ b/app/src/main/res/values-et/strings.xml @@ -460,7 +460,6 @@ Ei leitud Järeltöötlemine nurjus Eemalda lõpetatud allalaadimised - Jätka %s pooleliolevat allalaadimist Stopp Korduskatseid Suurim katsete arv enne allalaadimise tühistamist diff --git a/app/src/main/res/values-eu/strings.xml b/app/src/main/res/values-eu/strings.xml index 7b636d383..743c6b3fb 100644 --- a/app/src/main/res/values-eu/strings.xml +++ b/app/src/main/res/values-eu/strings.xml @@ -459,7 +459,6 @@ Ez aurkitua Post-prozesuak huts egin du Garbitu amaitutako deskargak - Berrekin burutzeke dauden %s transferentzia deskargetatik Gelditu Gehienezko saiakerak Deskarga ezeztatu aurretik saiatu beharreko aldi kopurua diff --git a/app/src/main/res/values-fr/strings.xml b/app/src/main/res/values-fr/strings.xml index b4388e39f..2091a62fe 100644 --- a/app/src/main/res/values-fr/strings.xml +++ b/app/src/main/res/values-fr/strings.xml @@ -466,7 +466,6 @@ Nombre maximum de tentatives avant d’annuler le téléchargement Utilisation des onglets par défaut, erreur lors de la lecture des onglets enregistrés Le serveur n’accepte pas les téléchargements multi-fils, veuillez réessayer avec @string/msg_threads = 1 - Continuer vos %s transferts en attente depuis Téléchargement Afficher les commentaires Désactiver pour ne pas afficher les commentaires Lecture automatique diff --git a/app/src/main/res/values-he/strings.xml b/app/src/main/res/values-he/strings.xml index 5e340d8b3..565f815a1 100644 --- a/app/src/main/res/values-he/strings.xml +++ b/app/src/main/res/values-he/strings.xml @@ -464,7 +464,6 @@ לא נמצא העיבוד המאוחר נכשל פינוי ההורדות שהסתיימו - ניתן להמשיך את %s ההורדות הממתינות שלך דרך ההורדות עצירה מספר הניסיונות החוזרים המרבי מספר הניסיונות החוזרים המרבי בטרם ביטול ההורדה diff --git a/app/src/main/res/values-hr/strings.xml b/app/src/main/res/values-hr/strings.xml index aa4ff9113..a981dcf5e 100644 --- a/app/src/main/res/values-hr/strings.xml +++ b/app/src/main/res/values-hr/strings.xml @@ -457,7 +457,6 @@ Nije pronađeno Naknadna obrada nije uspjela Obriši završena preuzimanja - Nastavite s prijenosima na čekanju za %s s preuzimanja Stop Maksimalnih ponovnih pokušaja Maksimalni broj pokušaja prije poništavanja preuzimanja diff --git a/app/src/main/res/values-id/strings.xml b/app/src/main/res/values-id/strings.xml index d52f5fafa..5fbdcffc1 100644 --- a/app/src/main/res/values-id/strings.xml +++ b/app/src/main/res/values-id/strings.xml @@ -453,7 +453,6 @@ Tidak ditemukan Pengolahan-pasca gagal Hapus unduhan yang sudah selesai - Lanjutkan %s transfer anda yang tertunda dari Unduhan Berhenti Percobaan maksimum Jumlah upaya maksimum sebelum membatalkan unduhan diff --git a/app/src/main/res/values-it/strings.xml b/app/src/main/res/values-it/strings.xml index c92292f99..73633ab03 100644 --- a/app/src/main/res/values-it/strings.xml +++ b/app/src/main/res/values-it/strings.xml @@ -457,7 +457,6 @@ Non trovato Post-processing fallito Pulisci i download completati - Continua i %s trasferimenti in corso dai Download Ferma Tentativi massimi Tentativi massimi prima di cancellare il download diff --git a/app/src/main/res/values-ja/strings.xml b/app/src/main/res/values-ja/strings.xml index 58ca2ebff..4c3aeb5c1 100644 --- a/app/src/main/res/values-ja/strings.xml +++ b/app/src/main/res/values-ja/strings.xml @@ -456,7 +456,6 @@ デフォルトのタブを使用します。保存されたタブの読み込みエラーが発生しました メインページに表示されるタブ 新しいバージョンが利用可能なときにアプリの更新を確認する通知を表示します - ダウンロードから %s の保留中の転送を続行します 従量制課金ネットワークの割り込み モバイルデータ通信に切り替える場合に便利ですが、一部のダウンロードは一時停止できません コメントを表示 diff --git a/app/src/main/res/values-ko/strings.xml b/app/src/main/res/values-ko/strings.xml index fdc76b04e..39b08347c 100644 --- a/app/src/main/res/values-ko/strings.xml +++ b/app/src/main/res/values-ko/strings.xml @@ -454,7 +454,6 @@ HTTP 찾을 수 없습니다 후처리 작업이 실패하였습니다 완료된 다운로드 비우기 - 대기중인 %s 다운로드를 지속하세요 멈추기 최대 재시도 횟수 다운로드를 취소하기 전까지 다시 시도할 최대 횟수 diff --git a/app/src/main/res/values-ms/strings.xml b/app/src/main/res/values-ms/strings.xml index daa120ea2..354e7b7de 100644 --- a/app/src/main/res/values-ms/strings.xml +++ b/app/src/main/res/values-ms/strings.xml @@ -453,7 +453,6 @@ Tidak ditemui Pemprosesan-pasca gagal Hapuskan senarai muat turun yang selesai - Teruskan %s pemindahan anda yang menunggu dari muat turun Berhenti Percubaan maksimum Jumlah percubaan maksimum sebelum membatalkan muat turun diff --git a/app/src/main/res/values-nb-rNO/strings.xml b/app/src/main/res/values-nb-rNO/strings.xml index 6262480b0..e0a08d0a7 100644 --- a/app/src/main/res/values-nb-rNO/strings.xml +++ b/app/src/main/res/values-nb-rNO/strings.xml @@ -458,7 +458,6 @@ Ikke funnet Etterbehandling mislyktes Tøm fullførte nedlastinger - Fortsett dine %s ventende overføringer fra Nedlastinger Stopp Maksimalt antall forsøk Maksimalt antall tilkoblingsforsøk før nedlastingen avblåses diff --git a/app/src/main/res/values-nl-rBE/strings.xml b/app/src/main/res/values-nl-rBE/strings.xml index f64ff6bf9..5c42bfd23 100644 --- a/app/src/main/res/values-nl-rBE/strings.xml +++ b/app/src/main/res/values-nl-rBE/strings.xml @@ -457,7 +457,6 @@ Niet gevonden Nabewerking mislukt Voltooide downloads wissen - Zet uw %s wachtende downloads verder via Downloads Stoppen Maximaal aantal pogingen Maximaal aantal pogingen vooraleer dat den download wordt geannuleerd diff --git a/app/src/main/res/values-nl/strings.xml b/app/src/main/res/values-nl/strings.xml index 6aecc2cd1..b9b86a292 100644 --- a/app/src/main/res/values-nl/strings.xml +++ b/app/src/main/res/values-nl/strings.xml @@ -457,7 +457,6 @@ Niet gevonden Nabewerking mislukt Voltooide downloads wissen - Zet je %s wachtende downloads voort via Downloads Stop Maximum aantal keer proberen Maximum aantal pogingen voordat de download wordt geannuleerd diff --git a/app/src/main/res/values-pa/strings.xml b/app/src/main/res/values-pa/strings.xml index b57564eba..0e579720a 100644 --- a/app/src/main/res/values-pa/strings.xml +++ b/app/src/main/res/values-pa/strings.xml @@ -453,7 +453,6 @@ ਨਹੀਂ ਲਭਿਆ Post-processing ਫੇਲ੍ਹ ਮੁਕੰਮਲ ਹੋਈਆਂ ਡਾਊਨਲੋਡ ਸਾਫ਼ ਕਰੋ - ਡਾਉਨਲੋਡਸ ਤੋਂ ਆਪਣੀਆਂ %s ਬਕਾਇਆ ਟ੍ਰਾਂਸਫਰ ਜਾਰੀ ਰੱਖੋ ਰੁੱਕੋ ਵੱਧ ਤੋਂ ਵੱਧ ਕੋਸ਼ਿਸ਼ਾਂ ਡਾਉਨਲੋਡ ਰੱਦ ਕਰਨ ਤੋਂ ਪਹਿਲਾਂ ਵੱਧ ਤੋਂ ਵੱਧ ਕੋਸ਼ਿਸ਼ਾਂ diff --git a/app/src/main/res/values-pl/strings.xml b/app/src/main/res/values-pl/strings.xml index ca1e52ff2..b7086b34f 100644 --- a/app/src/main/res/values-pl/strings.xml +++ b/app/src/main/res/values-pl/strings.xml @@ -459,7 +459,6 @@ Nie znaleziono Przetwarzanie końcowe nie powiodło się Wyczyść ukończone pobieranie - Kontynuuj %s oczekujące transfery z plików do pobrania Zatrzymaj Maksymalna liczba powtórzeń Maksymalna liczba prób przed anulowaniem pobierania diff --git a/app/src/main/res/values-pt-rBR/strings.xml b/app/src/main/res/values-pt-rBR/strings.xml index 0bdf4d006..5de1e6610 100644 --- a/app/src/main/res/values-pt-rBR/strings.xml +++ b/app/src/main/res/values-pt-rBR/strings.xml @@ -466,7 +466,6 @@ abrir em modo popup Não encontrado Falha no pós processamento Limpar downloads finalizados - Continuar seus %s downloads pendentes Parar Tentativas Máximas Número máximo de tentativas antes de cancelar o download diff --git a/app/src/main/res/values-pt/strings.xml b/app/src/main/res/values-pt/strings.xml index 6d55023d1..88fbb72a6 100644 --- a/app/src/main/res/values-pt/strings.xml +++ b/app/src/main/res/values-pt/strings.xml @@ -455,7 +455,6 @@ Não encontrado Pós-processamento falhado Limpar transferências concluídas - Continue as suas %s transferências pendentes das Transferências Parar Tentativas máximas Número máximo de tentativas antes de cancelar a transferência diff --git a/app/src/main/res/values-ru/strings.xml b/app/src/main/res/values-ru/strings.xml index 51771e1b1..80b587657 100644 --- a/app/src/main/res/values-ru/strings.xml +++ b/app/src/main/res/values-ru/strings.xml @@ -464,7 +464,6 @@ Загрузка завершена %s загрузок завершено Создать уникальное имя - Возобновить приостановленные загрузки (%s) Максимум попыток Количество попыток перед отменой загрузки Некоторые загрузки не поддерживают докачку и начнутся с начала diff --git a/app/src/main/res/values-sk/strings.xml b/app/src/main/res/values-sk/strings.xml index 36c0afd84..cbc201fd5 100644 --- a/app/src/main/res/values-sk/strings.xml +++ b/app/src/main/res/values-sk/strings.xml @@ -465,7 +465,6 @@ Nenájdené Post-spracovanie zlyhalo Vyčistiť dokončené sťahovania - Pokračujte v preberaní %s zo súborov na prevzatie Stop Maximum opakovaní Maximálny počet pokusov pred zrušením stiahnutia diff --git a/app/src/main/res/values-tr/strings.xml b/app/src/main/res/values-tr/strings.xml index 6c9c66f69..1cb6fafd4 100644 --- a/app/src/main/res/values-tr/strings.xml +++ b/app/src/main/res/values-tr/strings.xml @@ -452,7 +452,6 @@ Bulunamadı İşlem sonrası başarısız Tamamlanan indirmeleri temizle - Beklemedeki %s transferinize İndirmeler\'den devam edin Durdur Azami deneme sayısı İndirmeyi iptal etmeden önce maksimum deneme sayısı diff --git a/app/src/main/res/values-uk/strings.xml b/app/src/main/res/values-uk/strings.xml index fcce99e89..d43b8be66 100644 --- a/app/src/main/res/values-uk/strings.xml +++ b/app/src/main/res/values-uk/strings.xml @@ -471,7 +471,6 @@ Помилка зчитування збережених вкладок. Використовую типові вкладки. Вкладки, що відображаються на головній сторінці Показувати сповіщення з пропозицією оновити застосунок за наявності нової версії - Продовжити ваші %s відкладених переміщень із Завантажень Корисно під час переходу на мобільні дані, хоча деякі завантаження не можуть бути призупинені Показувати коментарі Вимнути відображення дописів diff --git a/app/src/main/res/values-vi/strings.xml b/app/src/main/res/values-vi/strings.xml index f8860acfd..ab0983e7a 100644 --- a/app/src/main/res/values-vi/strings.xml +++ b/app/src/main/res/values-vi/strings.xml @@ -452,7 +452,6 @@ Không tìm thấy Xử lý thất bại Dọn các tải về đã hoàn thành - Hãy tiếp tục %s tải về đang chờ Dừng Số lượt thử lại tối đa Số lượt thử lại trước khi hủy tải về diff --git a/app/src/main/res/values-zh-rTW/strings.xml b/app/src/main/res/values-zh-rTW/strings.xml index 310bae3a3..98b9cf381 100644 --- a/app/src/main/res/values-zh-rTW/strings.xml +++ b/app/src/main/res/values-zh-rTW/strings.xml @@ -450,7 +450,6 @@ 找不到 後處理失敗 清除已結束的下載 - 繼續從您所擱置中的下載 %s 傳輸 停止 最大重試次數 在取消下載前的最大嘗試數 diff --git a/app/src/main/res/values/strings.xml b/app/src/main/res/values/strings.xml index f929e0d2b..c2d8d70f2 100644 --- a/app/src/main/res/values/strings.xml +++ b/app/src/main/res/values/strings.xml @@ -526,6 +526,7 @@ paused queued post-processing + recovering Queue Action denied by the system @@ -560,7 +561,6 @@ Cannot recover this download Clear finished downloads Are you sure? - Continue your %s pending transfers from Downloads Stop Maximum retries Maximum number of attempts before canceling the download From 3ca461413ed3ae200852d5a19fb0f7ed67e2ff86 Mon Sep 17 00:00:00 2001 From: kapodamy Date: Sun, 24 Nov 2019 14:00:22 -0300 Subject: [PATCH 24/28] Merge branch 'dev' into dl-last-features --- .../us/shandian/giga/get/DownloadMission.java | 26 +++++++++++++++---- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/app/src/main/java/us/shandian/giga/get/DownloadMission.java b/app/src/main/java/us/shandian/giga/get/DownloadMission.java index 5ef72162c..917a0a148 100644 --- a/app/src/main/java/us/shandian/giga/get/DownloadMission.java +++ b/app/src/main/java/us/shandian/giga/get/DownloadMission.java @@ -1,6 +1,9 @@ package us.shandian.giga.get; +import android.os.Build; import android.os.Handler; +import android.system.ErrnoException; +import android.system.OsConstants; import android.util.Log; import androidx.annotation.Nullable; @@ -35,9 +38,6 @@ public class DownloadMission extends Mission { static final int BUFFER_SIZE = 64 * 1024; static final int BLOCK_SIZE = 512 * 1024; - @SuppressWarnings("SpellCheckingInspection") - private static final String INSUFFICIENT_STORAGE = "ENOSPC"; - private static final String TAG = "DownloadMission"; public static final int ERROR_NOTHING = -1; @@ -315,13 +315,29 @@ public class DownloadMission extends Mission { public synchronized void notifyError(int code, Exception err) { Log.e(TAG, "notifyError() code = " + code, err); + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.LOLLIPOP) { + if (err.getCause() instanceof ErrnoException) { + int errno = ((ErrnoException) err.getCause()).errno; + if (errno == OsConstants.ENOSPC) { + code = ERROR_INSUFFICIENT_STORAGE; + err = null; + } else if (errno == OsConstants.EACCES) { + code = ERROR_PERMISSION_DENIED; + err = null; + } + } + } + if (err instanceof IOException) { - if (!storage.canWrite() || err.getMessage().contains("Permission denied")) { + if (err.getMessage().contains("Permission denied")) { code = ERROR_PERMISSION_DENIED; err = null; - } else if (err.getMessage().contains(INSUFFICIENT_STORAGE)) { + } else if (err.getMessage().contains("ENOSPC")) { code = ERROR_INSUFFICIENT_STORAGE; err = null; + } else if (!storage.canWrite()) { + code = ERROR_FILE_CREATION; + err = null; } } From 84ec320df4f314cde85847e9f7d1373eb34740dc Mon Sep 17 00:00:00 2001 From: kapodamy Date: Tue, 26 Nov 2019 13:41:16 -0300 Subject: [PATCH 25/28] commit * rebase fixup, add null check * better ETA string * drop connection read timeout, for HSDPA networks * bump NPE version --- app/build.gradle | 2 +- .../main/java/org/schabi/newpipe/download/DownloadDialog.java | 2 +- app/src/main/java/us/shandian/giga/get/DownloadMission.java | 3 +-- .../main/java/us/shandian/giga/ui/adapter/MissionAdapter.java | 4 ++-- 4 files changed, 5 insertions(+), 6 deletions(-) diff --git a/app/build.gradle b/app/build.gradle index a1afd63a2..7e4707f99 100644 --- a/app/build.gradle +++ b/app/build.gradle @@ -62,7 +62,7 @@ dependencies { exclude module: 'support-annotations' }) - implementation 'com.github.TeamNewPipe:NewPipeExtractor:5c420340ceb39' + implementation 'com.github.TeamNewPipe:NewPipeExtractor:b6d3252' testImplementation 'junit:junit:4.12' testImplementation 'org.mockito:mockito-core:2.23.0' diff --git a/app/src/main/java/org/schabi/newpipe/download/DownloadDialog.java b/app/src/main/java/org/schabi/newpipe/download/DownloadDialog.java index 60b6192be..29208b0e0 100644 --- a/app/src/main/java/org/schabi/newpipe/download/DownloadDialog.java +++ b/app/src/main/java/org/schabi/newpipe/download/DownloadDialog.java @@ -780,7 +780,7 @@ public class DownloadDialog extends DialogFragment implements RadioGroup.OnCheck if (selectedStream.getFormat() == MediaFormat.M4A) { psName = Postprocessing.ALGORITHM_M4A_NO_DASH; - } else if (selectedStream.getFormat() == MediaFormat.OPUS) { + } else if (selectedStream.getFormat() == MediaFormat.WEBMA_OPUS) { psName = Postprocessing.ALGORITHM_OGG_FROM_WEBM_DEMUXER; } break; diff --git a/app/src/main/java/us/shandian/giga/get/DownloadMission.java b/app/src/main/java/us/shandian/giga/get/DownloadMission.java index 917a0a148..c0f85b321 100644 --- a/app/src/main/java/us/shandian/giga/get/DownloadMission.java +++ b/app/src/main/java/us/shandian/giga/get/DownloadMission.java @@ -228,7 +228,6 @@ public class DownloadMission extends Mission { // BUG workaround: switching between networks can freeze the download forever conn.setConnectTimeout(30000); - conn.setReadTimeout(10000); if (rangeStart >= 0) { String req = "bytes=" + rangeStart + "-"; @@ -316,7 +315,7 @@ public class DownloadMission extends Mission { public synchronized void notifyError(int code, Exception err) { Log.e(TAG, "notifyError() code = " + code, err); if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.LOLLIPOP) { - if (err.getCause() instanceof ErrnoException) { + if (err != null && err.getCause() instanceof ErrnoException) { int errno = ((ErrnoException) err.getCause()).errno; if (errno == OsConstants.ENOSPC) { code = ERROR_INSUFFICIENT_STORAGE; diff --git a/app/src/main/java/us/shandian/giga/ui/adapter/MissionAdapter.java b/app/src/main/java/us/shandian/giga/ui/adapter/MissionAdapter.java index e3a7f112a..8420e343b 100644 --- a/app/src/main/java/us/shandian/giga/ui/adapter/MissionAdapter.java +++ b/app/src/main/java/us/shandian/giga/ui/adapter/MissionAdapter.java @@ -308,10 +308,10 @@ public class MissionAdapter extends Adapter implements Handler.Callb etaStr = ""; } else { long eta = (long) Math.ceil((length - done) / averageSpeed); - etaStr = " @ ".concat(Utility.stringifySeconds(eta)); + etaStr = Utility.formatBytes((long) done) + "/" + Utility.stringifySeconds(eta) + " "; } - h.size.setText(sizeStr.concat(speedStr).concat(etaStr)); + h.size.setText(sizeStr.concat(etaStr).concat(speedStr)); h.lastTimestamp = now; h.lastDone = done; From aae8865bdd1a57379e748dd8ff4ed870ff3f7392 Mon Sep 17 00:00:00 2001 From: kapodamy Date: Thu, 5 Dec 2019 14:04:48 -0300 Subject: [PATCH 26/28] remove unused imports --- .../java/org/schabi/newpipe/fragments/MainFragment.java | 9 --------- .../us/shandian/giga/get/DownloadMissionRecover.java | 5 ----- .../java/us/shandian/giga/get/MissionRecoveryInfo.java | 1 - 3 files changed, 15 deletions(-) diff --git a/app/src/main/java/org/schabi/newpipe/fragments/MainFragment.java b/app/src/main/java/org/schabi/newpipe/fragments/MainFragment.java index 70e0d9fb1..720e0f216 100644 --- a/app/src/main/java/org/schabi/newpipe/fragments/MainFragment.java +++ b/app/src/main/java/org/schabi/newpipe/fragments/MainFragment.java @@ -2,15 +2,6 @@ package org.schabi.newpipe.fragments; import android.content.Context; import android.os.Bundle; -import androidx.annotation.NonNull; -import androidx.annotation.Nullable; -import com.google.android.material.tabs.TabLayout; -import androidx.fragment.app.Fragment; -import androidx.fragment.app.FragmentManager; -import androidx.fragment.app.FragmentPagerAdapter; -import androidx.viewpager.widget.ViewPager; -import androidx.appcompat.app.ActionBar; -import androidx.appcompat.app.AppCompatActivity; import android.util.Log; import android.view.LayoutInflater; import android.view.Menu; diff --git a/app/src/main/java/us/shandian/giga/get/DownloadMissionRecover.java b/app/src/main/java/us/shandian/giga/get/DownloadMissionRecover.java index eb660e564..14ac392a0 100644 --- a/app/src/main/java/us/shandian/giga/get/DownloadMissionRecover.java +++ b/app/src/main/java/us/shandian/giga/get/DownloadMissionRecover.java @@ -66,11 +66,6 @@ public class DownloadMissionRecover extends Thread { } private void tryRecover() throws ExtractionException, IOException, HttpError { - /*if (mMission.source.startsWith(MissionRecoveryInfo.DIRECT_SOURCE)) { - resolve(mMission.source.substring(MissionRecoveryInfo.DIRECT_SOURCE.length())); - return; - }*/ - if (mExtractor == null) { try { StreamingService svr = NewPipe.getServiceByUrl(mMission.source); diff --git a/app/src/main/java/us/shandian/giga/get/MissionRecoveryInfo.java b/app/src/main/java/us/shandian/giga/get/MissionRecoveryInfo.java index f6a3a3984..e52f35cc6 100644 --- a/app/src/main/java/us/shandian/giga/get/MissionRecoveryInfo.java +++ b/app/src/main/java/us/shandian/giga/get/MissionRecoveryInfo.java @@ -15,7 +15,6 @@ import java.io.Serializable; public class MissionRecoveryInfo implements Serializable, Parcelable { private static final long serialVersionUID = 0L; - //public static final String DIRECT_SOURCE = "direct-source://"; MediaFormat format; String desired; From 5a2cd93d13d08915857712e7a693ec370d34c5e4 Mon Sep 17 00:00:00 2001 From: kapodamy Date: Fri, 6 Dec 2019 16:30:07 -0300 Subject: [PATCH 27/28] remove netbeans editor-fold comments --- .../schabi/newpipe/streams/Mp4DashReader.java | 13 +++++-------- .../newpipe/streams/Mp4FromDashWriter.java | 16 ++++++++-------- .../newpipe/streams/OggFromWebMWriter.java | 9 +++------ .../org/schabi/newpipe/streams/WebMReader.java | 16 ++++++++-------- 4 files changed, 24 insertions(+), 30 deletions(-) diff --git a/app/src/main/java/org/schabi/newpipe/streams/Mp4DashReader.java b/app/src/main/java/org/schabi/newpipe/streams/Mp4DashReader.java index c52ebf3aa..0cfd856e1 100644 --- a/app/src/main/java/org/schabi/newpipe/streams/Mp4DashReader.java +++ b/app/src/main/java/org/schabi/newpipe/streams/Mp4DashReader.java @@ -15,7 +15,6 @@ import java.util.NoSuchElementException; */ public class Mp4DashReader { - // private static final int ATOM_MOOF = 0x6D6F6F66; private static final int ATOM_MFHD = 0x6D666864; private static final int ATOM_TRAF = 0x74726166; @@ -50,7 +49,7 @@ public class Mp4DashReader { private static final int HANDLER_VIDE = 0x76696465; private static final int HANDLER_SOUN = 0x736F756E; private static final int HANDLER_SUBT = 0x73756274; - // + private final DataReader stream; @@ -293,7 +292,8 @@ public class Mp4DashReader { return null; } - // + + private long readUint() throws IOException { return stream.readInt() & 0xffffffffL; } @@ -392,9 +392,7 @@ public class Mp4DashReader { return readBox(); } - // - // private Moof parse_moof(Box ref, int trackId) throws IOException { Moof obj = new Moof(); @@ -795,9 +793,8 @@ public class Mp4DashReader { return readFullBox(b); } - // - // + class Box { int type; @@ -1013,5 +1010,5 @@ public class Mp4DashReader { public TrunEntry info; public byte[] data; } -// + } diff --git a/app/src/main/java/org/schabi/newpipe/streams/Mp4FromDashWriter.java b/app/src/main/java/org/schabi/newpipe/streams/Mp4FromDashWriter.java index 420f77955..818f6148e 100644 --- a/app/src/main/java/org/schabi/newpipe/streams/Mp4FromDashWriter.java +++ b/app/src/main/java/org/schabi/newpipe/streams/Mp4FromDashWriter.java @@ -161,7 +161,7 @@ public class Mp4FromDashWriter { boolean singleChunk = tracks.length == 1 && tracks[0].kind == TrackKind.Audio; - // + for (int i = 0; i < readers.length; i++) { int samplesSize = 0; int sampleSizeChanges = 0; @@ -255,7 +255,7 @@ public class Mp4FromDashWriter { tracks[i].trak.tkhd.duration = sampleExtra[i];// this never should happen } } - // + boolean is64 = read > THRESHOLD_FOR_CO64; @@ -426,7 +426,7 @@ public class Mp4FromDashWriter { } } - // + private int writeEntry64(int offset, long value) throws IOException { outBackup(); @@ -469,9 +469,9 @@ public class Mp4FromDashWriter { lastWriteOffset = -1; } } - // - // + + private void outWrite(byte[] buffer) throws IOException { outWrite(buffer, buffer.length); } @@ -581,9 +581,9 @@ public class Mp4FromDashWriter { private int auxOffset() { return auxBuffer == null ? (int) writeOffset : auxBuffer.position(); } - // - // + + private int make_ftyp() throws IOException { byte[] buffer = new byte[]{ 0x00, 0x00, 0x00, 0x1C, 0x66, 0x74, 0x79, 0x70,// ftyp @@ -815,7 +815,7 @@ public class Mp4FromDashWriter { return buffer.array(); } - // + class TablesInfo { diff --git a/app/src/main/java/org/schabi/newpipe/streams/OggFromWebMWriter.java b/app/src/main/java/org/schabi/newpipe/streams/OggFromWebMWriter.java index 37bf9c6d7..20e88c4c7 100644 --- a/app/src/main/java/org/schabi/newpipe/streams/OggFromWebMWriter.java +++ b/app/src/main/java/org/schabi/newpipe/streams/OggFromWebMWriter.java @@ -308,7 +308,8 @@ public class OggFromWebMWriter implements Closeable { buffer.position(0); } - // + + @Nullable private SimpleBlock getNextBlock() throws IOException { SimpleBlock res; @@ -359,9 +360,7 @@ public class OggFromWebMWriter implements Closeable { return 0f; } - // - // private void clearSegmentTable() { segment_table_next_timestamp += TIME_SCALE_NS; packet_flag = FLAG_UNSET; @@ -407,9 +406,7 @@ public class OggFromWebMWriter implements Closeable { return true; } - // - // private void populate_crc32_table() { for (int i = 0; i < 0x100; i++) { int crc = i << 24; @@ -430,5 +427,5 @@ public class OggFromWebMWriter implements Closeable { return initial_crc; } - // + } diff --git a/app/src/main/java/org/schabi/newpipe/streams/WebMReader.java b/app/src/main/java/org/schabi/newpipe/streams/WebMReader.java index 4cb96d901..42875c364 100644 --- a/app/src/main/java/org/schabi/newpipe/streams/WebMReader.java +++ b/app/src/main/java/org/schabi/newpipe/streams/WebMReader.java @@ -15,7 +15,6 @@ import java.util.NoSuchElementException; */ public class WebMReader { - // private final static int ID_EMBL = 0x0A45DFA3; private final static int ID_EMBLReadVersion = 0x02F7; private final static int ID_EMBLDocType = 0x0282; @@ -44,7 +43,7 @@ public class WebMReader { private final static int ID_SimpleBlock = 0x23; private final static int ID_Block = 0x21; private final static int ID_GroupBlock = 0x20; -// + public enum TrackKind { Audio/*2*/, Video/*1*/, Other @@ -110,7 +109,8 @@ public class WebMReader { return segment; } - // + + private long readNumber(Element parent) throws IOException { int length = (int) parent.contentSize; long value = 0; @@ -225,9 +225,9 @@ public class WebMReader { stream.skipBytes(skip); } -// - // + + private boolean readEbml(Element ref, int minReadVersion, int minDocTypeVersion) throws IOException { Element elem = untilElement(ref, ID_EMBLReadVersion); if (elem == null) { @@ -389,9 +389,9 @@ public class WebMReader { return obj; } -// - // + + class Element { int type; @@ -536,5 +536,5 @@ public class WebMReader { } } -// + } From 03939555ace804af32113f50d061ebc7889d82fe Mon Sep 17 00:00:00 2001 From: kapodamy Date: Sat, 7 Dec 2019 00:16:01 -0300 Subject: [PATCH 28/28] add missing change after updating NPE use +webm_opus instead of +opus --- .../java/org/schabi/newpipe/util/SecondaryStreamHelper.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/app/src/main/java/org/schabi/newpipe/util/SecondaryStreamHelper.java b/app/src/main/java/org/schabi/newpipe/util/SecondaryStreamHelper.java index d2ebcd9f8..ab58bc917 100644 --- a/app/src/main/java/org/schabi/newpipe/util/SecondaryStreamHelper.java +++ b/app/src/main/java/org/schabi/newpipe/util/SecondaryStreamHelper.java @@ -52,10 +52,12 @@ public class SecondaryStreamHelper { } } + if (m4v) return null; + // retry, but this time in reverse order for (int i = audioStreams.size() - 1; i >= 0; i--) { AudioStream audio = audioStreams.get(i); - if (audio.getFormat() == (m4v ? MediaFormat.MP3 : MediaFormat.OPUS)) { + if (audio.getFormat() == MediaFormat.WEBMA_OPUS) { return audio; } }