be-gay-do-crimes
/
NewPipeExtractor
mirror of https://github.com/TeamNewPipe/NewPipeExtractor
1
0
Fork 0
Code Issues Releases Wiki Activity

Merge pull request #417 from Isira-Seneviratne/Use_Java_8_Date_Time_API 

Use the Java 8 Date/Time API.
This commit is contained in:
Tobias Groza 2020-11-01 16:49:06 +01:00 committed by GitHub
parent 6cc50b57e3 fcdfe7d939
commit 8cbdec675b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
16 changed files with 167 additions and 215 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
README.md
View File

@ -13,6 +13,8 @@ If you're using Gradle, you could add NewPipe Extractor as a dependency with the
1. Add `maven { url 'https://jitpack.io' }` to the `repositories` in your `build.gradle`. 1. Add `maven { url 'https://jitpack.io' }` to the `repositories` in your `build.gradle`.
2. Add `implementation 'com.github.TeamNewPipe:NewPipeExtractor:v0.20.1'`the `dependencies` in your `build.gradle`. Replace `v0.20.1` with the latest release. 2. Add `implementation 'com.github.TeamNewPipe:NewPipeExtractor:v0.20.1'`the `dependencies` in your `build.gradle`. Replace `v0.20.1` with the latest release.
**Note:** To use NewPipe Extractor in projects with a `minSdkVersion` below 26, [API desugaring](https://developer.android.com/studio/write/java8-support#library-desugaring) is required.
### Testing changes ### Testing changes
To test changes quickly you can build the library locally. A good approach would be to add something like the following to your `settings.gradle`: To test changes quickly you can build the library locally. A good approach would be to add something like the following to your `settings.gradle`:

4
build.gradle
View File

@ -2,8 +2,8 @@ allprojects {
apply plugin: 'java-library' apply plugin: 'java-library'
apply plugin: 'maven' apply plugin: 'maven'
sourceCompatibility = 1.7 sourceCompatibility = 1.8
targetCompatibility = 1.7 targetCompatibility = 1.8
version 'v0.20.2' version 'v0.20.2'
group 'com.github.TeamNewPipe' group 'com.github.TeamNewPipe'

47
extractor/src/main/java/org/schabi/newpipe/extractor/localization/DateWrapper.java
View File

@ -3,30 +3,61 @@ package org.schabi.newpipe.extractor.localization;
import edu.umd.cs.findbugs.annotations.NonNull; import edu.umd.cs.findbugs.annotations.NonNull;
import java.io.Serializable; import java.io.Serializable;
import java.time.OffsetDateTime;
import java.time.ZoneOffset;
import java.util.Calendar; import java.util.Calendar;
import java.util.GregorianCalendar;
/** /**
* A wrapper class that provides a field to describe if the date is precise or just an approximation. * A wrapper class that provides a field to describe if the date/time is precise or just an approximation.
*/ */
public class DateWrapper implements Serializable { public class DateWrapper implements Serializable {
@NonNull private final Calendar date; @NonNull private final OffsetDateTime offsetDateTime;
private final boolean isApproximation; private final boolean isApproximation;
public DateWrapper(@NonNull Calendar date) { /**
this(date, false); * @deprecated Use {@link #DateWrapper(OffsetDateTime)} instead.
*/
@Deprecated
public DateWrapper(@NonNull Calendar calendar) {
this(calendar, false);
} }
public DateWrapper(@NonNull Calendar date, boolean isApproximation) { /**
this.date = date; * @deprecated Use {@link #DateWrapper(OffsetDateTime, boolean)} instead.
*/
@Deprecated
public DateWrapper(@NonNull Calendar calendar, boolean isApproximation) {
offsetDateTime = OffsetDateTime.ofInstant(calendar.toInstant(), ZoneOffset.UTC);
this.isApproximation = isApproximation;
}
public DateWrapper(@NonNull OffsetDateTime offsetDateTime) {
this(offsetDateTime, false);
}
public DateWrapper(@NonNull OffsetDateTime offsetDateTime, boolean isApproximation) {
this.offsetDateTime = offsetDateTime.withOffsetSameInstant(ZoneOffset.UTC);
this.isApproximation = isApproximation; this.isApproximation = isApproximation;
} }
/** /**
* @return the wrapped date. * @return the wrapped date/time as a {@link Calendar}.
*
* @deprecated use {@link #offsetDateTime()} instead.
*/ */
@Deprecated
@NonNull @NonNull
public Calendar date() { public Calendar date() {
return date; return GregorianCalendar.from(offsetDateTime.toZonedDateTime());
}
/**
* @return the wrapped date/time.
*/
@NonNull
public OffsetDateTime offsetDateTime() {
return offsetDateTime;
} }
/** /**

75
extractor/src/main/java/org/schabi/newpipe/extractor/localization/TimeAgoParser.java
View File

@ -2,10 +2,11 @@ package org.schabi.newpipe.extractor.localization;
import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.timeago.PatternsHolder; import org.schabi.newpipe.extractor.timeago.PatternsHolder;
import org.schabi.newpipe.extractor.timeago.TimeAgoUnit;
import org.schabi.newpipe.extractor.utils.Parser; import org.schabi.newpipe.extractor.utils.Parser;
import java.util.Calendar; import java.time.OffsetDateTime;
import java.time.ZoneOffset;
import java.time.temporal.ChronoUnit;
import java.util.Collection; import java.util.Collection;
import java.util.Map; import java.util.Map;
import java.util.regex.Pattern; import java.util.regex.Pattern;
@ -16,7 +17,7 @@ import java.util.regex.Pattern;
*/ */
public class TimeAgoParser { public class TimeAgoParser {
private final PatternsHolder patternsHolder; private final PatternsHolder patternsHolder;
private final Calendar consistentNow; private final OffsetDateTime now;
/** /**
* Creates a helper to parse upload dates in the format '2 days ago'. * Creates a helper to parse upload dates in the format '2 days ago'.
@ -28,7 +29,7 @@ public class TimeAgoParser {
*/ */
public TimeAgoParser(PatternsHolder patternsHolder) { public TimeAgoParser(PatternsHolder patternsHolder) {
this.patternsHolder = patternsHolder; this.patternsHolder = patternsHolder;
consistentNow = Calendar.getInstance(); now = OffsetDateTime.now(ZoneOffset.UTC);
} }
/** /**
@ -42,14 +43,14 @@ public class TimeAgoParser {
* @throws ParsingException if the time unit could not be recognized * @throws ParsingException if the time unit could not be recognized
*/ */
public DateWrapper parse(String textualDate) throws ParsingException { public DateWrapper parse(String textualDate) throws ParsingException {
for (Map.Entry<TimeAgoUnit, Map<String, Integer>> caseUnitEntry : patternsHolder.specialCases().entrySet()) { for (Map.Entry<ChronoUnit, Map<String, Integer>> caseUnitEntry : patternsHolder.specialCases().entrySet()) {
final TimeAgoUnit timeAgoUnit = caseUnitEntry.getKey(); final ChronoUnit chronoUnit = caseUnitEntry.getKey();
for (Map.Entry<String, Integer> caseMapToAmountEntry : caseUnitEntry.getValue().entrySet()) { for (Map.Entry<String, Integer> caseMapToAmountEntry : caseUnitEntry.getValue().entrySet()) {
final String caseText = caseMapToAmountEntry.getKey(); final String caseText = caseMapToAmountEntry.getKey();
final Integer caseAmount = caseMapToAmountEntry.getValue(); final Integer caseAmount = caseMapToAmountEntry.getValue();
if (textualDateMatches(textualDate, caseText)) { if (textualDateMatches(textualDate, caseText)) {
return getResultFor(caseAmount, timeAgoUnit); return getResultFor(caseAmount, chronoUnit);
} }
} }
} }
@ -63,8 +64,8 @@ public class TimeAgoParser {
timeAgoAmount = 1; timeAgoAmount = 1;
} }
final TimeAgoUnit timeAgoUnit = parseTimeAgoUnit(textualDate); final ChronoUnit chronoUnit = parseChronoUnit(textualDate);
return getResultFor(timeAgoAmount, timeAgoUnit); return getResultFor(timeAgoAmount, chronoUnit);
} }
private int parseTimeAgoAmount(String textualDate) throws NumberFormatException { private int parseTimeAgoAmount(String textualDate) throws NumberFormatException {
@ -72,13 +73,13 @@ public class TimeAgoParser {
return Integer.parseInt(timeValueStr); return Integer.parseInt(timeValueStr);
} }
private TimeAgoUnit parseTimeAgoUnit(String textualDate) throws ParsingException { private ChronoUnit parseChronoUnit(String textualDate) throws ParsingException {
for (Map.Entry<TimeAgoUnit, Collection<String>> entry : patternsHolder.asMap().entrySet()) { for (Map.Entry<ChronoUnit, Collection<String>> entry : patternsHolder.asMap().entrySet()) {
final TimeAgoUnit timeAgoUnit = entry.getKey(); final ChronoUnit chronoUnit = entry.getKey();
for (String agoPhrase : entry.getValue()) { for (String agoPhrase : entry.getValue()) {
if (textualDateMatches(textualDate, agoPhrase)) { if (textualDateMatches(textualDate, agoPhrase)) {
return timeAgoUnit; return chronoUnit;
} }
} }
} }
@ -112,65 +113,35 @@ public class TimeAgoParser {
} }
} }
private DateWrapper getResultFor(int timeAgoAmount, TimeAgoUnit timeAgoUnit) { private DateWrapper getResultFor(int timeAgoAmount, ChronoUnit chronoUnit) {
final Calendar calendarTime = getNow(); OffsetDateTime offsetDateTime = now;
boolean isApproximation = false; boolean isApproximation = false;
switch (timeAgoUnit) { switch (chronoUnit) {
case SECONDS: case SECONDS:
calendarTime.add(Calendar.SECOND, -timeAgoAmount);
break;
case MINUTES: case MINUTES:
calendarTime.add(Calendar.MINUTE, -timeAgoAmount);
break;
case HOURS: case HOURS:
calendarTime.add(Calendar.HOUR_OF_DAY, -timeAgoAmount); offsetDateTime = offsetDateTime.minus(timeAgoAmount, chronoUnit);
break; break;
case DAYS: case DAYS:
calendarTime.add(Calendar.DAY_OF_MONTH, -timeAgoAmount);
isApproximation = true;
break;
case WEEKS: case WEEKS:
calendarTime.add(Calendar.WEEK_OF_YEAR, -timeAgoAmount);
isApproximation = true;
break;
case MONTHS: case MONTHS:
calendarTime.add(Calendar.MONTH, -timeAgoAmount); offsetDateTime = offsetDateTime.minus(timeAgoAmount, chronoUnit);
isApproximation = true; isApproximation = true;
break; break;
case YEARS: case YEARS:
calendarTime.add(Calendar.YEAR, -timeAgoAmount); // minusDays is needed to prevent `PrettyTime` from showing '12 months ago'.
// Prevent `PrettyTime` from showing '12 months ago'. offsetDateTime = offsetDateTime.minusYears(timeAgoAmount).minusDays(1);
calendarTime.add(Calendar.DAY_OF_MONTH, -1);
isApproximation = true; isApproximation = true;
break; break;
} }
if (isApproximation) { if (isApproximation) {
markApproximatedTime(calendarTime); offsetDateTime = offsetDateTime.truncatedTo(ChronoUnit.HOURS);
} }
return new DateWrapper(calendarTime, isApproximation); return new DateWrapper(offsetDateTime, isApproximation);
}
private Calendar getNow() {
return (Calendar) consistentNow.clone();
}
/**
* Marks the time as approximated by setting minutes, seconds and milliseconds to 0.
*
* @param calendarTime Time to be marked as approximated
*/
private void markApproximatedTime(Calendar calendarTime) {
calendarTime.set(Calendar.MINUTE, 0);
calendarTime.set(Calendar.SECOND, 0);
calendarTime.set(Calendar.MILLISECOND, 0);
} }
} }

21
extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCParsingHelper.java
View File

@ -2,28 +2,17 @@ package org.schabi.newpipe.extractor.services.media_ccc.extractors;
import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.exceptions.ParsingException;
import java.text.ParseException; import java.time.OffsetDateTime;
import java.text.SimpleDateFormat; import java.time.format.DateTimeParseException;
import java.util.Calendar;
import java.util.Date;
import java.util.TimeZone;
public final class MediaCCCParsingHelper { public final class MediaCCCParsingHelper {
private MediaCCCParsingHelper() { } private MediaCCCParsingHelper() { }
public static Calendar parseDateFrom(final String textualUploadDate) throws ParsingException { public static OffsetDateTime parseDateFrom(final String textualUploadDate) throws ParsingException {
final Date date;
try { try {
final SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd"); return OffsetDateTime.parse(textualUploadDate);
sdf.setTimeZone(TimeZone.getTimeZone("GMT")); } catch (DateTimeParseException e) {
date = sdf.parse(textualUploadDate);
} catch (ParseException e) {
throw new ParsingException("Could not parse date: \"" + textualUploadDate + "\"", e); throw new ParsingException("Could not parse date: \"" + textualUploadDate + "\"", e);
} }
final Calendar uploadDate = Calendar.getInstance();
uploadDate.setTime(date);
return uploadDate;
} }
} }

23
extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeParsingHelper.java
View File

@ -2,7 +2,6 @@ package org.schabi.newpipe.extractor.services.peertube;
import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonObject;
import org.schabi.newpipe.extractor.InfoItemsCollector; import org.schabi.newpipe.extractor.InfoItemsCollector;
import org.schabi.newpipe.extractor.Page; import org.schabi.newpipe.extractor.Page;
import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException; import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException;
@ -12,11 +11,10 @@ import org.schabi.newpipe.extractor.utils.JsonUtils;
import org.schabi.newpipe.extractor.utils.Parser; import org.schabi.newpipe.extractor.utils.Parser;
import org.schabi.newpipe.extractor.utils.Utils; import org.schabi.newpipe.extractor.utils.Utils;
import java.text.ParseException; import java.time.Instant;
import java.text.SimpleDateFormat; import java.time.OffsetDateTime;
import java.util.Calendar; import java.time.ZoneOffset;
import java.util.Date; import java.time.format.DateTimeParseException;
import java.util.TimeZone;
public class PeertubeParsingHelper { public class PeertubeParsingHelper {
public static final String START_KEY = "start"; public static final String START_KEY = "start";
@ -34,19 +32,12 @@ public class PeertubeParsingHelper {
} }
} }
public static Calendar parseDateFrom(final String textualUploadDate) throws ParsingException { public static OffsetDateTime parseDateFrom(final String textualUploadDate) throws ParsingException {
final Date date;
try { try {
final SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.S'Z'"); return OffsetDateTime.ofInstant(Instant.parse(textualUploadDate), ZoneOffset.UTC);
sdf.setTimeZone(TimeZone.getTimeZone("GMT")); } catch (DateTimeParseException e) {
date = sdf.parse(textualUploadDate);
} catch (ParseException e) {
throw new ParsingException("Could not parse date: \"" + textualUploadDate + "\"", e); throw new ParsingException("Could not parse date: \"" + textualUploadDate + "\"", e);
} }
final Calendar uploadDate = Calendar.getInstance();
uploadDate.setTime(date);
return uploadDate;
} }
public static Page getNextPage(final String prevPageUrl, final long total) { public static Page getNextPage(final String prevPageUrl, final long total) {

27
extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java
View File

@ -21,16 +21,18 @@ import org.schabi.newpipe.extractor.services.soundcloud.extractors.SoundcloudStr
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
import org.schabi.newpipe.extractor.utils.Parser; import org.schabi.newpipe.extractor.utils.Parser;
import org.schabi.newpipe.extractor.utils.Parser.RegexException; import org.schabi.newpipe.extractor.utils.Parser.RegexException;
import org.schabi.newpipe.extractor.utils.Utils;
import javax.annotation.Nonnull; import javax.annotation.Nonnull;
import java.io.IOException; import java.io.IOException;
import java.net.MalformedURLException; import java.net.MalformedURLException;
import java.net.URL; import java.net.URL;
import java.net.URLEncoder; import java.net.URLEncoder;
import java.text.ParseException; import java.time.OffsetDateTime;
import java.text.SimpleDateFormat; import java.time.format.DateTimeFormatter;
import java.util.*; import java.time.format.DateTimeParseException;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import static java.util.Collections.singletonList; import static java.util.Collections.singletonList;
import static org.schabi.newpipe.extractor.ServiceList.SoundCloud; import static org.schabi.newpipe.extractor.ServiceList.SoundCloud;
@ -95,23 +97,16 @@ public class SoundcloudParsingHelper {
} }
} }
public static Calendar parseDateFrom(String textualUploadDate) throws ParsingException { public static OffsetDateTime parseDateFrom(String textualUploadDate) throws ParsingException {
Date date;
try { try {
SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'"); return OffsetDateTime.parse(textualUploadDate);
sdf.setTimeZone(TimeZone.getTimeZone("GMT")); } catch (DateTimeParseException e1) {
date = sdf.parse(textualUploadDate);
} catch (ParseException e1) {
try { try {
date = new SimpleDateFormat("yyyy/MM/dd HH:mm:ss +0000").parse(textualUploadDate); return OffsetDateTime.parse(textualUploadDate, DateTimeFormatter.ofPattern("yyyy/MM/dd HH:mm:ss +0000"));
} catch (ParseException e2) { } catch (DateTimeParseException e2) {
throw new ParsingException("Could not parse date: \"" + textualUploadDate + "\"" + ", " + e1.getMessage(), e2); throw new ParsingException("Could not parse date: \"" + textualUploadDate + "\"" + ", " + e1.getMessage(), e2);
} }
} }
final Calendar uploadDate = Calendar.getInstance();
uploadDate.setTime(date);
return uploadDate;
} }
/** /**

27
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java
View File

@ -5,7 +5,6 @@ import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser; import com.grack.nanojson.JsonParser;
import com.grack.nanojson.JsonParserException; import com.grack.nanojson.JsonParserException;
import com.grack.nanojson.JsonWriter; import com.grack.nanojson.JsonWriter;
import org.jsoup.Jsoup; import org.jsoup.Jsoup;
import org.jsoup.nodes.Document; import org.jsoup.nodes.Document;
import org.schabi.newpipe.extractor.downloader.Response; import org.schabi.newpipe.extractor.downloader.Response;
@ -22,13 +21,18 @@ import java.io.UnsupportedEncodingException;
import java.net.MalformedURLException; import java.net.MalformedURLException;
import java.net.URL; import java.net.URL;
import java.net.URLDecoder; import java.net.URLDecoder;
import java.text.ParseException; import java.time.OffsetDateTime;
import java.text.SimpleDateFormat; import java.time.format.DateTimeParseException;
import java.util.*; import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import static org.schabi.newpipe.extractor.NewPipe.getDownloader; import static org.schabi.newpipe.extractor.NewPipe.getDownloader;
import static org.schabi.newpipe.extractor.utils.JsonUtils.EMPTY_STRING; import static org.schabi.newpipe.extractor.utils.JsonUtils.EMPTY_STRING;
import static org.schabi.newpipe.extractor.utils.Utils.*; import static org.schabi.newpipe.extractor.utils.Utils.HTTP;
import static org.schabi.newpipe.extractor.utils.Utils.HTTPS;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
/* /*
* Created by Christian Schabesberger on 02.03.16. * Created by Christian Schabesberger on 02.03.16.
@ -176,19 +180,12 @@ public class YoutubeParsingHelper {
} }
} }
public static Calendar parseDateFrom(String textualUploadDate) throws ParsingException { public static OffsetDateTime parseDateFrom(String textualUploadDate) throws ParsingException {
final Date date;
try { try {
final SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd"); return OffsetDateTime.parse(textualUploadDate);
sdf.setTimeZone(TimeZone.getTimeZone("GMT")); } catch (DateTimeParseException e) {
date = sdf.parse(textualUploadDate);
} catch (ParseException e) {
throw new ParsingException("Could not parse date: \"" + textualUploadDate + "\"", e); throw new ParsingException("Could not parse date: \"" + textualUploadDate + "\"", e);
} }
final Calendar uploadDate = Calendar.getInstance();
uploadDate.setTime(date);
return uploadDate;
} }
public static JsonObject getInitialData(String html) throws ParsingException { public static JsonObject getInitialData(String html) throws ParsingException {

20
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeFeedInfoItemExtractor.java
View File

@ -7,11 +7,9 @@ import org.schabi.newpipe.extractor.stream.StreamInfoItemExtractor;
import org.schabi.newpipe.extractor.stream.StreamType; import org.schabi.newpipe.extractor.stream.StreamType;
import javax.annotation.Nullable; import javax.annotation.Nullable;
import java.text.ParseException; import java.time.OffsetDateTime;
import java.text.SimpleDateFormat; import java.time.format.DateTimeFormatter;
import java.util.Calendar; import java.time.format.DateTimeParseException;
import java.util.Date;
import java.util.TimeZone;
public class YoutubeFeedInfoItemExtractor implements StreamInfoItemExtractor { public class YoutubeFeedInfoItemExtractor implements StreamInfoItemExtractor {
private final Element entryElement; private final Element entryElement;
@ -62,19 +60,11 @@ public class YoutubeFeedInfoItemExtractor implements StreamInfoItemExtractor {
@Nullable @Nullable
@Override @Override
public DateWrapper getUploadDate() throws ParsingException { public DateWrapper getUploadDate() throws ParsingException {
final Date date;
try { try {
final SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss+00:00"); return new DateWrapper(OffsetDateTime.parse(getTextualUploadDate(), DateTimeFormatter.ISO_OFFSET_DATE_TIME));
dateFormat.setTimeZone(TimeZone.getTimeZone("UTC")); } catch (DateTimeParseException e) {
date = dateFormat.parse(getTextualUploadDate());
} catch (ParseException e) {
throw new ParsingException("Could not parse date (\"" + getTextualUploadDate() + "\")", e); throw new ParsingException("Could not parse date (\"" + getTextualUploadDate() + "\")", e);
} }
final Calendar calendar = Calendar.getInstance();
calendar.setTime(date);
return new DateWrapper(calendar);
} }
@Override @Override

21
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java
View File

@ -43,11 +43,11 @@ import javax.annotation.Nonnull;
import javax.annotation.Nullable; import javax.annotation.Nullable;
import java.io.IOException; import java.io.IOException;
import java.io.UnsupportedEncodingException; import java.io.UnsupportedEncodingException;
import java.text.SimpleDateFormat; import java.time.LocalDate;
import java.time.OffsetDateTime;
import java.time.format.DateTimeFormatter;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Calendar;
import java.util.Collections; import java.util.Collections;
import java.util.Date;
import java.util.HashMap; import java.util.HashMap;
import java.util.LinkedHashMap; import java.util.LinkedHashMap;
import java.util.List; import java.util.List;
@ -157,23 +157,24 @@ public class YoutubeStreamExtractor extends StreamExtractor {
try { // Premiered 20 hours ago try { // Premiered 20 hours ago
TimeAgoParser timeAgoParser = TimeAgoPatternsManager.getTimeAgoParserFor(Localization.fromLocalizationCode("en")); TimeAgoParser timeAgoParser = TimeAgoPatternsManager.getTimeAgoParserFor(Localization.fromLocalizationCode("en"));
Calendar parsedTime = timeAgoParser.parse(time).date(); OffsetDateTime parsedTime = timeAgoParser.parse(time).offsetDateTime();
return new SimpleDateFormat("yyyy-MM-dd").format(parsedTime.getTime()); return DateTimeFormatter.ISO_LOCAL_DATE.format(parsedTime);
} catch (Exception ignored) { } catch (Exception ignored) {
} }
try { // Premiered Feb 21, 2020 try { // Premiered Feb 21, 2020
Date d = new SimpleDateFormat("MMM dd, YYYY", Locale.ENGLISH).parse(time); LocalDate localDate = LocalDate.parse(time,
return new SimpleDateFormat("yyyy-MM-dd").format(d.getTime()); DateTimeFormatter.ofPattern("MMM dd, YYYY", Locale.ENGLISH));
return DateTimeFormatter.ISO_LOCAL_DATE.format(localDate);
} catch (Exception ignored) { } catch (Exception ignored) {
} }
} }
try { try {
// TODO: this parses English formatted dates only, we need a better approach to parse the textual date // TODO: this parses English formatted dates only, we need a better approach to parse the textual date
Date d = new SimpleDateFormat("dd MMM yyyy", Locale.ENGLISH).parse( LocalDate localDate = LocalDate.parse(getTextFromObject(getVideoPrimaryInfoRenderer().getObject("dateText")),
getTextFromObject(getVideoPrimaryInfoRenderer().getObject("dateText"))); DateTimeFormatter.ofPattern("dd MMM yyyy", Locale.ENGLISH));
return new SimpleDateFormat("yyyy-MM-dd").format(d); return DateTimeFormatter.ISO_LOCAL_DATE.format(localDate);
} catch (Exception ignored) { } catch (Exception ignored) {
} }
throw new ParsingException("Could not get upload date"); throw new ParsingException("Could not get upload date");

22
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java
View File

@ -12,11 +12,14 @@ import org.schabi.newpipe.extractor.stream.StreamType;
import org.schabi.newpipe.extractor.utils.Utils; import org.schabi.newpipe.extractor.utils.Utils;
import javax.annotation.Nullable; import javax.annotation.Nullable;
import java.text.SimpleDateFormat; import java.time.Instant;
import java.util.Calendar; import java.time.OffsetDateTime;
import java.util.Date; import java.time.ZoneOffset;
import java.time.format.DateTimeFormatter;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.*; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.fixThumbnailUrl;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getUrlFromNavigationEndpoint;
import static org.schabi.newpipe.extractor.utils.JsonUtils.EMPTY_STRING; import static org.schabi.newpipe.extractor.utils.JsonUtils.EMPTY_STRING;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
@ -165,8 +168,7 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor {
} }
if (isPremiere()) { if (isPremiere()) {
final Date date = getDateFromPremiere().getTime(); return DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm").format(getDateFromPremiere());
return new SimpleDateFormat("yyyy-MM-dd HH:mm").format(date);
} }
final String publishedTimeText = getTextFromObject(videoInfo.getObject("publishedTimeText")); final String publishedTimeText = getTextFromObject(videoInfo.getObject("publishedTimeText"));
@ -250,15 +252,13 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor {
return videoInfo.has("upcomingEventData"); return videoInfo.has("upcomingEventData");
} }
private Calendar getDateFromPremiere() throws ParsingException { private OffsetDateTime getDateFromPremiere() throws ParsingException {
final JsonObject upcomingEventData = videoInfo.getObject("upcomingEventData"); final JsonObject upcomingEventData = videoInfo.getObject("upcomingEventData");
final String startTime = upcomingEventData.getString("startTime"); final String startTime = upcomingEventData.getString("startTime");
try { try {
final long startTimeTimestamp = Long.parseLong(startTime); return OffsetDateTime.ofInstant(Instant.ofEpochSecond(Long.parseLong(startTime)),
final Calendar calendar = Calendar.getInstance(); ZoneOffset.UTC);
calendar.setTime(new Date(startTimeTimestamp * 1000L));
return calendar;
} catch (Exception e) { } catch (Exception e) {
throw new ParsingException("Could not parse date from premiere: \"" + startTime + "\""); throw new ParsingException("Could not parse date from premiere: \"" + startTime + "\"");
} }

18
extractor/src/test/java/org/schabi/newpipe/extractor/services/DefaultStreamExtractorTest.java
View File

@ -12,14 +12,12 @@ import org.schabi.newpipe.extractor.stream.StreamType;
import org.schabi.newpipe.extractor.stream.SubtitlesStream; import org.schabi.newpipe.extractor.stream.SubtitlesStream;
import org.schabi.newpipe.extractor.stream.VideoStream; import org.schabi.newpipe.extractor.stream.VideoStream;
import java.text.SimpleDateFormat; import javax.annotation.Nullable;
import java.util.Calendar; import java.time.LocalDateTime;
import java.time.format.DateTimeFormatter;
import java.util.Collections; import java.util.Collections;
import java.util.List; import java.util.List;
import java.util.Locale; import java.util.Locale;
import java.util.TimeZone;
import javax.annotation.Nullable;
import static org.hamcrest.CoreMatchers.containsString; import static org.hamcrest.CoreMatchers.containsString;
import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.MatcherAssert.assertThat;
@ -171,13 +169,11 @@ public abstract class DefaultStreamExtractorTest extends DefaultExtractorTest<St
} else { } else {
assertNotNull(dateWrapper); assertNotNull(dateWrapper);
final Calendar expectedDate = Calendar.getInstance(); final LocalDateTime expectedDateTime = LocalDateTime.parse(expectedUploadDate(),
final Calendar actualDate = dateWrapper.date(); DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss.SSS"));
final LocalDateTime actualDateTime = dateWrapper.offsetDateTime().toLocalDateTime();
final SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.S"); assertEquals(expectedDateTime, actualDateTime);
sdf.setTimeZone(TimeZone.getTimeZone("GMT"));
expectedDate.setTime(sdf.parse(expectedUploadDate()));
assertEquals(expectedDate, actualDate);
} }
} }

19
extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelLocalizationTest.java
View File

@ -6,12 +6,15 @@ import org.schabi.newpipe.DownloaderTestImpl;
import org.schabi.newpipe.extractor.ListExtractor; import org.schabi.newpipe.extractor.ListExtractor;
import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.channel.ChannelExtractor; import org.schabi.newpipe.extractor.channel.ChannelExtractor;
import org.schabi.newpipe.extractor.localization.Localization;
import org.schabi.newpipe.extractor.localization.DateWrapper; import org.schabi.newpipe.extractor.localization.DateWrapper;
import org.schabi.newpipe.extractor.localization.Localization;
import org.schabi.newpipe.extractor.stream.StreamInfoItem; import org.schabi.newpipe.extractor.stream.StreamInfoItem;
import java.text.SimpleDateFormat; import java.time.format.DateTimeFormatter;
import java.util.*; import java.time.temporal.ChronoUnit;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import static org.junit.Assert.fail; import static org.junit.Assert.fail;
import static org.schabi.newpipe.extractor.ServiceList.YouTube; import static org.schabi.newpipe.extractor.ServiceList.YouTube;
@ -23,7 +26,7 @@ import static org.schabi.newpipe.extractor.services.DefaultTests.defaultTestRela
@Ignore("Should be ran manually from time to time, as it's too time consuming.") @Ignore("Should be ran manually from time to time, as it's too time consuming.")
public class YoutubeChannelLocalizationTest { public class YoutubeChannelLocalizationTest {
private static final boolean DEBUG = true; private static final boolean DEBUG = true;
private final SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm"); private final DateTimeFormatter dateTimeFormatter = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm");
@Test @Test
public void testAllSupportedLocalizations() throws Exception { public void testAllSupportedLocalizations() throws Exception {
@ -64,7 +67,7 @@ public class YoutubeChannelLocalizationTest {
+ "\n:::: " + item.getStreamType() + ", views = " + item.getViewCount(); + "\n:::: " + item.getStreamType() + ", views = " + item.getViewCount();
final DateWrapper uploadDate = item.getUploadDate(); final DateWrapper uploadDate = item.getUploadDate();
if (uploadDate != null) { if (uploadDate != null) {
String dateAsText = dateFormat.format(uploadDate.date().getTime()); String dateAsText = dateTimeFormatter.format(uploadDate.offsetDateTime());
debugMessage += "\n:::: " + item.getTextualUploadDate() + debugMessage += "\n:::: " + item.getTextualUploadDate() +
"\n:::: " + dateAsText; "\n:::: " + dateAsText;
} }
@ -107,13 +110,13 @@ public class YoutubeChannelLocalizationTest {
final DateWrapper currentUploadDate = currentItem.getUploadDate(); final DateWrapper currentUploadDate = currentItem.getUploadDate();
final String referenceDateString = referenceUploadDate == null ? "null" : final String referenceDateString = referenceUploadDate == null ? "null" :
dateFormat.format(referenceUploadDate.date().getTime()); dateTimeFormatter.format(referenceUploadDate.offsetDateTime());
final String currentDateString = currentUploadDate == null ? "null" : final String currentDateString = currentUploadDate == null ? "null" :
dateFormat.format(currentUploadDate.date().getTime()); dateTimeFormatter.format(currentUploadDate.offsetDateTime());
long difference = -1; long difference = -1;
if (referenceUploadDate != null && currentUploadDate != null) { if (referenceUploadDate != null && currentUploadDate != null) {
difference = Math.abs(referenceUploadDate.date().getTimeInMillis() - currentUploadDate.date().getTimeInMillis()); difference = ChronoUnit.MILLIS.between(referenceUploadDate.offsetDateTime(), currentUploadDate.offsetDateTime());
} }
final boolean areTimeEquals = difference < 5 * 60 * 1000L; final boolean areTimeEquals = difference < 5 * 60 * 1000L;

32
timeago-parser/src/main/java/org/schabi/newpipe/extractor/timeago/PatternsHolder.java
View File

@ -1,5 +1,6 @@
package org.schabi.newpipe.extractor.timeago; package org.schabi.newpipe.extractor.timeago;
import java.time.temporal.ChronoUnit;
import java.util.Collection; import java.util.Collection;
import java.util.LinkedHashMap; import java.util.LinkedHashMap;
import java.util.Map; import java.util.Map;
@ -16,7 +17,7 @@ public abstract class PatternsHolder {
private final Collection<String> months; private final Collection<String> months;
private final Collection<String> years; private final Collection<String> years;
private final Map<TimeAgoUnit, Map<String, Integer>> specialCases = new LinkedHashMap<>(); private final Map<ChronoUnit, Map<String, Integer>> specialCases = new LinkedHashMap<>();
protected PatternsHolder(String wordSeparator, Collection<String> seconds, Collection<String> minutes, protected PatternsHolder(String wordSeparator, Collection<String> seconds, Collection<String> minutes,
Collection<String> hours, Collection<String> days, Collection<String> hours, Collection<String> days,
@ -69,30 +70,25 @@ public abstract class PatternsHolder {
return years; return years;
} }
public Map<TimeAgoUnit, Map<String, Integer>> specialCases() { public Map<ChronoUnit, Map<String, Integer>> specialCases() {
return specialCases; return specialCases;
} }
protected void putSpecialCase(TimeAgoUnit unit, String caseText, int caseAmount) { protected void putSpecialCase(ChronoUnit unit, String caseText, int caseAmount) {
Map<String, Integer> item = specialCases.get(unit); Map<String, Integer> item = specialCases.computeIfAbsent(unit, k -> new LinkedHashMap<>());
if (item == null) {
item = new LinkedHashMap<>();
specialCases.put(unit, item);
}
item.put(caseText, caseAmount); item.put(caseText, caseAmount);
} }
public Map<TimeAgoUnit, Collection<String>> asMap() { public Map<ChronoUnit, Collection<String>> asMap() {
final Map<TimeAgoUnit, Collection<String>> returnMap = new LinkedHashMap<>(); final Map<ChronoUnit, Collection<String>> returnMap = new LinkedHashMap<>();
returnMap.put(TimeAgoUnit.SECONDS, seconds()); returnMap.put(ChronoUnit.SECONDS, seconds());
returnMap.put(TimeAgoUnit.MINUTES, minutes()); returnMap.put(ChronoUnit.MINUTES, minutes());
returnMap.put(TimeAgoUnit.HOURS, hours()); returnMap.put(ChronoUnit.HOURS, hours());
returnMap.put(TimeAgoUnit.DAYS, days()); returnMap.put(ChronoUnit.DAYS, days());
returnMap.put(TimeAgoUnit.WEEKS, weeks()); returnMap.put(ChronoUnit.WEEKS, weeks());
returnMap.put(TimeAgoUnit.MONTHS, months()); returnMap.put(ChronoUnit.MONTHS, months());
returnMap.put(TimeAgoUnit.YEARS, years()); returnMap.put(ChronoUnit.YEARS, years());
return returnMap; return returnMap;
} }

11
timeago-parser/src/main/java/org/schabi/newpipe/extractor/timeago/TimeAgoUnit.java
View File

@ -1,11 +0,0 @@
package org.schabi.newpipe.extractor.timeago;
public enum TimeAgoUnit {
SECONDS,
MINUTES,
HOURS,
DAYS,
WEEKS,
MONTHS,
YEARS
}

13
timeago-parser/src/main/java/org/schabi/newpipe/extractor/timeago/patterns/iw.java
View File

@ -5,7 +5,8 @@
package org.schabi.newpipe.extractor.timeago.patterns; package org.schabi.newpipe.extractor.timeago.patterns;
import org.schabi.newpipe.extractor.timeago.PatternsHolder; import org.schabi.newpipe.extractor.timeago.PatternsHolder;
import org.schabi.newpipe.extractor.timeago.TimeAgoUnit;
import java.time.temporal.ChronoUnit;
public class iw extends PatternsHolder { public class iw extends PatternsHolder {
private static final String WORD_SEPARATOR = " "; private static final String WORD_SEPARATOR = " ";
@ -26,10 +27,10 @@ public class iw extends PatternsHolder {
private iw() { private iw() {
super(WORD_SEPARATOR, SECONDS, MINUTES, HOURS, DAYS, WEEKS, MONTHS, YEARS); super(WORD_SEPARATOR, SECONDS, MINUTES, HOURS, DAYS, WEEKS, MONTHS, YEARS);
putSpecialCase(TimeAgoUnit.HOURS, "שעתיים", 2); putSpecialCase(ChronoUnit.HOURS, "שעתיים", 2);
putSpecialCase(TimeAgoUnit.DAYS, "יומיים", 2); putSpecialCase(ChronoUnit.DAYS, "יומיים", 2);
putSpecialCase(TimeAgoUnit.WEEKS, "שבועיים", 2); putSpecialCase(ChronoUnit.WEEKS, "שבועיים", 2);
putSpecialCase(TimeAgoUnit.MONTHS, "חודשיים", 2); putSpecialCase(ChronoUnit.MONTHS, "חודשיים", 2);
putSpecialCase(TimeAgoUnit.YEARS, "שנתיים", 2); putSpecialCase(ChronoUnit.YEARS, "שנתיים", 2);
} }
} }