be-gay-do-crimes
/
NewPipeExtractor
mirror of https://github.com/TeamNewPipe/NewPipeExtractor
1
0
Fork 0
Code Issues Releases Wiki Activity

[YouTube] Fix parsing short relative date formats (English only) (#1068)

This commit is contained in:
ThetaDev 2023-06-18 21:41:29 +02:00 committed by GitHub
parent d294ccb433
commit ad97f08048
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 195 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
extractor/src/main/java/org/schabi/newpipe/extractor/localization/TimeAgoParser.java
View File

@ -93,7 +93,8 @@ public class TimeAgoParser {
final String escapedSeparator = patternsHolder.wordSeparator().equals(" ")
// From JDK8 → \h - Treat horizontal spaces as a normal one
// (non-breaking space, thin space, etc.)
? "[ \\t\\xA0\\u1680\\u180e\\u2000-\\u200a\\u202f\\u205f\\u3000]"
// Also split the string on numbers to be able to parse strings like "2wk"
? "[ \\t\\xA0\\u1680\\u180e\\u2000-\\u200a\\u202f\\u205f\\u3000\\d]"
: Pattern.quote(patternsHolder.wordSeparator());
// (^|separator)pattern($|separator)

154
extractor/src/test/java/org/schabi/newpipe/extractor/utils/TimeagoTest.java Normal file
View File

@ -0,0 +1,154 @@
package org.schabi.newpipe.extractor.utils;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.localization.Localization;
import org.schabi.newpipe.extractor.localization.TimeAgoParser;
import org.schabi.newpipe.extractor.localization.TimeAgoPatternsManager;
import java.time.OffsetDateTime;
import java.time.ZoneOffset;
import java.time.temporal.ChronoUnit;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertTrue;
public class TimeagoTest {
private static TimeAgoParser parser;
private static OffsetDateTime now;
@BeforeAll
public static void setUp() {
parser = TimeAgoPatternsManager.getTimeAgoParserFor(Localization.DEFAULT);
now = OffsetDateTime.now(ZoneOffset.UTC);
}
@Test
void parseTimeago() throws ParsingException {
assertTimeWithin1s(
now.minus(1, ChronoUnit.SECONDS),
parser.parse("1 second ago").offsetDateTime()
);
assertTimeWithin1s(
now.minus(12, ChronoUnit.SECONDS),
parser.parse("12 second ago").offsetDateTime()
);
assertTimeWithin1s(
now.minus(1, ChronoUnit.MINUTES),
parser.parse("1 minute ago").offsetDateTime()
);
assertTimeWithin1s(
now.minus(23, ChronoUnit.MINUTES),
parser.parse("23 minutes ago").offsetDateTime()
);
assertTimeWithin1s(
now.minus(1, ChronoUnit.HOURS),
parser.parse("1 hour ago").offsetDateTime()
);
assertTimeWithin1s(
now.minus(8, ChronoUnit.HOURS),
parser.parse("8 hours ago").offsetDateTime()
);
assertEquals(
now.minus(1, ChronoUnit.DAYS).truncatedTo(ChronoUnit.HOURS),
parser.parse("1 day ago").offsetDateTime()
);
assertEquals(
now.minus(3, ChronoUnit.DAYS).truncatedTo(ChronoUnit.HOURS),
parser.parse("3 days ago").offsetDateTime()
);
assertEquals(
now.minus(1, ChronoUnit.WEEKS).truncatedTo(ChronoUnit.HOURS),
parser.parse("1 week ago").offsetDateTime()
);
assertEquals(
now.minus(3, ChronoUnit.WEEKS).truncatedTo(ChronoUnit.HOURS),
parser.parse("3 weeks ago").offsetDateTime()
);
assertEquals(
now.minus(1, ChronoUnit.MONTHS).truncatedTo(ChronoUnit.HOURS),
parser.parse("1 month ago").offsetDateTime()
);
assertEquals(
now.minus(3, ChronoUnit.MONTHS).truncatedTo(ChronoUnit.HOURS),
parser.parse("3 months ago").offsetDateTime()
);
assertEquals(
now.minus(1, ChronoUnit.YEARS).minusDays(1).truncatedTo(ChronoUnit.HOURS),
parser.parse("1 year ago").offsetDateTime()
);
assertEquals(
now.minus(3, ChronoUnit.YEARS).minusDays(1).truncatedTo(ChronoUnit.HOURS),
parser.parse("3 years ago").offsetDateTime()
);
}
@Test
void parseTimeagoShort() throws ParsingException {
final TimeAgoParser parser = TimeAgoPatternsManager.getTimeAgoParserFor(Localization.DEFAULT);
final OffsetDateTime now = OffsetDateTime.now(ZoneOffset.UTC);
assertTimeWithin1s(
now.minus(1, ChronoUnit.SECONDS),
parser.parse("1 sec ago").offsetDateTime()
);
assertTimeWithin1s(
now.minus(12, ChronoUnit.SECONDS),
parser.parse("12 sec ago").offsetDateTime()
);
assertTimeWithin1s(
now.minus(1, ChronoUnit.MINUTES),
parser.parse("1 min ago").offsetDateTime()
);
assertTimeWithin1s(
now.minus(23, ChronoUnit.MINUTES),
parser.parse("23 min ago").offsetDateTime()
);
assertTimeWithin1s(
now.minus(1, ChronoUnit.HOURS),
parser.parse("1 hr ago").offsetDateTime()
);
assertTimeWithin1s(
now.minus(8, ChronoUnit.HOURS),
parser.parse("8 hr ago").offsetDateTime()
);
assertEquals(
now.minus(1, ChronoUnit.DAYS).truncatedTo(ChronoUnit.HOURS),
parser.parse("1 day ago").offsetDateTime()
);
assertEquals(
now.minus(3, ChronoUnit.DAYS).truncatedTo(ChronoUnit.HOURS),
parser.parse("3 days ago").offsetDateTime()
);
assertEquals(
now.minus(1, ChronoUnit.WEEKS).truncatedTo(ChronoUnit.HOURS),
parser.parse("1 wk ago").offsetDateTime()
);
assertEquals(
now.minus(3, ChronoUnit.WEEKS).truncatedTo(ChronoUnit.HOURS),
parser.parse("3 wk ago").offsetDateTime()
);
assertEquals(
now.minus(1, ChronoUnit.MONTHS).truncatedTo(ChronoUnit.HOURS),
parser.parse("1 mo ago").offsetDateTime()
);
assertEquals(
now.minus(3, ChronoUnit.MONTHS).truncatedTo(ChronoUnit.HOURS),
parser.parse("3 mo ago").offsetDateTime()
);
assertEquals(
now.minus(1, ChronoUnit.YEARS).minusDays(1).truncatedTo(ChronoUnit.HOURS),
parser.parse("1 yr ago").offsetDateTime()
);
assertEquals(
now.minus(3, ChronoUnit.YEARS).minusDays(1).truncatedTo(ChronoUnit.HOURS),
parser.parse("3 yr ago").offsetDateTime()
);
}
void assertTimeWithin1s(final OffsetDateTime expected, final OffsetDateTime actual) {
final long delta = Math.abs(expected.toEpochSecond() - actual.toEpochSecond());
assertTrue(delta <= 1, String.format("Expected: %s\nActual: %s", expected, actual));
}
}

39
timeago-parser/raw/unique_patterns.json
View File

@ -415,46 +415,56 @@
"word_separator": " ",
"seconds": [
"second",
"seconds"
"seconds",
"sec"
],
"minutes": [
"minute",
"minutes"
"minutes",
"min"
],
"hours": [
"hour",
"hours"
"hours",
"h"
],
"days": [
"day",
"days"
"days",
"d"
],
"weeks": [
"week",
"weeks"
"weeks",
"w"
],
"months": [
"month",
"months"
"months",
"mo"
],
"years": [
"year",
"years"
"years",
"y"
]
},
"en-GB": {
"word_separator": " ",
"seconds": [
"second",
"seconds"
"seconds",
"sec"
],
"minutes": [
"minute",
"minutes"
"minutes",
"min"
],
"hours": [
"hour",
"hours"
"hours",
"hr"
],
"days": [
"day",
@ -462,15 +472,18 @@
],
"weeks": [
"week",
"weeks"
"weeks",
"wk"
],
"months": [
"month",
"months"
"months",
"mo"
],
"years": [
"year",
"years"
"years",
"yr"
]
},
"es": {

14
timeago-parser/src/main/java/org/schabi/newpipe/extractor/timeago/patterns/en.java
View File

@ -9,13 +9,13 @@ import org.schabi.newpipe.extractor.timeago.PatternsHolder;
public class en extends PatternsHolder {
private static final String WORD_SEPARATOR = " ";
private static final String[]
SECONDS /**/ = {"second", "seconds"},
MINUTES /**/ = {"minute", "minutes"},
HOURS /**/ = {"hour", "hours"},
DAYS /**/ = {"day", "days"},
WEEKS /**/ = {"week", "weeks"},
MONTHS /**/ = {"month", "months"},
YEARS /**/ = {"year", "years"};
SECONDS /**/ = {"second", "seconds", "sec"},
MINUTES /**/ = {"minute", "minutes", "min"},
HOURS /**/ = {"hour", "hours", "h"},
DAYS /**/ = {"day", "days", "d"},
WEEKS /**/ = {"week", "weeks", "w"},
MONTHS /**/ = {"month", "months", "mo"},
YEARS /**/ = {"year", "years", "y"};
private static final en INSTANCE = new en();

12
timeago-parser/src/main/java/org/schabi/newpipe/extractor/timeago/patterns/en_GB.java
View File

@ -9,13 +9,13 @@ import org.schabi.newpipe.extractor.timeago.PatternsHolder;
public class en_GB extends PatternsHolder {
private static final String WORD_SEPARATOR = " ";
private static final String[]
SECONDS /**/ = {"second", "seconds"},
MINUTES /**/ = {"minute", "minutes"},
HOURS /**/ = {"hour", "hours"},
SECONDS /**/ = {"second", "seconds", "sec"},
MINUTES /**/ = {"minute", "minutes", "min"},
HOURS /**/ = {"hour", "hours", "hr"},
DAYS /**/ = {"day", "days"},
WEEKS /**/ = {"week", "weeks"},
MONTHS /**/ = {"month", "months"},
YEARS /**/ = {"year", "years"};
WEEKS /**/ = {"week", "weeks", "wk"},
MONTHS /**/ = {"month", "months", "mo"},
YEARS /**/ = {"year", "years", "yr"};
private static final en_GB INSTANCE = new en_GB();