Switch from Calendar to OffsetDateTime in DateWrapper.

This commit is contained in:
Isira Seneviratne 2020-10-18 09:18:14 +05:30
parent ee3af63c04
commit 4f04cfccca
11 changed files with 130 additions and 166 deletions

View File

@ -3,30 +3,61 @@ package org.schabi.newpipe.extractor.localization;
import edu.umd.cs.findbugs.annotations.NonNull;
import java.io.Serializable;
import java.time.OffsetDateTime;
import java.time.ZoneOffset;
import java.util.Calendar;
import java.util.GregorianCalendar;
/**
* A wrapper class that provides a field to describe if the date is precise or just an approximation.
* A wrapper class that provides a field to describe if the date/time is precise or just an approximation.
*/
public class DateWrapper implements Serializable {
@NonNull private final Calendar date;
@NonNull private final OffsetDateTime offsetDateTime;
private final boolean isApproximation;
public DateWrapper(@NonNull Calendar date) {
this(date, false);
/**
* @deprecated Use {@link #DateWrapper(OffsetDateTime)} instead.
*/
@Deprecated
public DateWrapper(@NonNull Calendar calendar) {
this(calendar, false);
}
public DateWrapper(@NonNull Calendar date, boolean isApproximation) {
this.date = date;
/**
* @deprecated Use {@link #DateWrapper(OffsetDateTime, boolean)} instead.
*/
@Deprecated
public DateWrapper(@NonNull Calendar calendar, boolean isApproximation) {
offsetDateTime = OffsetDateTime.ofInstant(calendar.toInstant(), ZoneOffset.UTC);
this.isApproximation = isApproximation;
}
public DateWrapper(@NonNull OffsetDateTime offsetDateTime) {
this(offsetDateTime, false);
}
public DateWrapper(@NonNull OffsetDateTime offsetDateTime, boolean isApproximation) {
this.offsetDateTime = offsetDateTime.withOffsetSameInstant(ZoneOffset.UTC);
this.isApproximation = isApproximation;
}
/**
* @return the wrapped date.
* @return the wrapped date/time as a {@link Calendar}.
*
* @deprecated use {@link #offsetDateTime()} instead.
*/
@Deprecated
@NonNull
public Calendar date() {
return date;
return GregorianCalendar.from(offsetDateTime.toZonedDateTime());
}
/**
* @return the wrapped date/time.
*/
@NonNull
public OffsetDateTime offsetDateTime() {
return offsetDateTime;
}
/**

View File

@ -4,8 +4,9 @@ import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.timeago.PatternsHolder;
import org.schabi.newpipe.extractor.utils.Parser;
import java.time.OffsetDateTime;
import java.time.ZoneOffset;
import java.time.temporal.ChronoUnit;
import java.util.Calendar;
import java.util.Collection;
import java.util.Map;
import java.util.regex.Pattern;
@ -16,7 +17,7 @@ import java.util.regex.Pattern;
*/
public class TimeAgoParser {
private final PatternsHolder patternsHolder;
private final Calendar consistentNow;
private final OffsetDateTime now;
/**
* Creates a helper to parse upload dates in the format '2 days ago'.
@ -28,7 +29,7 @@ public class TimeAgoParser {
*/
public TimeAgoParser(PatternsHolder patternsHolder) {
this.patternsHolder = patternsHolder;
consistentNow = Calendar.getInstance();
now = OffsetDateTime.now(ZoneOffset.UTC);
}
/**
@ -113,64 +114,34 @@ public class TimeAgoParser {
}
private DateWrapper getResultFor(int timeAgoAmount, ChronoUnit chronoUnit) {
final Calendar calendarTime = getNow();
OffsetDateTime offsetDateTime = now;
boolean isApproximation = false;
switch (chronoUnit) {
case SECONDS:
calendarTime.add(Calendar.SECOND, -timeAgoAmount);
break;
case MINUTES:
calendarTime.add(Calendar.MINUTE, -timeAgoAmount);
break;
case HOURS:
calendarTime.add(Calendar.HOUR_OF_DAY, -timeAgoAmount);
offsetDateTime = offsetDateTime.minus(timeAgoAmount, chronoUnit);
break;
case DAYS:
calendarTime.add(Calendar.DAY_OF_MONTH, -timeAgoAmount);
isApproximation = true;
break;
case WEEKS:
calendarTime.add(Calendar.WEEK_OF_YEAR, -timeAgoAmount);
isApproximation = true;
break;
case MONTHS:
calendarTime.add(Calendar.MONTH, -timeAgoAmount);
offsetDateTime = offsetDateTime.minus(timeAgoAmount, chronoUnit);
isApproximation = true;
break;
case YEARS:
calendarTime.add(Calendar.YEAR, -timeAgoAmount);
// Prevent `PrettyTime` from showing '12 months ago'.
calendarTime.add(Calendar.DAY_OF_MONTH, -1);
// minusDays is needed to prevent `PrettyTime` from showing '12 months ago'.
offsetDateTime = offsetDateTime.minusYears(timeAgoAmount).minusDays(1);
isApproximation = true;
break;
}
if (isApproximation) {
markApproximatedTime(calendarTime);
offsetDateTime = offsetDateTime.truncatedTo(ChronoUnit.HOURS);
}
return new DateWrapper(calendarTime, isApproximation);
}
private Calendar getNow() {
return (Calendar) consistentNow.clone();
}
/**
* Marks the time as approximated by setting minutes, seconds and milliseconds to 0.
*
* @param calendarTime Time to be marked as approximated
*/
private void markApproximatedTime(Calendar calendarTime) {
calendarTime.set(Calendar.MINUTE, 0);
calendarTime.set(Calendar.SECOND, 0);
calendarTime.set(Calendar.MILLISECOND, 0);
return new DateWrapper(offsetDateTime, isApproximation);
}
}

View File

@ -2,28 +2,17 @@ package org.schabi.newpipe.extractor.services.media_ccc.extractors;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.Calendar;
import java.util.Date;
import java.util.TimeZone;
import java.time.OffsetDateTime;
import java.time.format.DateTimeParseException;
public final class MediaCCCParsingHelper {
private MediaCCCParsingHelper() { }
public static Calendar parseDateFrom(final String textualUploadDate) throws ParsingException {
final Date date;
public static OffsetDateTime parseDateFrom(final String textualUploadDate) throws ParsingException {
try {
final SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd");
sdf.setTimeZone(TimeZone.getTimeZone("GMT"));
date = sdf.parse(textualUploadDate);
} catch (ParseException e) {
return OffsetDateTime.parse(textualUploadDate);
} catch (DateTimeParseException e) {
throw new ParsingException("Could not parse date: \"" + textualUploadDate + "\"", e);
}
final Calendar uploadDate = Calendar.getInstance();
uploadDate.setTime(date);
return uploadDate;
}
}

View File

@ -2,7 +2,6 @@ package org.schabi.newpipe.extractor.services.peertube;
import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;
import org.schabi.newpipe.extractor.InfoItemsCollector;
import org.schabi.newpipe.extractor.Page;
import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException;
@ -12,11 +11,10 @@ import org.schabi.newpipe.extractor.utils.JsonUtils;
import org.schabi.newpipe.extractor.utils.Parser;
import org.schabi.newpipe.extractor.utils.Utils;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.Calendar;
import java.util.Date;
import java.util.TimeZone;
import java.time.Instant;
import java.time.OffsetDateTime;
import java.time.ZoneOffset;
import java.time.format.DateTimeParseException;
public class PeertubeParsingHelper {
public static final String START_KEY = "start";
@ -34,19 +32,12 @@ public class PeertubeParsingHelper {
}
}
public static Calendar parseDateFrom(final String textualUploadDate) throws ParsingException {
final Date date;
public static OffsetDateTime parseDateFrom(final String textualUploadDate) throws ParsingException {
try {
final SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.S'Z'");
sdf.setTimeZone(TimeZone.getTimeZone("GMT"));
date = sdf.parse(textualUploadDate);
} catch (ParseException e) {
return OffsetDateTime.ofInstant(Instant.parse(textualUploadDate), ZoneOffset.UTC);
} catch (DateTimeParseException e) {
throw new ParsingException("Could not parse date: \"" + textualUploadDate + "\"", e);
}
final Calendar uploadDate = Calendar.getInstance();
uploadDate.setTime(date);
return uploadDate;
}
public static Page getNextPage(final String prevPageUrl, final long total) {

View File

@ -21,16 +21,18 @@ import org.schabi.newpipe.extractor.services.soundcloud.extractors.SoundcloudStr
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
import org.schabi.newpipe.extractor.utils.Parser;
import org.schabi.newpipe.extractor.utils.Parser.RegexException;
import org.schabi.newpipe.extractor.utils.Utils;
import javax.annotation.Nonnull;
import java.io.IOException;
import java.net.MalformedURLException;
import java.net.URL;
import java.net.URLEncoder;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.*;
import java.time.OffsetDateTime;
import java.time.format.DateTimeFormatter;
import java.time.format.DateTimeParseException;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import static java.util.Collections.singletonList;
import static org.schabi.newpipe.extractor.ServiceList.SoundCloud;
@ -95,23 +97,16 @@ public class SoundcloudParsingHelper {
}
}
public static Calendar parseDateFrom(String textualUploadDate) throws ParsingException {
Date date;
public static OffsetDateTime parseDateFrom(String textualUploadDate) throws ParsingException {
try {
SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'");
sdf.setTimeZone(TimeZone.getTimeZone("GMT"));
date = sdf.parse(textualUploadDate);
} catch (ParseException e1) {
return OffsetDateTime.parse(textualUploadDate);
} catch (DateTimeParseException e1) {
try {
date = new SimpleDateFormat("yyyy/MM/dd HH:mm:ss +0000").parse(textualUploadDate);
} catch (ParseException e2) {
return OffsetDateTime.parse(textualUploadDate, DateTimeFormatter.ofPattern("yyyy/MM/dd HH:mm:ss +0000"));
} catch (DateTimeParseException e2) {
throw new ParsingException("Could not parse date: \"" + textualUploadDate + "\"" + ", " + e1.getMessage(), e2);
}
}
final Calendar uploadDate = Calendar.getInstance();
uploadDate.setTime(date);
return uploadDate;
}
/**

View File

@ -5,7 +5,6 @@ import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser;
import com.grack.nanojson.JsonParserException;
import com.grack.nanojson.JsonWriter;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.schabi.newpipe.extractor.downloader.Response;
@ -22,13 +21,18 @@ import java.io.UnsupportedEncodingException;
import java.net.MalformedURLException;
import java.net.URL;
import java.net.URLDecoder;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.*;
import java.time.OffsetDateTime;
import java.time.format.DateTimeParseException;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import static org.schabi.newpipe.extractor.NewPipe.getDownloader;
import static org.schabi.newpipe.extractor.utils.JsonUtils.EMPTY_STRING;
import static org.schabi.newpipe.extractor.utils.Utils.*;
import static org.schabi.newpipe.extractor.utils.Utils.HTTP;
import static org.schabi.newpipe.extractor.utils.Utils.HTTPS;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
/*
* Created by Christian Schabesberger on 02.03.16.
@ -176,19 +180,12 @@ public class YoutubeParsingHelper {
}
}
public static Calendar parseDateFrom(String textualUploadDate) throws ParsingException {
final Date date;
public static OffsetDateTime parseDateFrom(String textualUploadDate) throws ParsingException {
try {
final SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd");
sdf.setTimeZone(TimeZone.getTimeZone("GMT"));
date = sdf.parse(textualUploadDate);
} catch (ParseException e) {
return OffsetDateTime.parse(textualUploadDate);
} catch (DateTimeParseException e) {
throw new ParsingException("Could not parse date: \"" + textualUploadDate + "\"", e);
}
final Calendar uploadDate = Calendar.getInstance();
uploadDate.setTime(date);
return uploadDate;
}
public static JsonObject getInitialData(String html) throws ParsingException {

View File

@ -7,11 +7,9 @@ import org.schabi.newpipe.extractor.stream.StreamInfoItemExtractor;
import org.schabi.newpipe.extractor.stream.StreamType;
import javax.annotation.Nullable;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.Calendar;
import java.util.Date;
import java.util.TimeZone;
import java.time.OffsetDateTime;
import java.time.format.DateTimeFormatter;
import java.time.format.DateTimeParseException;
public class YoutubeFeedInfoItemExtractor implements StreamInfoItemExtractor {
private final Element entryElement;
@ -62,19 +60,11 @@ public class YoutubeFeedInfoItemExtractor implements StreamInfoItemExtractor {
@Nullable
@Override
public DateWrapper getUploadDate() throws ParsingException {
final Date date;
try {
final SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss+00:00");
dateFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
date = dateFormat.parse(getTextualUploadDate());
} catch (ParseException e) {
return new DateWrapper(OffsetDateTime.parse(getTextualUploadDate(), DateTimeFormatter.ISO_OFFSET_DATE_TIME));
} catch (DateTimeParseException e) {
throw new ParsingException("Could not parse date (\"" + getTextualUploadDate() + "\")", e);
}
final Calendar calendar = Calendar.getInstance();
calendar.setTime(date);
return new DateWrapper(calendar);
}
@Override

View File

@ -43,11 +43,11 @@ import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.text.SimpleDateFormat;
import java.time.LocalDate;
import java.time.OffsetDateTime;
import java.time.format.DateTimeFormatter;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.Collections;
import java.util.Date;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
@ -157,23 +157,24 @@ public class YoutubeStreamExtractor extends StreamExtractor {
try { // Premiered 20 hours ago
TimeAgoParser timeAgoParser = TimeAgoPatternsManager.getTimeAgoParserFor(Localization.fromLocalizationCode("en"));
Calendar parsedTime = timeAgoParser.parse(time).date();
return new SimpleDateFormat("yyyy-MM-dd").format(parsedTime.getTime());
OffsetDateTime parsedTime = timeAgoParser.parse(time).offsetDateTime();
return DateTimeFormatter.ISO_LOCAL_DATE.format(parsedTime);
} catch (Exception ignored) {
}
try { // Premiered Feb 21, 2020
Date d = new SimpleDateFormat("MMM dd, YYYY", Locale.ENGLISH).parse(time);
return new SimpleDateFormat("yyyy-MM-dd").format(d.getTime());
LocalDate localDate = LocalDate.parse(time,
DateTimeFormatter.ofPattern("MMM dd, YYYY", Locale.ENGLISH));
return DateTimeFormatter.ISO_LOCAL_DATE.format(localDate);
} catch (Exception ignored) {
}
}
try {
// TODO: this parses English formatted dates only, we need a better approach to parse the textual date
Date d = new SimpleDateFormat("dd MMM yyyy", Locale.ENGLISH).parse(
getTextFromObject(getVideoPrimaryInfoRenderer().getObject("dateText")));
return new SimpleDateFormat("yyyy-MM-dd").format(d);
LocalDate localDate = LocalDate.parse(getTextFromObject(getVideoPrimaryInfoRenderer().getObject("dateText")),
DateTimeFormatter.ofPattern("dd MMM yyyy", Locale.ENGLISH));
return DateTimeFormatter.ISO_LOCAL_DATE.format(localDate);
} catch (Exception ignored) {
}
throw new ParsingException("Could not get upload date");

View File

@ -12,11 +12,14 @@ import org.schabi.newpipe.extractor.stream.StreamType;
import org.schabi.newpipe.extractor.utils.Utils;
import javax.annotation.Nullable;
import java.text.SimpleDateFormat;
import java.util.Calendar;
import java.util.Date;
import java.time.Instant;
import java.time.OffsetDateTime;
import java.time.ZoneOffset;
import java.time.format.DateTimeFormatter;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.*;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.fixThumbnailUrl;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getUrlFromNavigationEndpoint;
import static org.schabi.newpipe.extractor.utils.JsonUtils.EMPTY_STRING;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
@ -165,8 +168,7 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor {
}
if (isPremiere()) {
final Date date = getDateFromPremiere().getTime();
return new SimpleDateFormat("yyyy-MM-dd HH:mm").format(date);
return DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm").format(getDateFromPremiere());
}
final String publishedTimeText = getTextFromObject(videoInfo.getObject("publishedTimeText"));
@ -250,15 +252,13 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor {
return videoInfo.has("upcomingEventData");
}
private Calendar getDateFromPremiere() throws ParsingException {
private OffsetDateTime getDateFromPremiere() throws ParsingException {
final JsonObject upcomingEventData = videoInfo.getObject("upcomingEventData");
final String startTime = upcomingEventData.getString("startTime");
try {
final long startTimeTimestamp = Long.parseLong(startTime);
final Calendar calendar = Calendar.getInstance();
calendar.setTime(new Date(startTimeTimestamp * 1000L));
return calendar;
return OffsetDateTime.ofInstant(Instant.ofEpochSecond(Long.parseLong(startTime)),
ZoneOffset.UTC);
} catch (Exception e) {
throw new ParsingException("Could not parse date from premiere: \"" + startTime + "\"");
}

View File

@ -12,14 +12,12 @@ import org.schabi.newpipe.extractor.stream.StreamType;
import org.schabi.newpipe.extractor.stream.SubtitlesStream;
import org.schabi.newpipe.extractor.stream.VideoStream;
import java.text.SimpleDateFormat;
import java.util.Calendar;
import javax.annotation.Nullable;
import java.time.LocalDateTime;
import java.time.format.DateTimeFormatter;
import java.util.Collections;
import java.util.List;
import java.util.Locale;
import java.util.TimeZone;
import javax.annotation.Nullable;
import static org.hamcrest.CoreMatchers.containsString;
import static org.hamcrest.MatcherAssert.assertThat;
@ -171,13 +169,11 @@ public abstract class DefaultStreamExtractorTest extends DefaultExtractorTest<St
} else {
assertNotNull(dateWrapper);
final Calendar expectedDate = Calendar.getInstance();
final Calendar actualDate = dateWrapper.date();
final LocalDateTime expectedDateTime = LocalDateTime.parse(expectedUploadDate(),
DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss.SSS"));
final LocalDateTime actualDateTime = dateWrapper.offsetDateTime().toLocalDateTime();
final SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.S");
sdf.setTimeZone(TimeZone.getTimeZone("GMT"));
expectedDate.setTime(sdf.parse(expectedUploadDate()));
assertEquals(expectedDate, actualDate);
assertEquals(expectedDateTime, actualDateTime);
}
}

View File

@ -6,12 +6,15 @@ import org.schabi.newpipe.DownloaderTestImpl;
import org.schabi.newpipe.extractor.ListExtractor;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.channel.ChannelExtractor;
import org.schabi.newpipe.extractor.localization.Localization;
import org.schabi.newpipe.extractor.localization.DateWrapper;
import org.schabi.newpipe.extractor.localization.Localization;
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
import java.text.SimpleDateFormat;
import java.util.*;
import java.time.format.DateTimeFormatter;
import java.time.temporal.ChronoUnit;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import static org.junit.Assert.fail;
import static org.schabi.newpipe.extractor.ServiceList.YouTube;
@ -23,7 +26,7 @@ import static org.schabi.newpipe.extractor.services.DefaultTests.defaultTestRela
@Ignore("Should be ran manually from time to time, as it's too time consuming.")
public class YoutubeChannelLocalizationTest {
private static final boolean DEBUG = true;
private final SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm");
private final DateTimeFormatter dateTimeFormatter = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm");
@Test
public void testAllSupportedLocalizations() throws Exception {
@ -64,7 +67,7 @@ public class YoutubeChannelLocalizationTest {
+ "\n:::: " + item.getStreamType() + ", views = " + item.getViewCount();
final DateWrapper uploadDate = item.getUploadDate();
if (uploadDate != null) {
String dateAsText = dateFormat.format(uploadDate.date().getTime());
String dateAsText = dateTimeFormatter.format(uploadDate.offsetDateTime());
debugMessage += "\n:::: " + item.getTextualUploadDate() +
"\n:::: " + dateAsText;
}
@ -107,13 +110,13 @@ public class YoutubeChannelLocalizationTest {
final DateWrapper currentUploadDate = currentItem.getUploadDate();
final String referenceDateString = referenceUploadDate == null ? "null" :
dateFormat.format(referenceUploadDate.date().getTime());
dateTimeFormatter.format(referenceUploadDate.offsetDateTime());
final String currentDateString = currentUploadDate == null ? "null" :
dateFormat.format(currentUploadDate.date().getTime());
dateTimeFormatter.format(currentUploadDate.offsetDateTime());
long difference = -1;
if (referenceUploadDate != null && currentUploadDate != null) {
difference = Math.abs(referenceUploadDate.date().getTimeInMillis() - currentUploadDate.date().getTimeInMillis());
difference = ChronoUnit.MILLIS.between(referenceUploadDate.offsetDateTime(), currentUploadDate.offsetDateTime());
}
final boolean areTimeEquals = difference < 5 * 60 * 1000L;