From 25ed72740bf39748a37d4b661102c06c60c9efaa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Mazur?= Date: Mon, 1 Apr 2024 16:46:26 +0200 Subject: [PATCH] Fixed deserialization Added auto retry for Readwise WIP: fix 429 error Updated browser plugins --- README.md | 2 +- build.gradle | 13 ++++++---- src/frontend/package-lock.json | 10 +++++--- .../reader/api/ReaderApiService.java | 24 +++++++------------ .../pocketstats/reader/api/ReaderItem.java | 15 ++++++------ .../reader/api/ReadwiseFetchParams.java | 9 +------ .../cybershu/pocketstats/reader/api/Tag.java | 13 ++++++++++ .../utils/InstantNanoSecondsConverter.java | 12 ++++++++++ src/main/resources/application.yaml | 10 +++++++- .../reader/api/ReaderApiServiceTest.groovy | 13 ++++++++-- 10 files changed, 79 insertions(+), 42 deletions(-) create mode 100644 src/main/java/eu/cybershu/pocketstats/reader/api/Tag.java create mode 100644 src/main/java/eu/cybershu/pocketstats/utils/InstantNanoSecondsConverter.java diff --git a/README.md b/README.md index 0721c93..3908a1d 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@ App for visualising [GetPocket](https://getpocket.com/) and [Reader](https://rea #### Environment variables - `BACKEND_URL` - backend url, default: http://localhost:8080 - `POCKET_CONSUMER_KEY` - get pocket consumer key -- `READER_ACCESS_KEY` - readwise access key +- `READER_ACCESS_TOKEN` - readwise access key - `MONGODB_HOST` - mongo db host - `MONGODB_URI` - full mongodb uri with login and pass diff --git a/build.gradle b/build.gradle index d339beb..12a014f 100644 --- a/build.gradle +++ b/build.gradle @@ -29,6 +29,7 @@ repositories { ext { mapstructVersion = "1.5.3.Final" + jacksonVersion = "2.17.0" lombokVersion = "1.18.20" lombokMapstructBindingVersion = "0.2.0" } @@ -38,10 +39,12 @@ dependencies { implementation 'org.springframework.boot:spring-boot-starter-web' implementation 'commons-validator:commons-validator:1.8.0' developmentOnly 'org.springframework.boot:spring-boot-devtools' - implementation 'com.fasterxml.jackson.core:jackson-databind:2.17.0' - implementation 'com.fasterxml.jackson.datatype:jackson-datatype-jsr310:2.17.0' - implementation group: 'com.fasterxml.jackson.dataformat', name: 'jackson-dataformat-xml', version: '2.17.0' - implementation group: 'com.fasterxml.jackson.core', name: 'jackson-databind', version: '2.17.0' + + implementation("com.fasterxml.jackson:jackson-bom:${jacksonVersion}") + implementation("com.fasterxml.jackson.core:jackson-core:${jacksonVersion}") + implementation("com.fasterxml.jackson.core:jackson-databind:${jacksonVersion}") + implementation("com.fasterxml.jackson.datatype:jackson-datatype-jsr310:${jacksonVersion}") + implementation 'org.springframework.boot:spring-boot-starter-data-mongodb' implementation group: 'javax.validation', name: 'validation-api', version: '2.0.1.Final' implementation 'com.google.code.findbugs:jsr305:3.0.2' @@ -56,6 +59,8 @@ dependencies { "org.projectlombok:lombok-mapstruct-binding:${lombokMapstructBindingVersion}" implementation 'com.google.guava:guava:31.1-jre' + implementation 'io.github.resilience4j:resilience4j-spring-boot3:2.2.0' + // Tests testImplementation('org.springframework.boot:spring-boot-starter-test') { exclude group: 'org.junit.vintage', module: 'junit-vintage-engine' diff --git a/src/frontend/package-lock.json b/src/frontend/package-lock.json index 1113d9f..30fce91 100644 --- a/src/frontend/package-lock.json +++ b/src/frontend/package-lock.json @@ -4719,9 +4719,9 @@ } }, "node_modules/caniuse-lite": { - "version": "1.0.30001457", - "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001457.tgz", - "integrity": "sha512-SDIV6bgE1aVbK6XyxdURbUE89zY7+k1BBBaOwYwkNCglXlel/E7mELiHC64HQ+W0xSKlqWhV9Wh7iHxUjMs4fA==", + "version": "1.0.30001603", + "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001603.tgz", + "integrity": "sha512-iL2iSS0eDILMb9n5yKQoTBim9jMZ0Yrk8g0N9K7UzYyWnfIKzXBZD5ngpM37ZcL/cv0Mli8XtVMRYMQAfFpi5Q==", "funding": [ { "type": "opencollective", @@ -4730,6 +4730,10 @@ { "type": "tidelift", "url": "https://tidelift.com/funding/github/npm/caniuse-lite" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" } ] }, diff --git a/src/main/java/eu/cybershu/pocketstats/reader/api/ReaderApiService.java b/src/main/java/eu/cybershu/pocketstats/reader/api/ReaderApiService.java index 76e78b5..085122a 100644 --- a/src/main/java/eu/cybershu/pocketstats/reader/api/ReaderApiService.java +++ b/src/main/java/eu/cybershu/pocketstats/reader/api/ReaderApiService.java @@ -1,16 +1,10 @@ package eu.cybershu.pocketstats.reader.api; -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.databind.DeserializationFeature; import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.SerializationFeature; -import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; -import eu.cybershu.pocketstats.pocket.api.ApiXHeaders; -import eu.cybershu.pocketstats.utils.RequestUtils; +import io.github.resilience4j.ratelimiter.annotation.RateLimiter; import lombok.extern.slf4j.Slf4j; import org.springframework.http.HttpStatus; import org.springframework.stereotype.Service; -import org.springframework.util.StringUtils; import java.io.IOException; import java.net.URI; @@ -20,7 +14,6 @@ import java.time.Duration; import java.util.LinkedList; import java.util.List; -import java.util.Map; /** * Service for interacting with the Readwise Reader API: https://readwise.io/reader_api @@ -40,12 +33,10 @@ public ReaderApiService() { .followRedirects(HttpClient.Redirect.NORMAL) .connectTimeout(Duration.ofSeconds(60)) .build(); - this.mapper = new ObjectMapper(); - this.mapper - .registerModule(new JavaTimeModule()) - .configure(SerializationFeature.WRITE_DATE_TIMESTAMPS_AS_NANOSECONDS, false) - .configure(DeserializationFeature.READ_DATE_TIMESTAMPS_AS_NANOSECONDS, false) - .setSerializationInclusion(JsonInclude.Include.NON_NULL); + this.mapper = new ObjectMapper().findAndRegisterModules(); +// this.mapper.disable(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS) +// .disable(DeserializationFeature.READ_DATE_TIMESTAMPS_AS_NANOSECONDS) +// } /** @@ -75,6 +66,7 @@ public List fetchList(String accessToken, ReadwiseFetchParams params .builder() .category(params.category()) .location(params.location()) + .updatedAfter(params.updatedAfter()) .pageCursor(response.nextPageCursor()) .build(); } while (pageParams.pageCursor() != null); @@ -82,6 +74,7 @@ public List fetchList(String accessToken, ReadwiseFetchParams params return items; } + @RateLimiter(name = "readwise-api") private ReaderListResponse fetchPage(String accessToken, ReadwiseFetchPaginationParams params) throws IOException, InterruptedException { log.info("Fetching Readwise list with params: {}", params); @@ -102,7 +95,8 @@ private ReaderListResponse fetchPage(String accessToken, ReadwiseFetchPagination case 200: String body = response.body(); - return mapper.readValue(body, ReaderListResponse.class); + ReaderListResponse results = mapper.readValue(body, ReaderListResponse.class); + return results; case 401: break; case 400: diff --git a/src/main/java/eu/cybershu/pocketstats/reader/api/ReaderItem.java b/src/main/java/eu/cybershu/pocketstats/reader/api/ReaderItem.java index caf1914..d1e8327 100644 --- a/src/main/java/eu/cybershu/pocketstats/reader/api/ReaderItem.java +++ b/src/main/java/eu/cybershu/pocketstats/reader/api/ReaderItem.java @@ -1,14 +1,13 @@ package eu.cybershu.pocketstats.reader.api; -import com.fasterxml.jackson.annotation.JsonFormat; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; -import lombok.AllArgsConstructor; -import lombok.Data; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import eu.cybershu.pocketstats.utils.InstantNanoSecondsConverter; import java.time.Instant; -import java.util.List; +import java.util.Map; /** { @@ -44,15 +43,15 @@ public record ReaderItem( @JsonProperty String source, @JsonProperty Category category, @JsonProperty Location location, - //@JsonProperty List tags, + @JsonProperty Map tags, @JsonProperty("site_name") String siteName, @JsonProperty("word_count") int wordCount, @JsonProperty - @JsonFormat(shape = JsonFormat.Shape.NUMBER, without = JsonFormat.Feature.WRITE_DATE_TIMESTAMPS_AS_NANOSECONDS) Instant created_at, + @JsonDeserialize(converter = InstantNanoSecondsConverter.class) Instant created_at, @JsonProperty - @JsonFormat(shape = JsonFormat.Shape.NUMBER, without = JsonFormat.Feature.WRITE_DATE_TIMESTAMPS_AS_NANOSECONDS) Instant updated_at, + @JsonDeserialize(converter = InstantNanoSecondsConverter.class) Instant updated_at, @JsonProperty - @JsonFormat(shape = JsonFormat.Shape.NUMBER, without = JsonFormat.Feature.WRITE_DATE_TIMESTAMPS_AS_NANOSECONDS) Instant published_date, + String published_date, @JsonProperty String notes, @JsonProperty String summary, @JsonProperty("image_url") String imageUrl, diff --git a/src/main/java/eu/cybershu/pocketstats/reader/api/ReadwiseFetchParams.java b/src/main/java/eu/cybershu/pocketstats/reader/api/ReadwiseFetchParams.java index ad46788..0aa6857 100644 --- a/src/main/java/eu/cybershu/pocketstats/reader/api/ReadwiseFetchParams.java +++ b/src/main/java/eu/cybershu/pocketstats/reader/api/ReadwiseFetchParams.java @@ -16,17 +16,10 @@ @Value @Builder public class ReadwiseFetchParams { - String updatedAfter; + Instant updatedAfter; Location location; Category category; - public static class ReadwiseFetchParamsBuilder { - public ReadwiseFetchParamsBuilder updatedAfter(Instant updatedAfter) { - this.updatedAfter = updatedAfter.toString(); - return this; - } - } - public String toQueryParams() { StringBuilder builder = new StringBuilder(); if (updatedAfter != null) { diff --git a/src/main/java/eu/cybershu/pocketstats/reader/api/Tag.java b/src/main/java/eu/cybershu/pocketstats/reader/api/Tag.java new file mode 100644 index 0000000..0b9ac76 --- /dev/null +++ b/src/main/java/eu/cybershu/pocketstats/reader/api/Tag.java @@ -0,0 +1,13 @@ +package eu.cybershu.pocketstats.reader.api; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; + +@JsonInclude(JsonInclude.Include.NON_NULL) +@JsonIgnoreProperties(ignoreUnknown = true) +public record Tag( + @JsonProperty("name") String name, + @JsonProperty("type") String type +) { +} diff --git a/src/main/java/eu/cybershu/pocketstats/utils/InstantNanoSecondsConverter.java b/src/main/java/eu/cybershu/pocketstats/utils/InstantNanoSecondsConverter.java new file mode 100644 index 0000000..a1246f3 --- /dev/null +++ b/src/main/java/eu/cybershu/pocketstats/utils/InstantNanoSecondsConverter.java @@ -0,0 +1,12 @@ +package eu.cybershu.pocketstats.utils; + +import com.fasterxml.jackson.databind.util.StdConverter; + +import java.time.Instant; + +public class InstantNanoSecondsConverter extends StdConverter { + @Override + public Instant convert(String value) { + return Instant.parse(value); + } +} diff --git a/src/main/resources/application.yaml b/src/main/resources/application.yaml index 57c4554..19aafe1 100644 --- a/src/main/resources/application.yaml +++ b/src/main/resources/application.yaml @@ -38,4 +38,12 @@ spring: authentication-database: admin auto-index-creation: true database: ${MONGODB_DB:db} - uri: ${MONGODB_URI} \ No newline at end of file + uri: ${MONGODB_URI} + +resilience4j.ratelimiter: + instances: + readwise-api: + limitForPeriod: 20 + limitRefreshPeriod: 70s + timeoutDuration: 60s + eventConsumerBufferSize: 100 \ No newline at end of file diff --git a/src/test/groovy/eu/cybershu/pocketstats/reader/api/ReaderApiServiceTest.groovy b/src/test/groovy/eu/cybershu/pocketstats/reader/api/ReaderApiServiceTest.groovy index a6fc543..0f7b025 100644 --- a/src/test/groovy/eu/cybershu/pocketstats/reader/api/ReaderApiServiceTest.groovy +++ b/src/test/groovy/eu/cybershu/pocketstats/reader/api/ReaderApiServiceTest.groovy @@ -2,7 +2,7 @@ package eu.cybershu.pocketstats.reader.api import spock.lang.Specification -import java.time.Instant +import java.time.* class ReaderApiServiceTest extends Specification { private String accessToken @@ -15,7 +15,7 @@ class ReaderApiServiceTest extends Specification { def "test connection"() { given: - Instant readFrom = Instant.now().minusSeconds(60*60*24*10) + Instant readFrom = instantFrom("2023-03-28", "01:00:00") when: def response = readerApiService.fetchList(accessToken, @@ -28,4 +28,13 @@ class ReaderApiServiceTest extends Specification { then: response.size() > 0 } + + Instant instantFromLocalDateTime(LocalDateTime localDateTime) { + return localDateTime.atZone(ZoneId.systemDefault()).toInstant() + } + + Instant instantFrom(String strDate, String strTime) { + var ldt = LocalDateTime.of(LocalDate.parse(strDate), LocalTime.parse(strTime)) + return ldt.toInstant(ZoneId.systemDefault().rules.getOffset(ldt)) + } }