Fixed deserialization

Added auto retry for Readwise
WIP: fix 429 error
Updated browser plugins

README.md

@@ -9,7 +9,7 @@ App for visualising [GetPocket](https://getpocket.com/) and [Reader](https://rea
 #### Environment variables
 - `BACKEND_URL` - backend url, default: http://localhost:8080
 - `POCKET_CONSUMER_KEY` - get pocket consumer key
-- `READER_ACCESS_KEY` - readwise access key
+- `READER_ACCESS_TOKEN` - readwise access key
 - `MONGODB_HOST` - mongo db host 
 - `MONGODB_URI` - full mongodb uri with login and pass
 

build.gradle

@@ -29,6 +29,7 @@ repositories {
 
 ext {
     mapstructVersion = "1.5.3.Final"
+    jacksonVersion = "2.17.0"
     lombokVersion = "1.18.20"
     lombokMapstructBindingVersion = "0.2.0"
 }
@@ -38,10 +39,12 @@ dependencies {
     implementation 'org.springframework.boot:spring-boot-starter-web'
     implementation 'commons-validator:commons-validator:1.8.0'
     developmentOnly 'org.springframework.boot:spring-boot-devtools'
-    implementation 'com.fasterxml.jackson.core:jackson-databind:2.17.0'
-    implementation 'com.fasterxml.jackson.datatype:jackson-datatype-jsr310:2.17.0'
-    implementation group: 'com.fasterxml.jackson.dataformat', name: 'jackson-dataformat-xml', version: '2.17.0'
-    implementation group: 'com.fasterxml.jackson.core', name: 'jackson-databind', version: '2.17.0'
+
+    implementation("com.fasterxml.jackson:jackson-bom:${jacksonVersion}")
+    implementation("com.fasterxml.jackson.core:jackson-core:${jacksonVersion}")
+    implementation("com.fasterxml.jackson.core:jackson-databind:${jacksonVersion}")
+    implementation("com.fasterxml.jackson.datatype:jackson-datatype-jsr310:${jacksonVersion}")
+
     implementation 'org.springframework.boot:spring-boot-starter-data-mongodb'
     implementation group: 'javax.validation', name: 'validation-api', version: '2.0.1.Final'
     implementation 'com.google.code.findbugs:jsr305:3.0.2'
@@ -56,6 +59,8 @@ dependencies {
             "org.projectlombok:lombok-mapstruct-binding:${lombokMapstructBindingVersion}"
     implementation 'com.google.guava:guava:31.1-jre'
 
+    implementation 'io.github.resilience4j:resilience4j-spring-boot3:2.2.0'
+
     //	Tests
     testImplementation('org.springframework.boot:spring-boot-starter-test') {
         exclude group: 'org.junit.vintage', module: 'junit-vintage-engine'

src/main/java/eu/cybershu/pocketstats/reader/api/ReaderApiService.java

@@ -1,16 +1,10 @@
 package eu.cybershu.pocketstats.reader.api;
 
-import com.fasterxml.jackson.annotation.JsonInclude;
-import com.fasterxml.jackson.databind.DeserializationFeature;
 import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.SerializationFeature;
-import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
-import eu.cybershu.pocketstats.pocket.api.ApiXHeaders;
-import eu.cybershu.pocketstats.utils.RequestUtils;
+import io.github.resilience4j.ratelimiter.annotation.RateLimiter;
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.http.HttpStatus;
 import org.springframework.stereotype.Service;
-import org.springframework.util.StringUtils;
 
 import java.io.IOException;
 import java.net.URI;
@@ -20,7 +14,6 @@
 import java.time.Duration;
 import java.util.LinkedList;
 import java.util.List;
-import java.util.Map;
 
 /**
  * Service for interacting with the Readwise Reader API: https://readwise.io/reader_api
@@ -40,12 +33,10 @@ public ReaderApiService() {
                 .followRedirects(HttpClient.Redirect.NORMAL)
                 .connectTimeout(Duration.ofSeconds(60))
                 .build();
-        this.mapper = new ObjectMapper();
-        this.mapper
-                .registerModule(new JavaTimeModule())
-                .configure(SerializationFeature.WRITE_DATE_TIMESTAMPS_AS_NANOSECONDS, false)
-                .configure(DeserializationFeature.READ_DATE_TIMESTAMPS_AS_NANOSECONDS, false)
-                .setSerializationInclusion(JsonInclude.Include.NON_NULL);
+        this.mapper = new ObjectMapper().findAndRegisterModules();
+//        this.mapper.disable(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS)
+//        .disable(DeserializationFeature.READ_DATE_TIMESTAMPS_AS_NANOSECONDS)
+//
     }
 
     /**
@@ -75,13 +66,15 @@ public List<ReaderItem> fetchList(String accessToken, ReadwiseFetchParams params
                     .builder()
                     .category(params.category())
                     .location(params.location())
+                    .updatedAfter(params.updatedAfter())
                     .pageCursor(response.nextPageCursor())
                     .build();
         } while (pageParams.pageCursor() != null);
 
         return items;
     }
 
+    @RateLimiter(name = "readwise-api")
     private ReaderListResponse fetchPage(String accessToken, ReadwiseFetchPaginationParams params) throws IOException, InterruptedException {
         log.info("Fetching Readwise list with params: {}", params);
 
@@ -102,7 +95,8 @@ private ReaderListResponse fetchPage(String accessToken, ReadwiseFetchPagination
             case 200:
                 String body = response.body();
 
-                return mapper.readValue(body, ReaderListResponse.class);
+                ReaderListResponse results = mapper.readValue(body, ReaderListResponse.class);
+                return results;
             case 401:
                 break;
             case 400:

src/main/java/eu/cybershu/pocketstats/reader/api/ReaderItem.java

@@ -1,14 +1,13 @@
 package eu.cybershu.pocketstats.reader.api;
 
-import com.fasterxml.jackson.annotation.JsonFormat;
 import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
 import com.fasterxml.jackson.annotation.JsonInclude;
 import com.fasterxml.jackson.annotation.JsonProperty;
-import lombok.AllArgsConstructor;
-import lombok.Data;
+import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
+import eu.cybershu.pocketstats.utils.InstantNanoSecondsConverter;
 
 import java.time.Instant;
-import java.util.List;
+import java.util.Map;
 
 /**
  {
@@ -44,15 +43,15 @@ public record ReaderItem(
         @JsonProperty String source,
         @JsonProperty Category category,
         @JsonProperty Location location,
-        //@JsonProperty List<String> tags,
+        @JsonProperty Map<String, Tag> tags,
         @JsonProperty("site_name") String siteName,
         @JsonProperty("word_count") int wordCount,
         @JsonProperty
-        @JsonFormat(shape = JsonFormat.Shape.NUMBER, without = JsonFormat.Feature.WRITE_DATE_TIMESTAMPS_AS_NANOSECONDS) Instant created_at,
+        @JsonDeserialize(converter = InstantNanoSecondsConverter.class) Instant created_at,
         @JsonProperty
-        @JsonFormat(shape = JsonFormat.Shape.NUMBER, without = JsonFormat.Feature.WRITE_DATE_TIMESTAMPS_AS_NANOSECONDS) Instant updated_at,
+        @JsonDeserialize(converter = InstantNanoSecondsConverter.class) Instant updated_at,
         @JsonProperty
-        @JsonFormat(shape = JsonFormat.Shape.NUMBER, without = JsonFormat.Feature.WRITE_DATE_TIMESTAMPS_AS_NANOSECONDS) Instant published_date,
+        String published_date,
         @JsonProperty String notes,
         @JsonProperty String summary,
         @JsonProperty("image_url") String imageUrl,

src/main/java/eu/cybershu/pocketstats/reader/api/ReadwiseFetchParams.java

@@ -16,17 +16,10 @@
 @Value
 @Builder
 public class ReadwiseFetchParams {
-    String updatedAfter;
+    Instant updatedAfter;
     Location location;
     Category category;
 
-    public static class ReadwiseFetchParamsBuilder {
-        public ReadwiseFetchParamsBuilder updatedAfter(Instant updatedAfter) {
-            this.updatedAfter = updatedAfter.toString();
-            return this;
-        }
-    }
-
     public String toQueryParams() {
         StringBuilder builder = new StringBuilder();
         if (updatedAfter != null) {

src/main/java/eu/cybershu/pocketstats/reader/api/Tag.java

@@ -0,0 +1,13 @@
+package eu.cybershu.pocketstats.reader.api;
+
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+@JsonInclude(JsonInclude.Include.NON_NULL)
+@JsonIgnoreProperties(ignoreUnknown = true)
+public record Tag(
+        @JsonProperty("name") String name,
+        @JsonProperty("type") String type
+) {
+}

src/main/java/eu/cybershu/pocketstats/utils/InstantNanoSecondsConverter.java

@@ -0,0 +1,12 @@
+package eu.cybershu.pocketstats.utils;
+
+import com.fasterxml.jackson.databind.util.StdConverter;
+
+import java.time.Instant;
+
+public class InstantNanoSecondsConverter extends StdConverter<String, Instant> {
+    @Override
+    public Instant convert(String value) {
+        return Instant.parse(value);
+    }
+}

src/main/resources/application.yaml

@@ -38,4 +38,12 @@ spring:
       authentication-database: admin
       auto-index-creation: true
       database: ${MONGODB_DB:db}
-      uri: ${MONGODB_URI}
+      uri: ${MONGODB_URI}
+
+resilience4j.ratelimiter:
+  instances:
+    readwise-api:
+      limitForPeriod: 20
+      limitRefreshPeriod: 70s
+      timeoutDuration: 60s
+      eventConsumerBufferSize: 100

src/test/groovy/eu/cybershu/pocketstats/reader/api/ReaderApiServiceTest.groovy

@@ -2,7 +2,7 @@ package eu.cybershu.pocketstats.reader.api
 
 import spock.lang.Specification
 
-import java.time.Instant
+import java.time.*
 
 class ReaderApiServiceTest extends Specification {
     private String accessToken
@@ -15,7 +15,7 @@ class ReaderApiServiceTest extends Specification {
 
     def "test connection"() {
         given:
-        Instant readFrom = Instant.now().minusSeconds(60*60*24*10)
+        Instant readFrom = instantFrom("2023-03-28", "01:00:00")
 
         when:
         def response = readerApiService.fetchList(accessToken,
@@ -28,4 +28,13 @@ class ReaderApiServiceTest extends Specification {
         then:
         response.size() > 0
     }
+
+    Instant instantFromLocalDateTime(LocalDateTime localDateTime) {
+        return localDateTime.atZone(ZoneId.systemDefault()).toInstant()
+    }
+
+    Instant instantFrom(String strDate, String strTime) {
+        var ldt = LocalDateTime.of(LocalDate.parse(strDate), LocalTime.parse(strTime))
+        return ldt.toInstant(ZoneId.systemDefault().rules.getOffset(ldt))
+    }
 }