diff --git a/.gitignore b/.gitignore index 8c1d49c..0dd648f 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,6 @@ +.env +*.env + # Compiled class file *.class @@ -171,3 +174,4 @@ src/main/java/eu/cybershu/pocketstats/pocket/api/.DS_Store src/.DS_Store src/main/.DS_Store src/main/raporting/.DS_Store +!/.env diff --git a/README.md b/README.md index 03bc74b..0721c93 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # PocketStats -App for visualising [GetPocket](https://getpocket.com/) usage stats. +App for visualising [GetPocket](https://getpocket.com/) and [Reader](https://read.readwise.io/) usage stats. ## Config @@ -9,6 +9,7 @@ App for visualising [GetPocket](https://getpocket.com/) usage stats. #### Environment variables - `BACKEND_URL` - backend url, default: http://localhost:8080 - `POCKET_CONSUMER_KEY` - get pocket consumer key +- `READER_ACCESS_KEY` - readwise access key - `MONGODB_HOST` - mongo db host - `MONGODB_URI` - full mongodb uri with login and pass @@ -25,8 +26,6 @@ example env variables for local instance workig with dockerized MongoDB #### Database Docker service file with mongodb can be find here: [mongodb.yml](https://github.com/michmzr/PocketStats/blob/master/src/main/docker/mongodb.yml) - - ### Frontend #### Environment variables - `VUE_APP_BACKEND_URL` - java backend url diff --git a/build.gradle b/build.gradle index 1643bd8..d339beb 100644 --- a/build.gradle +++ b/build.gradle @@ -36,11 +36,12 @@ ext { dependencies { implementation 'org.springframework.boot:spring-boot-starter-actuator' implementation 'org.springframework.boot:spring-boot-starter-web' - implementation 'commons-validator:commons-validator:1.7' + implementation 'commons-validator:commons-validator:1.8.0' developmentOnly 'org.springframework.boot:spring-boot-devtools' - implementation 'com.fasterxml.jackson.core:jackson-databind' - implementation group: 'com.fasterxml.jackson.dataformat', name: 'jackson-dataformat-xml', version: '2.14.2' - implementation group: 'com.fasterxml.jackson.core', name: 'jackson-databind', version: '2.14.2' + implementation 'com.fasterxml.jackson.core:jackson-databind:2.17.0' + implementation 'com.fasterxml.jackson.datatype:jackson-datatype-jsr310:2.17.0' + implementation group: 'com.fasterxml.jackson.dataformat', name: 'jackson-dataformat-xml', version: '2.17.0' + implementation group: 'com.fasterxml.jackson.core', name: 'jackson-databind', version: '2.17.0' implementation 'org.springframework.boot:spring-boot-starter-data-mongodb' implementation group: 'javax.validation', name: 'validation-api', version: '2.0.1.Final' implementation 'com.google.code.findbugs:jsr305:3.0.2' @@ -125,4 +126,4 @@ task copyFrontendToBuild(type: Copy) { dependsOn 'vueBuild' from "$projectDir/src/frontend/dist/" into "$buildDir/resources/main/static" -} \ No newline at end of file +} diff --git a/src/main/java/eu/cybershu/pocketstats/reader/api/Category.java b/src/main/java/eu/cybershu/pocketstats/reader/api/Category.java new file mode 100644 index 0000000..66956f7 --- /dev/null +++ b/src/main/java/eu/cybershu/pocketstats/reader/api/Category.java @@ -0,0 +1,37 @@ +package eu.cybershu.pocketstats.reader.api; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonValue; + +public enum Category { + ARTICLE("article"), + EMAIL("email"), + RSS("rss"), + HIGHLIGHT("highlight"), + NOTE("note"), + PDF("pdf"), + EPUB("epub"), + TWEET("tweet"), + VIDEO("video"); + + private final String value; + + Category(String value) { + this.value = value; + } + + @JsonValue + public String getValue() { + return value; + } + + @JsonCreator + public static Category forValue(String value) { + for (Category category : Category.values()) { + if (category.getValue().equals(value)) { + return category; + } + } + throw new IllegalArgumentException("Unknown category: " + value); + } +} diff --git a/src/main/java/eu/cybershu/pocketstats/reader/api/Location.java b/src/main/java/eu/cybershu/pocketstats/reader/api/Location.java new file mode 100644 index 0000000..a32ed59 --- /dev/null +++ b/src/main/java/eu/cybershu/pocketstats/reader/api/Location.java @@ -0,0 +1,34 @@ +package eu.cybershu.pocketstats.reader.api; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonValue; + +//values: new, later, shortlist, archive, feed +public enum Location { + NEW("new"), + LATER("later"), + SHORTLIST("shortlist"), + ARCHIVE("archive"), + FEED("feed"); + + private final String value; + + Location(String value) { + this.value = value; + } + + @JsonValue + public String getValue() { + return value; + } + + @JsonCreator + public static Location forValue(String value) { + for (Location location : Location.values()) { + if (location.getValue().equals(value)) { + return location; + } + } + throw new IllegalArgumentException("Unknown location: " + value); + } +} diff --git a/src/main/java/eu/cybershu/pocketstats/reader/api/ReaderApiService.java b/src/main/java/eu/cybershu/pocketstats/reader/api/ReaderApiService.java new file mode 100644 index 0000000..76e78b5 --- /dev/null +++ b/src/main/java/eu/cybershu/pocketstats/reader/api/ReaderApiService.java @@ -0,0 +1,127 @@ +package eu.cybershu.pocketstats.reader.api; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.SerializationFeature; +import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; +import eu.cybershu.pocketstats.pocket.api.ApiXHeaders; +import eu.cybershu.pocketstats.utils.RequestUtils; +import lombok.extern.slf4j.Slf4j; +import org.springframework.http.HttpStatus; +import org.springframework.stereotype.Service; +import org.springframework.util.StringUtils; + +import java.io.IOException; +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.time.Duration; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; + +/** + * Service for interacting with the Readwise Reader API: https://readwise.io/reader_api + */ +@Slf4j +@Service +public class ReaderApiService { + private final String readewiseReaderListUrl = "https://readwise.io/api/v3/list/?"; + + private final HttpClient client; + + private final ObjectMapper mapper; + + public ReaderApiService() { + this.client = HttpClient + .newBuilder() + .followRedirects(HttpClient.Redirect.NORMAL) + .connectTimeout(Duration.ofSeconds(60)) + .build(); + this.mapper = new ObjectMapper(); + this.mapper + .registerModule(new JavaTimeModule()) + .configure(SerializationFeature.WRITE_DATE_TIMESTAMPS_AS_NANOSECONDS, false) + .configure(DeserializationFeature.READ_DATE_TIMESTAMPS_AS_NANOSECONDS, false) + .setSerializationInclusion(JsonInclude.Include.NON_NULL); + } + + /** + * Key Type Description Required + * id string The document's unique id. Using this parameter it will return just one document, if found. no + * updatedAfter string (formatted as ISO 8601 date) Fetch only documents updated after this date no + * location string The document's location, could be one of: new, later, shortlist, archive, feed no + * category string The document's category, could be one of: article, email, rss, highlight, note, pdf, epub, tweet, video no + * pageCursor string A string returned by a previous request to this endpoint. Use it to get the next page of documents if there are too many for one request. no + */ + public List fetchList(String accessToken, ReadwiseFetchParams params) throws IOException, InterruptedException { + log.info("Fetching Readwise list with params: {}", params); + + List items = new LinkedList<>(); + ReadwiseFetchPaginationParams pageParams = ReadwiseFetchPaginationParams + .builder() + .category(params.category()) + .location(params.location()) + .pageCursor(null) + .build(); + + do { + ReaderListResponse response = fetchPage(accessToken, pageParams); + items.addAll(response.results()); + + pageParams = ReadwiseFetchPaginationParams + .builder() + .category(params.category()) + .location(params.location()) + .pageCursor(response.nextPageCursor()) + .build(); + } while (pageParams.pageCursor() != null); + + return items; + } + + private ReaderListResponse fetchPage(String accessToken, ReadwiseFetchPaginationParams params) throws IOException, InterruptedException { + log.info("Fetching Readwise list with params: {}", params); + + String url = readewiseReaderListUrl + params.toQueryParams(); + log.debug("url: {}", url); + + HttpRequest request = HttpRequest.newBuilder() + .GET() + .uri(URI.create(url)) + .header("Authorization", "Token " + accessToken) + .build(); + + HttpResponse response = client.send(request, HttpResponse.BodyHandlers.ofString()); + + logResponse(response); + + switch (response.statusCode()) { + case 200: + String body = response.body(); + + return mapper.readValue(body, ReaderListResponse.class); + case 401: + break; + case 400: + case 403: + case 500: + case 504: + default: + throw new IllegalStateException("Not expected http response code " + response.statusCode()); + } + + throw new IllegalStateException("Not expected http response code " + response.statusCode()); + } + + private void logResponse(HttpResponse response) { + log.debug("status: {}", response.statusCode()); + + // print response body + if (response.statusCode() != HttpStatus.OK.value()) { + log.debug("response: {}", response.body()); + } + } +} diff --git a/src/main/java/eu/cybershu/pocketstats/reader/api/ReaderItem.java b/src/main/java/eu/cybershu/pocketstats/reader/api/ReaderItem.java new file mode 100644 index 0000000..ddb5b26 --- /dev/null +++ b/src/main/java/eu/cybershu/pocketstats/reader/api/ReaderItem.java @@ -0,0 +1,56 @@ +package eu.cybershu.pocketstats.reader.api; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; +import lombok.AllArgsConstructor; +import lombok.Data; + +import java.time.Instant; +import java.util.List; + +/** + { + * "id": "01gkqt8nbms4t698abcdvcswvf", + * "url": "https://readwise.io/new/read/01gkqt8nbms4t698abcdvcswvf", + * "source_url": "https://www.vanityfair.com/news/2022/10/covid-origins-investigation-wuhan-lab", + * "title": "COVID-19 Origins: Investigating a “Complex and Grave Situation” Inside a Wuhan Lab", + * "author": "Condé Nast", + * "source": "Reader add from import URL", + * "category": "article", + * "location": "new", + * "tags": {}, + * "site_name": "Vanity Fair", + * "word_count": 9601, + * "created_at": "2022-12-08T02:50:35.662027+00:00", + * "updated_at": "2023-03-22T13:29:41.827456+00:00", + * "published_date": "2022-10-28", + * "notes": "", + * "summary": "The Wuhan Institute of Virology, the cutting-edge ...", + * "image_url": "https://media.vanityfair.com/photos/63599642578d980751943b65/16:9/w_1280,c_limit/vf-1022-covid-trackers-site-story.jpg", + * "parent_id": null, + * "reading_progress": 0, + * } + */ +@JsonInclude(JsonInclude.Include.NON_NULL) +@JsonIgnoreProperties(ignoreUnknown = true) +public record ReaderItem( + @JsonProperty String id, + @JsonProperty String url, + @JsonProperty("source_url") String sourceUrl, + @JsonProperty String title, + @JsonProperty String author, + @JsonProperty String source, + @JsonProperty Category category, + @JsonProperty Location location, + //@JsonProperty List tags, + @JsonProperty("site_name") String siteName, + @JsonProperty("word_count") int wordCount, + @JsonProperty Instant created_at, + @JsonProperty Instant updated_at, + @JsonProperty Instant published_date, + @JsonProperty String notes, + @JsonProperty String summary, + @JsonProperty("image_url") String imageUrl, + @JsonProperty double reading_progress +) {} diff --git a/src/main/java/eu/cybershu/pocketstats/reader/api/ReaderListResponse.java b/src/main/java/eu/cybershu/pocketstats/reader/api/ReaderListResponse.java new file mode 100644 index 0000000..859f2d9 --- /dev/null +++ b/src/main/java/eu/cybershu/pocketstats/reader/api/ReaderListResponse.java @@ -0,0 +1,87 @@ +package eu.cybershu.pocketstats.reader.api; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; +import lombok.Data; + +import java.util.List; + +/** + * { + * "count": 2304, + * "nextPageCursor": "01gm6kjzabcd609yepjrmcgz8a", + * "results": [ + * { + * "id": "01gwfvp9pyaabcdgmx14f6ha0", + * "url": "https://readiwise.io/feed/read/01gwfvp9pyaabcdgmx14f6ha0", + * "source_url": "https://www.driverlesscrocodile.com/values/ends-and-meanings-3-alasdair-macintyre-virtue-mortality-and-story-in-heroic-societies/", + * "title": "Ends and Meanings (3): Alasdair MacIntyre virtue, mortality and story in heroic societies", + * "author": "Stuart Patience", + * "source": "Reader RSS", + * "category": "rss", + * "location": "feed", + * "tags": {}, + * "site_name": "Driverless Crocodile", + * "word_count": 819, + * "created_at": "2023-03-26T21:02:51.618751+00:00", + * "updated_at": "2023-03-26T21:02:55.453827+00:00", + * "notes": "", + * "published_date": "2023-03-22", + * "summary": "Without … a place in the social order, ...", + * "image_url": "https://i0.wp.com/www.driverlesscrocodile.com/wp-content/uploads/2019/10/cropped-driverlesscrocodile-icon-e1571123201159-4.jpg?fit=32%2C32&ssl=1", + * "parent_id": null, + * "reading_progress": 0.15, + * }, + * { + * "id": "01gkqtdz9xabcd5gt96khreyb", + * "url": "https://readiwise.io/new/read/01gkqtdz9xabcd5gt96khreyb", + * "source_url": "https://www.vanityfair.com/hollywood/2017/08/the-story-of-the-ducktales-theme-music", + * "title": "The Story of the DuckTales Theme, History’s Catchiest Single Minute of Music", + * "author": "Darryn King", + * "source": "Reader add from import URL", + * "category": "article", + * "location": "new", + * "tags": {}, + * "site_name": "Vanity Fair", + * "word_count": 2678, + * "created_at": "2022-12-08T02:53:29.639650+00:00", + * "updated_at": "2022-12-13T20:37:42.544298+00:00", + * "published_date": "2017-08-09", + * "notes": "A sample note", + * "summary": "A woo-hoo heard around the world.", + * "image_url": "https://media.vanityfair.com/photos/598b1452f7f0a433bd4d149c/16:9/w_1280,c_limit/t-ducktales-woohoo-song.png", + * "parent_id": null, + * "reading_progress": 0.5, + * }, + * { + * "id": "01gkqt8nbms4t698abcdvcswvf", + * "url": "https://readwise.io/new/read/01gkqt8nbms4t698abcdvcswvf", + * "source_url": "https://www.vanityfair.com/news/2022/10/covid-origins-investigation-wuhan-lab", + * "title": "COVID-19 Origins: Investigating a “Complex and Grave Situation” Inside a Wuhan Lab", + * "author": "Condé Nast", + * "source": "Reader add from import URL", + * "category": "article", + * "location": "new", + * "tags": {}, + * "site_name": "Vanity Fair", + * "word_count": 9601, + * "created_at": "2022-12-08T02:50:35.662027+00:00", + * "updated_at": "2023-03-22T13:29:41.827456+00:00", + * "published_date": "2022-10-28", + * "notes": "", + * "summary": "The Wuhan Institute of Virology, the cutting-edge ...", + * "image_url": "https://media.vanityfair.com/photos/63599642578d980751943b65/16:9/w_1280,c_limit/vf-1022-covid-trackers-site-story.jpg", + * "parent_id": null, + * "reading_progress": 0, + * } + * ] + * } + */ +@JsonInclude(JsonInclude.Include.NON_NULL) +@JsonIgnoreProperties(ignoreUnknown = true) +public record ReaderListResponse( + @JsonProperty("count") int count, + @JsonProperty("nextPageCursor") String nextPageCursor, + @JsonProperty("results") List results +) {} diff --git a/src/main/java/eu/cybershu/pocketstats/reader/api/ReadwiseFetchPaginationParams.java b/src/main/java/eu/cybershu/pocketstats/reader/api/ReadwiseFetchPaginationParams.java new file mode 100644 index 0000000..70b9b0c --- /dev/null +++ b/src/main/java/eu/cybershu/pocketstats/reader/api/ReadwiseFetchPaginationParams.java @@ -0,0 +1,47 @@ +package eu.cybershu.pocketstats.reader.api; + +import lombok.Builder; +import lombok.Value; + +import java.time.Instant; + +/** + * * Key Type Description Required + * * id string The document's unique id. Using this parameter it will return just one document, if found. no + * * updatedAfter string (formatted as ISO 8601 date) Fetch only documents updated after this date no + * * location string The document's location, could be one of: new, later, shortlist, archive, feed no + * * category string The document's category, could be one of: article, email, rss, highlight, note, pdf, epub, tweet, video no + * * pageCursor string A string returned by a previous request to this endpoint. Use it to get the next page of documents if there are too many for one request. + */ +@Value +@Builder +public class ReadwiseFetchPaginationParams { + String updatedAfter; + Location location; + Category category; + String pageCursor; + + public static class ReadwiseFetchPaginationParamsBuilder { + public ReadwiseFetchPaginationParamsBuilder updatedAfter(Instant updatedAfter) { + this.updatedAfter = updatedAfter.toString(); + return this; + } + } + + public String toQueryParams() { + StringBuilder builder = new StringBuilder(); + if (updatedAfter != null) { + builder.append("updatedAfter=").append(updatedAfter).append("&"); + } + if (location != null) { + builder.append("location=").append(location).append("&"); + } + if (category != null) { + builder.append("category=").append(category).append("&"); + } + if (pageCursor != null) { + builder.append("pageCursor=").append(pageCursor).append("&"); + } + return builder.toString(); + } +} diff --git a/src/main/java/eu/cybershu/pocketstats/reader/api/ReadwiseFetchParams.java b/src/main/java/eu/cybershu/pocketstats/reader/api/ReadwiseFetchParams.java new file mode 100644 index 0000000..ad46788 --- /dev/null +++ b/src/main/java/eu/cybershu/pocketstats/reader/api/ReadwiseFetchParams.java @@ -0,0 +1,43 @@ +package eu.cybershu.pocketstats.reader.api; + +import lombok.Builder; +import lombok.Value; + +import java.time.Instant; + +/** + * * Key Type Description Required + * * id string The document's unique id. Using this parameter it will return just one document, if found. no + * * updatedAfter string (formatted as ISO 8601 date) Fetch only documents updated after this date no + * * location string The document's location, could be one of: new, later, shortlist, archive, feed no + * * category string The document's category, could be one of: article, email, rss, highlight, note, pdf, epub, tweet, video no + * * pageCursor string A string returned by a previous request to this endpoint. Use it to get the next page of documents if there are too many for one request. + */ +@Value +@Builder +public class ReadwiseFetchParams { + String updatedAfter; + Location location; + Category category; + + public static class ReadwiseFetchParamsBuilder { + public ReadwiseFetchParamsBuilder updatedAfter(Instant updatedAfter) { + this.updatedAfter = updatedAfter.toString(); + return this; + } + } + + public String toQueryParams() { + StringBuilder builder = new StringBuilder(); + if (updatedAfter != null) { + builder.append("updatedAfter=").append(updatedAfter).append("&"); + } + if (location != null) { + builder.append("location=").append(location).append("&"); + } + if (category != null) { + builder.append("category=").append(category).append("&"); + } + return builder.toString(); + } +} diff --git a/src/main/java/eu/cybershu/pocketstats/utils/RequestUtils.java b/src/main/java/eu/cybershu/pocketstats/utils/RequestUtils.java index a05dc16..5f77943 100644 --- a/src/main/java/eu/cybershu/pocketstats/utils/RequestUtils.java +++ b/src/main/java/eu/cybershu/pocketstats/utils/RequestUtils.java @@ -18,4 +18,17 @@ public static HttpRequest.BodyPublisher ofFormData(Map data) { } return HttpRequest.BodyPublishers.ofString(builder.toString()); } + + public static String mapToUrlQueries(Map data) { + var builder = new StringBuilder(); + for (Map.Entry entry : data.entrySet()) { + if (!builder.isEmpty()) { + builder.append("&"); + } + builder.append(URLEncoder.encode(entry.getKey().toString(), StandardCharsets.UTF_8)); + builder.append("="); + builder.append(URLEncoder.encode(entry.getValue().toString(), StandardCharsets.UTF_8)); + } + return builder.toString(); + } } diff --git a/src/test/groovy/eu/cybershu/pocketstats/reader/api/ReaderApiServiceTest.groovy b/src/test/groovy/eu/cybershu/pocketstats/reader/api/ReaderApiServiceTest.groovy new file mode 100644 index 0000000..a6fc543 --- /dev/null +++ b/src/test/groovy/eu/cybershu/pocketstats/reader/api/ReaderApiServiceTest.groovy @@ -0,0 +1,31 @@ +package eu.cybershu.pocketstats.reader.api + +import spock.lang.Specification + +import java.time.Instant + +class ReaderApiServiceTest extends Specification { + private String accessToken + + private ReaderApiService readerApiService = new ReaderApiService() + + void setup() { + accessToken = System.getenv("READER_ACCESS_TOKEN") + } + + def "test connection"() { + given: + Instant readFrom = Instant.now().minusSeconds(60*60*24*10) + + when: + def response = readerApiService.fetchList(accessToken, + ReadwiseFetchParams + .builder() + .updatedAfter(readFrom) + .build() + ) + + then: + response.size() > 0 + } +}