Add support for arXiv (#61)

J535D165 · Sep 14, 2023 · 1949d8f · 1949d8f
1 parent 49def2c
commit 1949d8f
Show file tree

Hide file tree

Showing 2 changed files with 20 additions and 0 deletions.
diff --git a/datahugger/config.py b/datahugger/config.py
@@ -1,3 +1,4 @@
+from datahugger.services import ArXivDataset
 from datahugger.services import DataDryadDataset
 from datahugger.services import DataOneDataset
 from datahugger.services import DataverseDataset
@@ -12,6 +13,7 @@
 
 # fast lookup
 SERVICES_NETLOC = {
+    "arxiv.org": ArXivDataset,
     "zenodo.org": ZenodoDataset,
     "github.com": GitHubDataset,
     "datadryad.org": DataDryadDataset,

diff --git a/datahugger/services.py b/datahugger/services.py
@@ -323,3 +323,21 @@ def _get(
     def files(self):
         # at the moment, .files is not available for HuggingFace
         raise AttributeError("'files' is not available for HuggingFace")
+
+
+class ArXivDataset(DatasetDownloader, DatasetResult):
+    """Downloader for ArXiv publication."""
+
+    REGEXP_ID = r"https://arxiv\.org/abs/(?P<record_id>.*)"
+
+    @property
+    def files(self):
+        return [
+            {
+                "link": f"https://arxiv.org/pdf/{self._params['record_id']}.pdf",
+                "name": self._params["record_id"].split("/")[-1] + ".pdf",
+                "size": None,
+                "hash": None,
+                "hash_type": None,
+            }
+        ]