Use session cookie (v1.5) (#29)

- Add -c/--cookie
- Remove email and password login, exit with error if -c not provided
- Closes #28

README.md

@@ -1,5 +1,5 @@
 # FantiaDL
-Download media and other data from Fantia fanclubs and posts. An email and password must be provided using the -e and -p arguments or with a .netrc file.
+Download media and other data from Fantia fanclubs and posts. A session cookie must be provided with the -c/--cookie argument directly or by passing the path to a legacy Netscape cookies file. Please see the [About Session Cookies](#about-session-cookies) section.
 
 ```
 usage: fantiadl.py [options] url
@@ -9,11 +9,8 @@ positional arguments:
 
 optional arguments:
   -h, --help            show this help message and exit
-  -e EMAIL, --email EMAIL
-                        fantia email
-  -p PASSWORD, --password PASSWORD
-                        fantia password
-  -n, --netrc           login with .netrc
+  -c SESSION_COOKIE, --cookie SESSION_COOKIE
+                        _session_id cookie or cookies.txt
   -q, --quiet           suppress output
   -v, --version         show program's version number and exit
 
@@ -39,6 +36,32 @@ download options:
 
 When parsing for external links using `-x`, a .crawljob file is created in your root directory (either the directory provided with `-o` or the directory the script is being run from) that can be parsed by [JDownloader](http://jdownloader.org/). As posts are parsed, links will be appended and assigned their appropriate post directories for download. You can import this file manually into JDownloader (File -> Load Linkcontainer) or setup the Folder Watch plugin to watch your root directory for .crawljob files.
 
+## About Session Cookies
+Due to recent changes imposed by Fantia, providing an email and password to login from the command line is no longer supported. In order to login, you will need to provide the `_session_id` cookie for your Fantia login session using -c/--cookie. After logging in normally on your browser, this value can then be extracted and used with FantiaDL. This value expires and may need to be updated with some regularity.
+
+### Mozilla Firefox
+1. On https://fantia.jp, press Ctrl + Shift + I to open Developer Tools.
+2. Select the Storage tab at the top. In the sidebar, select https://fantia.jp under the Cookies heading.
+3. Locate the `_session_id` cookie name. Click on the value to copy it.
+
+### Google Chrome
+1. On https://fantia.jp, press Ctrl + Shift + I to open DevTools.
+2. Select the Application tab at the top. In the sidebar, expand Cookies under the Storage heading and select https://fantia.jp.
+3. Locate the `_session_id` cookie name. Click on the value to copy it.
+
+### Third-Party Extensions (cookies.txt)
+You also have the option of passing the path to a legacy Netscape format cookies file with -c/--cookie, e.g. `-c ~/cookies.txt`. Using an extension like [cookies.txt](https://chrome.google.com/webstore/detail/cookiestxt/njabckikapfpffapmjgojcnbfjonfjfg), create a text file matching the accepted format:
+
+```
+# Netscape HTTP Cookie File
+# https://curl.haxx.se/rfc/cookie_spec.html
+# This is a generated file! Do not edit.
+
+fantia.jp	FALSE	/	FALSE	1595755239	_session_id	a1b2c3d4...
+```
+
+Only the `_session_id` cookie is required.
+
 ## Download
 Check the [releases page](https://github.com/bitbybyte/fantiadl/releases/latest) for the latest binaries.
 

fantiadl.py

@@ -14,7 +14,7 @@
 __copyright__ = "Copyright 2020 bitbybyte"
 
 __license__ = "MIT"
-__version__ = "1.4.3"
+__version__ = "1.5"
 
 BASE_HOST = "fantia.jp"
 
@@ -23,9 +23,10 @@
     cmdl_version = __version__
     cmdl_parser = argparse.ArgumentParser(usage=cmdl_usage, conflict_handler="resolve")
 
-    cmdl_parser.add_argument("-e", "--email", dest="email", metavar="EMAIL", help="fantia email")
-    cmdl_parser.add_argument("-p", "--password", dest="password", metavar="PASSWORD", help="fantia password")
-    cmdl_parser.add_argument("-n", "--netrc", action="store_true", dest="netrc", help="login with .netrc")
+    cmdl_parser.add_argument("-c", "--cookie", dest="session_arg", metavar="SESSION_COOKIE", help="_session_id cookie or cookies.txt")
+    cmdl_parser.add_argument("-e", "--email", dest="email", metavar="EMAIL", help=argparse.SUPPRESS)
+    cmdl_parser.add_argument("-p", "--password", dest="password", metavar="PASSWORD", help=argparse.SUPPRESS)
+    cmdl_parser.add_argument("-n", "--netrc", action="store_true", dest="netrc", help=argparse.SUPPRESS)
     cmdl_parser.add_argument("-q", "--quiet", action="store_true", dest="quiet", help="suppress output")
     cmdl_parser.add_argument("-v", "--version", action="version", version=cmdl_version)
     cmdl_parser.add_argument("url", action="store", nargs="*", help="fanclub or post URL")
@@ -44,27 +45,35 @@
 
     cmdl_opts = cmdl_parser.parse_args()
 
+    session_arg = cmdl_opts.session_arg
     email = cmdl_opts.email
     password = cmdl_opts.password
 
+    if (email or password or cmdl_opts.netrc) and not session_arg:
+        sys.exit("Logging in from the command line is no longer supported. Please provide a session cookie using -c/--cookie. See the README for more information.")
+
     if not cmdl_opts.download_fanclubs and not cmdl_opts.url:
         sys.exit("Error: No valid input provided")
 
-    if cmdl_opts.netrc:
-        login = netrc.netrc().authenticators(BASE_HOST)
-        if login:
-            email = login[0]
-            password = login[2]
-        else:
-            sys.exit("Error: No Fantia login found in .netrc")
-    else:
-        if not email:
-            email = input("Email: ")
-        if not password:
-            password = getpass.getpass("Password: ")
+    if not session_arg:
+        session_arg = input("Fantia session cookie (_session_id or cookies.txt path): ")
+
+    # if cmdl_opts.netrc:
+    #     login = netrc.netrc().authenticators(BASE_HOST)
+    #     if login:
+    #         email = login[0]
+    #         password = login[2]
+    #     else:
+    #         sys.exit("Error: No Fantia login found in .netrc")
+    # else:
+    #     if not email:
+    #         email = input("Email: ")
+    #     if not password:
+    #         password = getpass.getpass("Password: ")
 
     try:
-        downloader = models.FantiaDownloader(email=email, password=password, dump_metadata=cmdl_opts.dump_metadata, parse_for_external_links=cmdl_opts.parse_for_external_links, autostart_crawljob=cmdl_opts.autostart_crawljob, download_thumb=cmdl_opts.download_thumb, directory=cmdl_opts.output_path, quiet=cmdl_opts.quiet, continue_on_error=cmdl_opts.continue_on_error, use_server_filenames=cmdl_opts.use_server_filenames, mark_incomplete_posts=cmdl_opts.mark_incomplete_posts)
+        downloader = models.FantiaDownloader(session_arg=session_arg, dump_metadata=cmdl_opts.dump_metadata, parse_for_external_links=cmdl_opts.parse_for_external_links, autostart_crawljob=cmdl_opts.autostart_crawljob, download_thumb=cmdl_opts.download_thumb, directory=cmdl_opts.output_path, quiet=cmdl_opts.quiet, continue_on_error=cmdl_opts.continue_on_error, use_server_filenames=cmdl_opts.use_server_filenames, mark_incomplete_posts=cmdl_opts.mark_incomplete_posts)
+        # downloader = models.FantiaDownloader(email=email, password=password, dump_metadata=cmdl_opts.dump_metadata, parse_for_external_links=cmdl_opts.parse_for_external_links, autostart_crawljob=cmdl_opts.autostart_crawljob, download_thumb=cmdl_opts.download_thumb, directory=cmdl_opts.output_path, quiet=cmdl_opts.quiet, continue_on_error=cmdl_opts.continue_on_error, use_server_filenames=cmdl_opts.use_server_filenames, mark_incomplete_posts=cmdl_opts.mark_incomplete_posts)
         if cmdl_opts.download_fanclubs:
             try:
                 downloader.download_followed_fanclubs(limit=cmdl_opts.limit)

models.py

@@ -1,10 +1,13 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 
+# from bs4 import BeautifulSoup
 import requests
+
 from urllib.parse import unquote
 from urllib.parse import urljoin
 from urllib.parse import urlparse
+import http.cookiejar
 import json
 import mimetypes
 import os
@@ -18,8 +21,10 @@
 
 CRAWLJOB_FILENAME = "external_links.crawljob" # TODO: Set as flag
 
-LOGIN_URL = "https://fantia.jp/auth/login"
-LOGIN_CALLBACK_URL = "https://fantia.jp/auth/toranoana/callback?code={}&state={}"
+BASE_URL = "https://fantia.jp/"
+
+LOGIN_SIGNIN_URL = "https://fantia.jp/sessions/signin"
+LOGIN_SESSION_URL = "https://fantia.jp/sessions"
 
 ME_API = "https://fantia.jp/api/v1/me"
 
@@ -46,9 +51,11 @@ def __init__(self, fanclub_id):
 
 
 class FantiaDownloader:
-    def __init__(self, email, password, chunk_size=1024 * 1024 * 5, dump_metadata=False, parse_for_external_links=False, autostart_crawljob=False, download_thumb=False, directory=None, quiet=True, continue_on_error=False, use_server_filenames=False, mark_incomplete_posts=False):
-        self.email = email
-        self.password = password
+    def __init__(self, session_arg, chunk_size=1024 * 1024 * 5, dump_metadata=False, parse_for_external_links=False, autostart_crawljob=False, download_thumb=False, directory=None, quiet=True, continue_on_error=False, use_server_filenames=False, mark_incomplete_posts=False):
+    # def __init__(self, email, password, session_cookie=None, chunk_size=1024 * 1024 * 5, dump_metadata=False, parse_for_external_links=False, autostart_crawljob=False, download_thumb=False, directory=None, quiet=True, continue_on_error=False, use_server_filenames=False, mark_incomplete_posts=False):
+        # self.email = email
+        # self.password = password
+        self.session_arg = session_arg
         self.chunk_size = chunk_size
         self.dump_metadata = dump_metadata
         self.parse_for_external_links = parse_for_external_links
@@ -74,31 +81,45 @@ def output(self, output):
 
     def login(self):
         """Login to Fantia using the provided email and password."""
-        login = self.session.get(LOGIN_URL)
-        auth_url = login.url.replace("id.fantia.jp/auth/", "id.fantia.jp/authorize")
-
-        login_json = {
-            "Email": self.email,
-            "Password": self.password
-        }
-
-        login_headers = {
-            "X-Not-Redirection": "true"
-        }
 
-        auth_response = self.session.post(auth_url, json=login_json, headers=login_headers).json()
-        if auth_response["status"] == "OK":
-            auth_payload = auth_response["payload"]
-
-            callback = self.session.get(LOGIN_CALLBACK_URL.format(auth_payload["code"], auth_payload["state"]))
-            if not callback.cookies["_session_id"]:
-                sys.exit("Error: Failed to retrieve session key from callback")
-
-            check_user = self.session.get(ME_API)
-            if not (check_user.ok or check_user.status_code == 304):
-                sys.exit("Error: Invalid session")
-        else:
-            sys.exit("Error: Failed to login. Please verify your username and password")
+        try:
+            with open(self.session_arg, "r") as cookies_file:
+                cookies = http.cookiejar.MozillaCookieJar(self.session_arg)
+                cookies.load()
+                self.session.cookies = cookies
+        except FileNotFoundError:
+            login_cookie = requests.cookies.create_cookie(domain="fantia.jp", name="_session_id", value=self.session_arg)
+            self.session.cookies.set_cookie(login_cookie)
+
+        check_user = self.session.get(ME_API)
+        if not (check_user.ok or check_user.status_code == 304):
+            sys.exit("Error: Invalid session. Please verify your session cookie")
+
+        # Login flow, requires reCAPTCHA token
+
+        # login_json = {
+        #     "utf8": "✓",
+        #     "button": "",
+        #     "user[email]": self.email,
+        #     "user[password]": self.password,
+        # }
+
+        # login_session = self.session.get(LOGIN_SIGNIN_URL)
+        # login_page = BeautifulSoup(login_session.text, "html.parser")
+        # authenticity_token = login_page.select_one("input[name=\"authenticity_token\"]")["value"]
+        # print(login_page.select_one("input[name=\"recaptcha_response\"]"))
+        # login_json["authenticity_token"] = authenticity_token
+        # login_json["recaptcha_response"] = ...
+
+        # create_session = self.session.post(LOGIN_SESSION_URL, data=login_json)
+        # if not create_session.headers.get("Location"):
+        #     sys.exit("Error: Bad login form data")
+        # elif create_session.headers["Location"] == LOGIN_SIGNIN_URL:
+        #     sys.exit("Error: Failed to login. Please verify your username and password")
+
+        # check_user = self.session.get(ME_API)
+        # if not (check_user.ok or check_user.status_code == 304):
+        #     sys.exit("Error: Invalid session")
 
     def process_content_type(self, url):
         """Process the Content-Type from a request header and use it to build a filename."""