Merge branch 'feature/add_type_hints'

clevercsv/__main__.py

@@ -10,7 +10,7 @@
 from ._optional import import_optional_dependency
 
 
-def main():
+def main() -> None:
  # Check that necessary dependencies are available
  import_optional_dependency("wilderness")
 

clevercsv/_regexes.py

@@ -10,7 +10,7 @@
 # Regular expressions for number formats #
 ##########################################
 
-PATTERN_NUMBER_1: Pattern = regex.compile(
+PATTERN_NUMBER_1: Pattern[str] = regex.compile(
  r"^(?=[+-\.\d])"
  r"[+-]?"
  r"(?:0|[1-9]\d*)?"
@@ -35,19 +35,19 @@
  r"$"
 )
 
-PATTERN_NUMBER_2: Pattern = regex.compile(
+PATTERN_NUMBER_2: Pattern[str] = regex.compile(
  r"[+-]?(?:[1-9]|[1-9]\d{0,2})(?:\,\d{3})+\.\d*"
 )
 
-PATTERN_NUMBER_3: Pattern = regex.compile(
+PATTERN_NUMBER_3: Pattern[str] = regex.compile(
  r"[+-]?(?:[1-9]|[1-9]\d{0,2})(?:\.\d{3})+\,\d*"
 )
 
 ##############################################
 # Regular expressions for url, email, and ip #
 ##############################################
 
-PATTERN_URL: Pattern = regex.compile(
+PATTERN_URL: Pattern[str] = regex.compile(
  r"("
  r"(https?|ftp):\/\/(?!\-)"
  r")?"
@@ -62,17 +62,17 @@
  r"(\.[a-z]+)?"
 )
 
-PATTERN_EMAIL: Pattern = regex.compile(
+PATTERN_EMAIL: Pattern[str] = regex.compile(
  r"(^[a-zA-Z0-9_.+-]+@[a-zA-Z0-9-]+\.[a-zA-Z0-9-.]+$)"
 )
 
-PATTERN_IPV4: Pattern = regex.compile(r"(?:\d{1,3}\.){3}\d{1,3}")
+PATTERN_IPV4: Pattern[str] = regex.compile(r"(?:\d{1,3}\.){3}\d{1,3}")
 
 #################################################
 # Regular expressions related to time notations #
 #################################################
 
-PATTERN_TIME_HHMMSSZZ: Pattern = regex.compile(
+PATTERN_TIME_HHMMSSZZ: Pattern[str] = regex.compile(
  r"(0[0-9]|1[0-9]|2[0-3])"
  r":"
  r"([0-5][0-9])"
@@ -84,21 +84,23 @@
  r"([0-5][0-9])"
 )
 
-PATTERN_TIME_HHMMSS: Pattern = regex.compile(
+PATTERN_TIME_HHMMSS: Pattern[str] = regex.compile(
  r"(0[0-9]|1[0-9]|2[0-3]):([0-5][0-9]):([0-5][0-9])"
 )
 
-PATTERN_TIME_HHMM_1: Pattern = regex.compile(
+PATTERN_TIME_HHMM_1: Pattern[str] = regex.compile(
  r"(0[0-9]|1[0-9]|2[0-3]):([0-5][0-9])"
 )
 
-PATTERN_TIME_HHMM_2: Pattern = regex.compile(
+PATTERN_TIME_HHMM_2: Pattern[str] = regex.compile(
  r"(0[0-9]|1[0-9]|2[0-3])([0-5][0-9])"
 )
 
-PATTERN_TIME_HH: Pattern = regex.compile(r"(0[0-9]|1[0-9]|2[0-3])([0-5][0-9])")
+PATTERN_TIME_HH: Pattern[str] = regex.compile(
+ r"(0[0-9]|1[0-9]|2[0-3])([0-5][0-9])"
+)
 
-PATTERN_TIME_HMM: Pattern = regex.compile(
+PATTERN_TIME_HMM: Pattern[str] = regex.compile(
  r"([0-9]|1[0-9]|2[0-3]):([0-5][0-9])"
 )
 
@@ -109,7 +111,7 @@
 # Regex for various date formats. See
 # https://github.com/alan-turing-institute/CleverCSV/blob/master/notes/date_regex/dateregex_annotated.txt
 # for an explanation.
-PATTERN_DATE: Pattern = regex.compile(
+PATTERN_DATE: Pattern[str] = regex.compile(
  r"("
  r"(0[1-9]|1[0-2])"
  r"("
@@ -238,7 +240,7 @@
 ALPHANUM_SPECIALS: str = regex.escape(r"".join(SPECIALS_ALLOWED))
 
 # Regex for alphanumeric text
-PATTERN_ALPHANUM: Pattern = regex.compile(
+PATTERN_ALPHANUM: Pattern[str] = regex.compile(
  r"("
  r"\p{N}?\p{L}+"
  r"["
@@ -254,7 +256,7 @@
  r"".join(SPECIALS_ALLOWED) + r"".join(QUOTED_SPECIALS_ALLOWED)
 )
 # Regex for alphanumeric text in quoted strings
-PATTERN_ALPHANUM_QUOTED: Pattern = regex.compile(
+PATTERN_ALPHANUM_QUOTED: Pattern[str] = regex.compile(
  r"("
  r"\p{N}?\p{L}+"
  r"["
@@ -270,21 +272,21 @@
 # Regular expression for currency #
 ###################################
 
-PATTERN_CURRENCY: Pattern = regex.compile(r"\p{Sc}\s?(.*)")
+PATTERN_CURRENCY: Pattern[str] = regex.compile(r"\p{Sc}\s?(.*)")
 
 #####################################
 # Regular expression for unix paths #
 #####################################
 
-PATTERN_UNIX_PATH: Pattern = regex.compile(
+PATTERN_UNIX_PATH: Pattern[str] = regex.compile(
  r"[~.]?(?:\/[a-zA-Z0-9\.\-\_]+)+\/?"
 )
 
 ################################################
 # Map of regular expresions for type detection #
 ################################################
 
-DEFAULT_TYPE_REGEXES: Dict[str, Pattern] = {
+DEFAULT_TYPE_REGEXES: Dict[str, Pattern[str]] = {
  "number_1": PATTERN_NUMBER_1,
  "number_2": PATTERN_NUMBER_2,
  "number_3": PATTERN_NUMBER_3,

clevercsv/_types.py

@@ -10,14 +10,17 @@
 from typing import Any
 from typing import Mapping
 from typing import Type
+from typing import TypeVar
 from typing import Union
 
 from clevercsv.dialect import SimpleDialect
 
 AnyPath = Union[str, bytes, "os.PathLike[str]", "os.PathLike[bytes]"]
+StrPath = Union[str, "os.PathLike[str]"]
 _OpenFile = Union[AnyPath, int]
 _DictRow = Mapping[str, Any]
 _DialectLike = Union[str, csv.Dialect, Type[csv.Dialect], SimpleDialect]
+_T = TypeVar("_T")
 
 if sys.version_info >= (3, 8):
  from typing import Dict as _DictReadMapping

clevercsv/consistency.py

@@ -88,7 +88,7 @@ def cached_is_known_type(cell: str, is_quoted: bool) -> bool:
  self._cached_is_known_type = cached_is_known_type
 
  def detect(
- self, data: str, delimiters: Optional[Iterable[str]] = None
+ self, data: str, delimiters: Optional[List[str]] = None
  ) -> Optional[SimpleDialect]:
  """Detect the dialect using the consistency measure
 
@@ -192,7 +192,7 @@ def get_best_dialects(
  return [d for d, score in scores.items() if score.Q == Qmax]
 
  def compute_type_score(
- self, data: str, dialect: SimpleDialect, eps=DEFAULT_EPS_TYPE
+ self, data: str, dialect: SimpleDialect, eps: float = DEFAULT_EPS_TYPE
  ) -> float:
  """Compute the type score"""
  total = known = 0
@@ -211,8 +211,10 @@ def detect_dialect_consistency(
  delimiters: Optional[Iterable[str]] = None,
  skip: bool = True,
  verbose: bool = False,
-):
+) -> Optional[SimpleDialect]:
  """Helper function that wraps ConsistencyDetector"""
  # Mostly kept for backwards compatibility
  consistency_detector = ConsistencyDetector(skip=skip, verbose=verbose)
+ if delimiters is not None:
+ delimiters = list(delimiters)
  return consistency_detector.detect(data, delimiters=delimiters)

clevercsv/console/__init__.py

@@ -3,6 +3,6 @@
 from .application import build_application
 
 
-def main():
+def main() -> int:
  app = build_application()
  return app.run()

clevercsv/console/application.py

@@ -64,7 +64,7 @@ class CleverCSVApplication(Application):
  ),
  }
 
- def __init__(self):
+ def __init__(self) -> None:
  super().__init__(
  "clevercsv",
  version=__version__,
@@ -74,7 +74,7 @@ def __init__(self):
  extra_sections=self._extra,
  )
 
- def register(self):
+ def register(self) -> None:
  self.add_argument(
  "-V",
  "--version",

clevercsv/console/commands/_utils.py

@@ -1,9 +1,14 @@
 # -*- coding: utf-8 -*-
 
+from typing import Any
+from typing import List
+from typing import Optional
+
 from clevercsv import __version__
+from clevercsv.dialect import SimpleDialect
 
 
-def parse_int(val, name):
+def parse_int(val: Any, name: str) -> Optional[int]:
  """Parse a number to an integer if possible"""
  if val is None:
  return val
@@ -15,7 +20,13 @@ def parse_int(val, name):
  )
 
 
-def generate_code(filename, dialect, encoding, use_pandas=False):
+def generate_code(
+ filename: str,
+ dialect: SimpleDialect,
+ encoding: Optional[str],
+ use_pandas: bool = False,
+) -> List[str]:
+ assert dialect.quotechar is not None
  d = '"\\t"' if dialect.delimiter == "\t" else f'"{dialect.delimiter}"'
  q = '"%s"' % (dialect.quotechar.replace('"', '\\"'))
  e = repr(f"{dialect.escapechar}").replace("'", '"')
@@ -26,15 +37,17 @@ def generate_code(filename, dialect, encoding, use_pandas=False):
  "import clevercsv",
  ]
  if use_pandas:
- return base + [
+ return [
+ *base,
  "",
  f'df = clevercsv.read_dataframe("{filename}", delimiter={d}, '
  f"quotechar={q}, escapechar={e})",
  "",
  ]
 
  enc = "None" if encoding is None else f'"{encoding}"'
- lines = base + [
+ lines = [
+ *base,
  "",
  f'with open("{filename}", "r", newline="", encoding={enc}) as fp:',
  " reader = clevercsv.reader(fp, "

clevercsv/console/commands/code.py

@@ -21,15 +21,15 @@ class CodeCommand(Command):
  "and copy the generated code to a Python script."
  )
 
- def __init__(self):
+ def __init__(self) -> None:
  super().__init__(
  name="code",
  title="Generate Python code to import a CSV file",
  description=self._description,
  extra_sections={"CleverCSV": "Part of the CleverCSV suite"},
  )
 
- def register(self):
+ def register(self) -> None:
  self.add_argument("path", help="Path to the CSV file")
  self.add_argument(
  "-e",

clevercsv/console/commands/detect.py

@@ -18,15 +18,15 @@
 class DetectCommand(Command):
  _description = "Detect the dialect of a CSV file."
 
- def __init__(self):
+ def __init__(self) -> None:
  super().__init__(
  name="detect",
  title="Detect the dialect of a CSV file",
  description=self._description,
  extra_sections={"CleverCSV": "Part of the CleverCSV suite"},
  )
 
- def register(self):
+ def register(self) -> None:
  self.add_argument("path", help="Path to the CSV file")
  self.add_argument(
  "-c",
@@ -100,7 +100,7 @@ def register(self):
  help="Add the runtime of the detection to the detection output.",
  )
 
- def handle(self):
+ def handle(self) -> int:
  verbose = self.args.verbose
  num_chars = parse_int(self.args.num_chars, "num-chars")
  method = "consistency" if self.args.consistency else "auto"

clevercsv/console/commands/explore.py

@@ -26,15 +26,15 @@ class ExploreCommand(Command):
  "to read the file as a Pandas dataframe."
  )
 
- def __init__(self):
+ def __init__(self) -> None:
  super().__init__(
  name="explore",
  title="Explore the CSV file in an interactive Python shell",
  description=self._description,
  extra_sections={"CleverCSV": "Part of the CleverCSV suite"},
  )
 
- def register(self):
+ def register(self) -> None:
  self.add_argument("path", help="Path to the CSV file")
  self.add_argument(
  "-e",