move 'util.Formatter' into its own 'formatter' module

gallery_dl/formatter.py

@@ -0,0 +1,267 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2021 Mike Fährmann
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+
+"""String formatters"""
+
+import json
+import string
+import _string
+import operator
+from . import text, util
+
+_CACHE = {}
+_CONVERSIONS = None
+
+
+def parse(format_string, default=None):
+    key = format_string, default
+
+    try:
+        return _CACHE[key]
+    except KeyError:
+        formatter = _CACHE[key] = StringFormatter(format_string, default)
+        return formatter
+
+
+class StringFormatter():
+    """Custom, extended version of string.Formatter
+
+    This string formatter implementation is a mostly performance-optimized
+    variant of the original string.Formatter class. Unnecessary features have
+    been removed (positional arguments, unused argument check) and new
+    formatting options have been added.
+
+    Extra Conversions:
+    - "l": calls str.lower on the target value
+    - "u": calls str.upper
+    - "c": calls str.capitalize
+    - "C": calls string.capwords
+    - "j". calls json.dumps
+    - "t": calls str.strip
+    - "d": calls text.parse_timestamp
+    - "U": calls urllib.parse.unquote
+    - "S": calls util.to_string()
+    - "T": calls util.to_timestamü()
+    - Example: {f!l} -> "example"; {f!u} -> "EXAMPLE"
+
+    Extra Format Specifiers:
+    - "?<before>/<after>/":
+        Adds <before> and <after> to the actual value if it evaluates to True.
+        Otherwise the whole replacement field becomes an empty string.
+        Example: {f:?-+/+-/} -> "-+Example+-" (if "f" contains "Example")
+                             -> ""            (if "f" is None, 0, "")
+
+    - "L<maxlen>/<replacement>/":
+        Replaces the output with <replacement> if its length (in characters)
+        exceeds <maxlen>. Otherwise everything is left as is.
+        Example: {f:L5/too long/} -> "foo"      (if "f" is "foo")
+                                  -> "too long" (if "f" is "foobar")
+
+    - "J<separator>/":
+        Joins elements of a list (or string) using <separator>
+        Example: {f:J - /} -> "a - b - c" (if "f" is ["a", "b", "c"])
+
+    - "R<old>/<new>/":
+        Replaces all occurrences of <old> with <new>
+        Example: {f:R /_/} -> "f_o_o_b_a_r" (if "f" is "f o o b a r")
+    """
+
+    def __init__(self, format_string, default=None):
+        self.default = default
+        self.result = []
+        self.fields = []
+
+        for literal_text, field_name, format_spec, conv in \
+                _string.formatter_parser(format_string):
+            if literal_text:
+                self.result.append(literal_text)
+            if field_name:
+                self.fields.append((
+                    len(self.result),
+                    self._field_access(field_name, format_spec, conv),
+                ))
+                self.result.append("")
+
+        if len(self.result) == 1:
+            if self.fields:
+                self.format_map = self.fields[0][1]
+            else:
+                self.format_map = lambda _: format_string
+            del self.result, self.fields
+
+    def format_map(self, kwdict):
+        """Apply 'kwdict' to the initial format_string and return its result"""
+        result = self.result
+        for index, func in self.fields:
+            result[index] = func(kwdict)
+        return "".join(result)
+
+    def _field_access(self, field_name, format_spec, conversion):
+        fmt = parse_format_spec(format_spec, conversion)
+
+        if "|" in field_name:
+            return self._apply_list([
+                parse_field_name(fn)
+                for fn in field_name.split("|")
+            ], fmt)
+        else:
+            key, funcs = parse_field_name(field_name)
+            if funcs:
+                return self._apply(key, funcs, fmt)
+            return self._apply_simple(key, fmt)
+
+    def _apply(self, key, funcs, fmt):
+        def wrap(kwdict):
+            try:
+                obj = kwdict[key]
+                for func in funcs:
+                    obj = func(obj)
+            except Exception:
+                obj = self.default
+            return fmt(obj)
+        return wrap
+
+    def _apply_simple(self, key, fmt):
+        def wrap(kwdict):
+            return fmt(kwdict[key] if key in kwdict else self.default)
+        return wrap
+
+    def _apply_list(self, lst, fmt):
+        def wrap(kwdict):
+            for key, funcs in lst:
+                try:
+                    obj = kwdict[key]
+                    for func in funcs:
+                        obj = func(obj)
+                    if obj:
+                        break
+                except Exception:
+                    pass
+            else:
+                obj = self.default
+            return fmt(obj)
+        return wrap
+
+
+def parse_field_name(field_name):
+    first, rest = _string.formatter_field_name_split(field_name)
+    funcs = []
+
+    for is_attr, key in rest:
+        if is_attr:
+            func = operator.attrgetter
+        else:
+            func = operator.itemgetter
+            try:
+                if ":" in key:
+                    start, _, stop = key.partition(":")
+                    stop, _, step = stop.partition(":")
+                    start = int(start) if start else None
+                    stop = int(stop) if stop else None
+                    step = int(step) if step else None
+                    key = slice(start, stop, step)
+            except TypeError:
+                pass  # key is an integer
+
+        funcs.append(func(key))
+
+    return first, funcs
+
+
+def parse_format_spec(format_spec, conversion):
+    fmt = build_format_func(format_spec)
+    if not conversion:
+        return fmt
+
+    global _CONVERSIONS
+    if _CONVERSIONS is None:
+        _CONVERSIONS = {
+            "l": str.lower,
+            "u": str.upper,
+            "c": str.capitalize,
+            "C": string.capwords,
+            "j": json.dumps,
+            "t": str.strip,
+            "T": util.to_timestamp,
+            "d": text.parse_timestamp,
+            "U": text.unescape,
+            "S": util.to_string,
+            "s": str,
+            "r": repr,
+            "a": ascii,
+        }
+
+    conversion = _CONVERSIONS[conversion]
+    if fmt is format:
+        return conversion
+    else:
+        def chain(obj):
+            return fmt(conversion(obj))
+        return chain
+
+
+def build_format_func(format_spec):
+    if format_spec:
+        fmt = format_spec[0]
+        if fmt == "?":
+            return _parse_optional(format_spec)
+        if fmt == "L":
+            return _parse_maxlen(format_spec)
+        if fmt == "J":
+            return _parse_join(format_spec)
+        if fmt == "R":
+            return _parse_replace(format_spec)
+        return _default_format(format_spec)
+    return format
+
+
+def _parse_optional(format_spec):
+    before, after, format_spec = format_spec.split("/", 2)
+    before = before[1:]
+    fmt = build_format_func(format_spec)
+
+    def optional(obj):
+        return before + fmt(obj) + after if obj else ""
+    return optional
+
+
+def _parse_maxlen(format_spec):
+    maxlen, replacement, format_spec = format_spec.split("/", 2)
+    maxlen = text.parse_int(maxlen[1:])
+    fmt = build_format_func(format_spec)
+
+    def mlen(obj):
+        obj = fmt(obj)
+        return obj if len(obj) <= maxlen else replacement
+    return mlen
+
+
+def _parse_join(format_spec):
+    separator, _, format_spec = format_spec.partition("/")
+    separator = separator[1:]
+    fmt = build_format_func(format_spec)
+
+    def join(obj):
+        return fmt(separator.join(obj))
+    return join
+
+
+def _parse_replace(format_spec):
+    old, new, format_spec = format_spec.split("/", 2)
+    old = old[1:]
+    fmt = build_format_func(format_spec)
+
+    def replace(obj):
+        return fmt(obj.replace(old, new))
+    return replace
+
+
+def _default_format(format_spec):
+    def wrap(obj):
+        return format(obj, format_spec)
+    return wrap

gallery_dl/job.py

@@ -15,7 +15,7 @@ import operator
 import functools
 import collections
 from . import extractor, downloader, postprocessor
-from . import config, text, util, output, exception
+from . import config, text, util, formatter, output, exception
 from .extractor.message import Message
 
 
@@ -409,7 +409,7 @@ class DownloadJob(Job):
             path = util.expand_path(archive)
             try:
                 if "{" in path:
-                    path = util.Formatter(path).format_map(kwdict)
+                    path = formatter.parse(path).format_map(kwdict)
                 self.archive = util.DownloadArchive(path, self.extractor)
             except Exception as exc:
                 self.extractor.log.warning(

gallery_dl/output.py

@@ -11,7 +11,7 @@ import sys
 import shutil
 import logging
 import unicodedata
-from . import config, util
+from . import config, util, formatter
 
 
 # --------------------------------------------------------------------
@@ -92,13 +92,13 @@ class Formatter(logging.Formatter):
         if isinstance(fmt, dict):
             for key in ("debug", "info", "warning", "error"):
                 value = fmt[key] if key in fmt else LOG_FORMAT
-                fmt[key] = (util.Formatter(value).format_map,
+                fmt[key] = (formatter.parse(value).format_map,
                             "{asctime" in value)
         else:
             if fmt == LOG_FORMAT:
                 fmt = (fmt.format_map, False)
             else:
-                fmt = (util.Formatter(fmt).format_map, "{asctime" in fmt)
+                fmt = (formatter.parse(fmt).format_map, "{asctime" in fmt)
             fmt = {"debug": fmt, "info": fmt, "warning": fmt, "error": fmt}
 
         self.formats = fmt

gallery_dl/postprocessor/exec.py

@@ -9,7 +9,7 @@
 """Execute processes"""
 
 from .common import PostProcessor
-from .. import util
+from .. import util, formatter
 import subprocess
 
 
@@ -33,7 +33,7 @@ class ExecPP(PostProcessor):
             self.args = args
             execute = self.exec_string
         else:
-            self.args = [util.Formatter(arg) for arg in args]
+            self.args = [formatter.parse(arg) for arg in args]
             execute = self.exec_list
 
         events = options.get("event")

gallery_dl/postprocessor/metadata.py

@@ -9,7 +9,7 @@
 """Write metadata to external files"""
 
 from .common import PostProcessor
-from .. import util
+from .. import util, formatter
 import os
 
 
@@ -24,7 +24,7 @@ class MetadataPP(PostProcessor):
             cfmt = options.get("content-format") or options.get("format")
             if isinstance(cfmt, list):
                 cfmt = "\n".join(cfmt) + "\n"
-            self._content_fmt = util.Formatter(cfmt).format_map
+            self._content_fmt = formatter.parse(cfmt).format_map
             ext = "txt"
         elif mode == "tags":
             self.write = self._write_tags
@@ -45,10 +45,10 @@ class MetadataPP(PostProcessor):
         extfmt = options.get("extension-format")
         if filename:
             self._filename = self._filename_custom
-            self._filename_fmt = util.Formatter(filename).format_map
+            self._filename_fmt = formatter.parse(filename).format_map
         elif extfmt:
             self._filename = self._filename_extfmt
-            self._extension_fmt = util.Formatter(extfmt).format_map
+            self._extension_fmt = formatter.parse(extfmt).format_map
         else:
             self.extension = options.get("extension", ext)
 

gallery_dl/util.py

@@ -15,18 +15,15 @@ import json
 import time
 import random
 import shutil
-import string
-import _string
 import sqlite3
 import binascii
 import datetime
-import operator
 import functools
 import itertools
 import urllib.parse
 from http.cookiejar import Cookie
 from email.utils import mktime_tz, parsedate_tz
-from . import text, exception
+from . import text, exception, formatter
 
 
 def bencode(num, alphabet="0123456789"):
@@ -563,244 +560,6 @@ class ExtendedUrl():
         return self.value
 
 
-class Formatter():
-    """Custom, extended version of string.Formatter
-
-    This string formatter implementation is a mostly performance-optimized
-    variant of the original string.Formatter class. Unnecessary features have
-    been removed (positional arguments, unused argument check) and new
-    formatting options have been added.
-
-    Extra Conversions:
-    - "l": calls str.lower on the target value
-    - "u": calls str.upper
-    - "c": calls str.capitalize
-    - "C": calls string.capwords
-    - "j". calls json.dumps
-    - "t": calls str.strip
-    - "d": calls text.parse_timestamp
-    - "U": calls urllib.parse.unquote
-    - "S": calls util.to_string()
-    - "T": calls util.to_timestamü()
-    - Example: {f!l} -> "example"; {f!u} -> "EXAMPLE"
-
-    Extra Format Specifiers:
-    - "?<before>/<after>/":
-        Adds <before> and <after> to the actual value if it evaluates to True.
-        Otherwise the whole replacement field becomes an empty string.
-        Example: {f:?-+/+-/} -> "-+Example+-" (if "f" contains "Example")
-                             -> ""            (if "f" is None, 0, "")
-
-    - "L<maxlen>/<replacement>/":
-        Replaces the output with <replacement> if its length (in characters)
-        exceeds <maxlen>. Otherwise everything is left as is.
-        Example: {f:L5/too long/} -> "foo"      (if "f" is "foo")
-                                  -> "too long" (if "f" is "foobar")
-
-    - "J<separator>/":
-        Joins elements of a list (or string) using <separator>
-        Example: {f:J - /} -> "a - b - c" (if "f" is ["a", "b", "c"])
-
-    - "R<old>/<new>/":
-        Replaces all occurrences of <old> with <new>
-        Example: {f:R /_/} -> "f_o_o_b_a_r" (if "f" is "f o o b a r")
-    """
-    CACHE = {}
-    CONVERSIONS = {
-        "l": str.lower,
-        "u": str.upper,
-        "c": str.capitalize,
-        "C": string.capwords,
-        "j": json.dumps,
-        "t": str.strip,
-        "T": to_timestamp,
-        "d": text.parse_timestamp,
-        "U": urllib.parse.unquote,
-        "S": to_string,
-        "s": str,
-        "r": repr,
-        "a": ascii,
-    }
-
-    def __init__(self, format_string, default=None):
-        self.default = default
-        key = (format_string, default)
-
-        try:
-            self.result, self.fields = self.CACHE[key]
-        except KeyError:
-            self.result = []
-            self.fields = []
-
-            for literal_text, field_name, format_spec, conv in \
-                    _string.formatter_parser(format_string):
-                if literal_text:
-                    self.result.append(literal_text)
-                if field_name:
-                    self.fields.append((
-                        len(self.result),
-                        self._field_access(field_name, format_spec, conv),
-                    ))
-                    self.result.append("")
-
-            self.CACHE[key] = (self.result, self.fields)
-
-        if len(self.result) == 1:
-            if self.fields:
-                self.format_map = self.fields[0][1]
-            else:
-                self.format_map = lambda _: format_string
-            del self.result, self.fields
-
-    def format_map(self, kwdict):
-        """Apply 'kwdict' to the initial format_string and return its result"""
-        result = self.result
-        for index, func in self.fields:
-            result[index] = func(kwdict)
-        return "".join(result)
-
-    def _field_access(self, field_name, format_spec, conversion):
-        fmt = self._parse_format_spec(format_spec, conversion)
-
-        if "|" in field_name:
-            return self._apply_list([
-                self._parse_field_name(fn)
-                for fn in field_name.split("|")
-            ], fmt)
-        else:
-            key, funcs = self._parse_field_name(field_name)
-            if funcs:
-                return self._apply(key, funcs, fmt)
-            return self._apply_simple(key, fmt)
-
-    @staticmethod
-    def _parse_field_name(field_name):
-        first, rest = _string.formatter_field_name_split(field_name)
-        funcs = []
-
-        for is_attr, key in rest:
-            if is_attr:
-                func = operator.attrgetter
-            else:
-                func = operator.itemgetter
-                try:
-                    if ":" in key:
-                        start, _, stop = key.partition(":")
-                        stop, _, step = stop.partition(":")
-                        start = int(start) if start else None
-                        stop = int(stop) if stop else None
-                        step = int(step) if step else None
-                        key = slice(start, stop, step)
-                except TypeError:
-                    pass  # key is an integer
-
-            funcs.append(func(key))
-
-        return first, funcs
-
-    def _parse_format_spec(self, format_spec, conversion):
-        fmt = self._build_format_func(format_spec)
-        if not conversion:
-            return fmt
-
-        conversion = self.CONVERSIONS[conversion]
-        if fmt is format:
-            return conversion
-        else:
-            def chain(obj):
-                return fmt(conversion(obj))
-            return chain
-
-    def _build_format_func(self, format_spec):
-        if format_spec:
-            fmt = format_spec[0]
-            if fmt == "?":
-                return self._parse_optional(format_spec)
-            if fmt == "L":
-                return self._parse_maxlen(format_spec)
-            if fmt == "J":
-                return self._parse_join(format_spec)
-            if fmt == "R":
-                return self._parse_replace(format_spec)
-            return self._default_format(format_spec)
-        return format
-
-    def _apply(self, key, funcs, fmt):
-        def wrap(kwdict):
-            try:
-                obj = kwdict[key]
-                for func in funcs:
-                    obj = func(obj)
-            except Exception:
-                obj = self.default
-            return fmt(obj)
-        return wrap
-
-    def _apply_simple(self, key, fmt):
-        def wrap(kwdict):
-            return fmt(kwdict[key] if key in kwdict else self.default)
-        return wrap
-
-    def _apply_list(self, lst, fmt):
-        def wrap(kwdict):
-            for key, funcs in lst:
-                try:
-                    obj = kwdict[key]
-                    for func in funcs:
-                        obj = func(obj)
-                    if obj:
-                        break
-                except Exception:
-                    pass
-            else:
-                obj = self.default
-            return fmt(obj)
-        return wrap
-
-    def _parse_optional(self, format_spec):
-        before, after, format_spec = format_spec.split("/", 2)
-        before = before[1:]
-        fmt = self._build_format_func(format_spec)
-
-        def optional(obj):
-            return before + fmt(obj) + after if obj else ""
-        return optional
-
-    def _parse_maxlen(self, format_spec):
-        maxlen, replacement, format_spec = format_spec.split("/", 2)
-        maxlen = text.parse_int(maxlen[1:])
-        fmt = self._build_format_func(format_spec)
-
-        def mlen(obj):
-            obj = fmt(obj)
-            return obj if len(obj) <= maxlen else replacement
-        return mlen
-
-    def _parse_join(self, format_spec):
-        separator, _, format_spec = format_spec.partition("/")
-        separator = separator[1:]
-        fmt = self._build_format_func(format_spec)
-
-        def join(obj):
-            return fmt(separator.join(obj))
-        return join
-
-    def _parse_replace(self, format_spec):
-        old, new, format_spec = format_spec.split("/", 2)
-        old = old[1:]
-        fmt = self._build_format_func(format_spec)
-
-        def replace(obj):
-            return fmt(obj.replace(old, new))
-        return replace
-
-    @staticmethod
-    def _default_format(format_spec):
-        def wrap(obj):
-            return format(obj, format_spec)
-        return wrap
-
-
 class PathFormat():
     EXTENSION_MAP = {
         "jpeg": "jpg",
@@ -821,13 +580,13 @@ class PathFormat():
             elif isinstance(filename_fmt, dict):
                 self.filename_conditions = [
                     (compile_expression(expr),
-                     Formatter(fmt, kwdefault).format_map)
+                     formatter.parse(fmt, kwdefault).format_map)
                     for expr, fmt in filename_fmt.items() if expr
                 ]
                 self.build_filename = self.build_filename_conditional
                 filename_fmt = filename_fmt.get("", extractor.filename_fmt)
 
-            self.filename_formatter = Formatter(
+            self.filename_formatter = formatter.parse(
                 filename_fmt, kwdefault).format_map
         except Exception as exc:
             raise exception.FilenameFormatError(exc)
@@ -839,7 +598,7 @@ class PathFormat():
             elif isinstance(directory_fmt, dict):
                 self.directory_conditions = [
                     (compile_expression(expr), [
-                        Formatter(fmt, kwdefault).format_map
+                        formatter.parse(fmt, kwdefault).format_map
                         for fmt in fmts
                     ])
                     for expr, fmts in directory_fmt.items() if expr
@@ -848,7 +607,7 @@ class PathFormat():
                 directory_fmt = directory_fmt.get("", extractor.directory_fmt)
 
             self.directory_formatters = [
-                Formatter(dirfmt, kwdefault).format_map
+                formatter.parse(dirfmt, kwdefault).format_map
                 for dirfmt in directory_fmt
             ]
         except Exception as exc:
@@ -1009,12 +768,12 @@ class PathFormat():
 
     def build_filename_conditional(self, kwdict):
         try:
-            for condition, formatter in self.filename_conditions:
+            for condition, fmt in self.filename_conditions:
                 if condition(kwdict):
                     break
             else:
-                formatter = self.filename_formatter
-            return self.clean_path(self.clean_segment(formatter(kwdict)))
+                fmt = self.filename_formatter
+            return self.clean_path(self.clean_segment(fmt(kwdict)))
         except Exception as exc:
             raise exception.FilenameFormatError(exc)
 
@@ -1025,8 +784,8 @@ class PathFormat():
         strip = self.strip
 
         try:
-            for formatter in self.directory_formatters:
-                segment = formatter(kwdict).strip()
+            for fmt in self.directory_formatters:
+                segment = fmt(kwdict).strip()
                 if strip:
                     # remove trailing dots and spaces (#647)
                     segment = segment.rstrip(strip)
@@ -1047,8 +806,8 @@ class PathFormat():
                     break
             else:
                 formatters = self.directory_formatters
-            for formatter in formatters:
-                segment = formatter(kwdict).strip()
+            for fmt in formatters:
+                segment = fmt(kwdict).strip()
                 if strip:
                     segment = segment.rstrip(strip)
                 if segment:

scripts/run_tests.sh

@@ -2,7 +2,7 @@
 
 DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
 
-TESTS_CORE=(cache config cookies downloader extractor job oauth output postprocessor text util)
+TESTS_CORE=(cache config cookies downloader extractor formatter job oauth output postprocessor text util)
 TESTS_RESULTS=(results)
 
 

test/test_formatter.py

@@ -0,0 +1,183 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+# Copyright 2021 Mike Fährmann
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+
+import os
+import sys
+import unittest
+import datetime
+
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+from gallery_dl import formatter  # noqa E402
+
+
+class TestFormatter(unittest.TestCase):
+
+    kwdict = {
+        "a": "hElLo wOrLd",
+        "b": "äöü",
+        "d": {"a": "foo", "b": 0, "c": None},
+        "l": ["a", "b", "c"],
+        "n": None,
+        "s": " \n\r\tSPACE    ",
+        "u": "'< / >'",
+        "t": 1262304000,
+        "dt": datetime.datetime(2010, 1, 1),
+        "name": "Name",
+        "title1": "Title",
+        "title2": "",
+        "title3": None,
+        "title4": 0,
+    }
+
+    def test_conversions(self):
+        self._run_test("{a!l}", "hello world")
+        self._run_test("{a!u}", "HELLO WORLD")
+        self._run_test("{a!c}", "Hello world")
+        self._run_test("{a!C}", "Hello World")
+        self._run_test("{s!t}", "SPACE")
+        self._run_test("{a!U}", self.kwdict["a"])
+        self._run_test("{u!U}", "'< / >'")
+        self._run_test("{a!s}", self.kwdict["a"])
+        self._run_test("{a!r}", "'" + self.kwdict["a"] + "'")
+        self._run_test("{a!a}", "'" + self.kwdict["a"] + "'")
+        self._run_test("{b!a}", "'\\xe4\\xf6\\xfc'")
+        self._run_test("{a!S}", self.kwdict["a"])
+        self._run_test("{l!S}", "a, b, c")
+        self._run_test("{n!S}", "")
+        self._run_test("{t!d}", datetime.datetime(2010, 1, 1))
+        self._run_test("{t!d:%Y-%m-%d}", "2010-01-01")
+        self._run_test("{dt!T}", "1262304000")
+        self._run_test("{l!j}", '["a", "b", "c"]')
+
+        with self.assertRaises(KeyError):
+            self._run_test("{a!q}", "hello world")
+
+    def test_optional(self):
+        self._run_test("{name}{title1}", "NameTitle")
+        self._run_test("{name}{title1:?//}", "NameTitle")
+        self._run_test("{name}{title1:? **/''/}", "Name **Title''")
+
+        self._run_test("{name}{title2}", "Name")
+        self._run_test("{name}{title2:?//}", "Name")
+        self._run_test("{name}{title2:? **/''/}", "Name")
+
+        self._run_test("{name}{title3}", "NameNone")
+        self._run_test("{name}{title3:?//}", "Name")
+        self._run_test("{name}{title3:? **/''/}", "Name")
+
+        self._run_test("{name}{title4}", "Name0")
+        self._run_test("{name}{title4:?//}", "Name")
+        self._run_test("{name}{title4:? **/''/}", "Name")
+
+    def test_missing(self):
+        replacement = "None"
+
+        self._run_test("{missing}", replacement)
+        self._run_test("{missing.attr}", replacement)
+        self._run_test("{missing[key]}", replacement)
+        self._run_test("{missing:?a//}", "")
+
+        self._run_test("{name[missing]}", replacement)
+        self._run_test("{name[missing].attr}", replacement)
+        self._run_test("{name[missing][key]}", replacement)
+        self._run_test("{name[missing]:?a//}", "")
+
+    def test_missing_custom_default(self):
+        replacement = default = "foobar"
+        self._run_test("{missing}"     , replacement, default)
+        self._run_test("{missing.attr}", replacement, default)
+        self._run_test("{missing[key]}", replacement, default)
+        self._run_test("{missing:?a//}", "a" + default, default)
+
+    def test_alternative(self):
+        self._run_test("{a|z}"    , "hElLo wOrLd")
+        self._run_test("{z|a}"    , "hElLo wOrLd")
+        self._run_test("{z|y|a}"  , "hElLo wOrLd")
+        self._run_test("{z|y|x|a}", "hElLo wOrLd")
+        self._run_test("{z|n|a|y}", "hElLo wOrLd")
+
+        self._run_test("{z|a!C}"      , "Hello World")
+        self._run_test("{z|a:Rh/C/}"  , "CElLo wOrLd")
+        self._run_test("{z|a!C:RH/C/}", "Cello World")
+        self._run_test("{z|y|x:?</>/}", "")
+
+        self._run_test("{d[c]|d[b]|d[a]}", "foo")
+        self._run_test("{d[a]|d[b]|d[c]}", "foo")
+        self._run_test("{d[z]|d[y]|d[x]}", "None")
+
+    def test_indexing(self):
+        self._run_test("{l[0]}" , "a")
+        self._run_test("{a[6]}" , "w")
+
+    def test_slicing(self):
+        v = self.kwdict["a"]
+        self._run_test("{a[1:10]}"  , v[1:10])
+        self._run_test("{a[-10:-1]}", v[-10:-1])
+        self._run_test("{a[5:]}" , v[5:])
+        self._run_test("{a[50:]}", v[50:])
+        self._run_test("{a[:5]}" , v[:5])
+        self._run_test("{a[:50]}", v[:50])
+        self._run_test("{a[:]}"  , v)
+        self._run_test("{a[1:10:2]}"  , v[1:10:2])
+        self._run_test("{a[-10:-1:2]}", v[-10:-1:2])
+        self._run_test("{a[5::2]}" , v[5::2])
+        self._run_test("{a[50::2]}", v[50::2])
+        self._run_test("{a[:5:2]}" , v[:5:2])
+        self._run_test("{a[:50:2]}", v[:50:2])
+        self._run_test("{a[::]}"   , v)
+
+    def test_maxlen(self):
+        v = self.kwdict["a"]
+        self._run_test("{a:L5/foo/}" , "foo")
+        self._run_test("{a:L50/foo/}", v)
+        self._run_test("{a:L50/foo/>50}", " " * 39 + v)
+        self._run_test("{a:L50/foo/>51}", "foo")
+        self._run_test("{a:Lab/foo/}", "foo")
+
+    def test_join(self):
+        self._run_test("{l:J}"       , "abc")
+        self._run_test("{l:J,}"      , "a,b,c")
+        self._run_test("{l:J,/}"     , "a,b,c")
+        self._run_test("{l:J,/>20}"  , "               a,b,c")
+        self._run_test("{l:J - }"    , "a - b - c")
+        self._run_test("{l:J - /}"   , "a - b - c")
+        self._run_test("{l:J - />20}", "           a - b - c")
+
+        self._run_test("{a:J/}"      , self.kwdict["a"])
+        self._run_test("{a:J, /}"    , ", ".join(self.kwdict["a"]))
+
+    def test_replace(self):
+        self._run_test("{a:Rh/C/}"  , "CElLo wOrLd")
+        self._run_test("{a!l:Rh/C/}", "Cello world")
+        self._run_test("{a!u:Rh/C/}", "HELLO WORLD")
+
+        self._run_test("{a!l:Rl/_/}", "he__o wor_d")
+        self._run_test("{a!l:Rl//}" , "heo word")
+        self._run_test("{name:Rame/othing/}", "Nothing")
+
+    def test_chain_special(self):
+        # multiple replacements
+        self._run_test("{a:Rh/C/RE/e/RL/l/}", "Cello wOrld")
+        self._run_test("{d[b]!s:R1/Q/R2/A/R0/Y/}", "Y")
+
+        # join-and-replace
+        self._run_test("{l:J-/Rb/E/}", "a-E-c")
+
+        # optional-and-maxlen
+        self._run_test("{d[a]:?</>/L1/too long/}", "<too long>")
+        self._run_test("{d[c]:?</>/L5/too long/}", "")
+
+    def _run_test(self, format_string, result, default=None):
+        fmt = formatter.parse(format_string, default)
+        output = fmt.format_map(self.kwdict)
+        self.assertEqual(output, result, format_string)
+
+
+if __name__ == '__main__':
+    unittest.main()

test/test_util.py

@@ -260,169 +260,6 @@ class TestCookiesTxt(unittest.TestCase):
         )
 
 
-class TestFormatter(unittest.TestCase):
-
-    kwdict = {
-        "a": "hElLo wOrLd",
-        "b": "äöü",
-        "d": {"a": "foo", "b": 0, "c": None},
-        "l": ["a", "b", "c"],
-        "n": None,
-        "s": " \n\r\tSPACE    ",
-        "u": "%27%3C%20/%20%3E%27",
-        "t": 1262304000,
-        "dt": datetime.datetime(2010, 1, 1),
-        "name": "Name",
-        "title1": "Title",
-        "title2": "",
-        "title3": None,
-        "title4": 0,
-    }
-
-    def test_conversions(self):
-        self._run_test("{a!l}", "hello world")
-        self._run_test("{a!u}", "HELLO WORLD")
-        self._run_test("{a!c}", "Hello world")
-        self._run_test("{a!C}", "Hello World")
-        self._run_test("{s!t}", "SPACE")
-        self._run_test("{a!U}", self.kwdict["a"])
-        self._run_test("{u!U}", "'< / >'")
-        self._run_test("{a!s}", self.kwdict["a"])
-        self._run_test("{a!r}", "'" + self.kwdict["a"] + "'")
-        self._run_test("{a!a}", "'" + self.kwdict["a"] + "'")
-        self._run_test("{b!a}", "'\\xe4\\xf6\\xfc'")
-        self._run_test("{a!S}", self.kwdict["a"])
-        self._run_test("{l!S}", "a, b, c")
-        self._run_test("{n!S}", "")
-        self._run_test("{t!d}", datetime.datetime(2010, 1, 1))
-        self._run_test("{t!d:%Y-%m-%d}", "2010-01-01")
-        self._run_test("{dt!T}", "1262304000")
-        self._run_test("{l!j}", '["a", "b", "c"]')
-
-        with self.assertRaises(KeyError):
-            self._run_test("{a!q}", "hello world")
-
-    def test_optional(self):
-        self._run_test("{name}{title1}", "NameTitle")
-        self._run_test("{name}{title1:?//}", "NameTitle")
-        self._run_test("{name}{title1:? **/''/}", "Name **Title''")
-
-        self._run_test("{name}{title2}", "Name")
-        self._run_test("{name}{title2:?//}", "Name")
-        self._run_test("{name}{title2:? **/''/}", "Name")
-
-        self._run_test("{name}{title3}", "NameNone")
-        self._run_test("{name}{title3:?//}", "Name")
-        self._run_test("{name}{title3:? **/''/}", "Name")
-
-        self._run_test("{name}{title4}", "Name0")
-        self._run_test("{name}{title4:?//}", "Name")
-        self._run_test("{name}{title4:? **/''/}", "Name")
-
-    def test_missing(self):
-        replacement = "None"
-
-        self._run_test("{missing}", replacement)
-        self._run_test("{missing.attr}", replacement)
-        self._run_test("{missing[key]}", replacement)
-        self._run_test("{missing:?a//}", "")
-
-        self._run_test("{name[missing]}", replacement)
-        self._run_test("{name[missing].attr}", replacement)
-        self._run_test("{name[missing][key]}", replacement)
-        self._run_test("{name[missing]:?a//}", "")
-
-    def test_missing_custom_default(self):
-        replacement = default = "foobar"
-        self._run_test("{missing}"     , replacement, default)
-        self._run_test("{missing.attr}", replacement, default)
-        self._run_test("{missing[key]}", replacement, default)
-        self._run_test("{missing:?a//}", "a" + default, default)
-
-    def test_alternative(self):
-        self._run_test("{a|z}"    , "hElLo wOrLd")
-        self._run_test("{z|a}"    , "hElLo wOrLd")
-        self._run_test("{z|y|a}"  , "hElLo wOrLd")
-        self._run_test("{z|y|x|a}", "hElLo wOrLd")
-        self._run_test("{z|n|a|y}", "hElLo wOrLd")
-
-        self._run_test("{z|a!C}"      , "Hello World")
-        self._run_test("{z|a:Rh/C/}"  , "CElLo wOrLd")
-        self._run_test("{z|a!C:RH/C/}", "Cello World")
-        self._run_test("{z|y|x:?</>/}", "")
-
-        self._run_test("{d[c]|d[b]|d[a]}", "foo")
-        self._run_test("{d[a]|d[b]|d[c]}", "foo")
-        self._run_test("{d[z]|d[y]|d[x]}", "None")
-
-    def test_indexing(self):
-        self._run_test("{l[0]}" , "a")
-        self._run_test("{a[6]}" , "w")
-
-    def test_slicing(self):
-        v = self.kwdict["a"]
-        self._run_test("{a[1:10]}"  , v[1:10])
-        self._run_test("{a[-10:-1]}", v[-10:-1])
-        self._run_test("{a[5:]}" , v[5:])
-        self._run_test("{a[50:]}", v[50:])
-        self._run_test("{a[:5]}" , v[:5])
-        self._run_test("{a[:50]}", v[:50])
-        self._run_test("{a[:]}"  , v)
-        self._run_test("{a[1:10:2]}"  , v[1:10:2])
-        self._run_test("{a[-10:-1:2]}", v[-10:-1:2])
-        self._run_test("{a[5::2]}" , v[5::2])
-        self._run_test("{a[50::2]}", v[50::2])
-        self._run_test("{a[:5:2]}" , v[:5:2])
-        self._run_test("{a[:50:2]}", v[:50:2])
-        self._run_test("{a[::]}"   , v)
-
-    def test_maxlen(self):
-        v = self.kwdict["a"]
-        self._run_test("{a:L5/foo/}" , "foo")
-        self._run_test("{a:L50/foo/}", v)
-        self._run_test("{a:L50/foo/>50}", " " * 39 + v)
-        self._run_test("{a:L50/foo/>51}", "foo")
-        self._run_test("{a:Lab/foo/}", "foo")
-
-    def test_join(self):
-        self._run_test("{l:J}"       , "abc")
-        self._run_test("{l:J,}"      , "a,b,c")
-        self._run_test("{l:J,/}"     , "a,b,c")
-        self._run_test("{l:J,/>20}"  , "               a,b,c")
-        self._run_test("{l:J - }"    , "a - b - c")
-        self._run_test("{l:J - /}"   , "a - b - c")
-        self._run_test("{l:J - />20}", "           a - b - c")
-
-        self._run_test("{a:J/}"      , self.kwdict["a"])
-        self._run_test("{a:J, /}"    , ", ".join(self.kwdict["a"]))
-
-    def test_replace(self):
-        self._run_test("{a:Rh/C/}"  , "CElLo wOrLd")
-        self._run_test("{a!l:Rh/C/}", "Cello world")
-        self._run_test("{a!u:Rh/C/}", "HELLO WORLD")
-
-        self._run_test("{a!l:Rl/_/}", "he__o wor_d")
-        self._run_test("{a!l:Rl//}" , "heo word")
-        self._run_test("{name:Rame/othing/}", "Nothing")
-
-    def test_chain_special(self):
-        # multiple replacements
-        self._run_test("{a:Rh/C/RE/e/RL/l/}", "Cello wOrld")
-        self._run_test("{d[b]!s:R1/Q/R2/A/R0/Y/}", "Y")
-
-        # join-and-replace
-        self._run_test("{l:J-/Rb/E/}", "a-E-c")
-
-        # optional-and-maxlen
-        self._run_test("{d[a]:?</>/L1/too long/}", "<too long>")
-        self._run_test("{d[c]:?</>/L5/too long/}", "")
-
-    def _run_test(self, format_string, result, default=None):
-        formatter = util.Formatter(format_string, default)
-        output = formatter.format_map(self.kwdict)
-        self.assertEqual(output, result, format_string)
-
-
 class TestOther(unittest.TestCase):
 
     def test_bencode(self):