refactor(config): first pass of typing annotations for pkgcore.config.*

In a few spots (optionxform for example), I'm realigning
variable names w/ the class we're overriding.  No changes
visible for consuming code for that.

For the rest, I'm annotating the types, but this is the first step.
Specifically:
1) arg_type needs to be an enum (api change) to constrict the allowed values
2) multiple protocols need to be created to enumerate the expected interfaces
   for what's being passed in (central/config manager for example)
3) this first pass of type annotations is already flushing out code impedence.
   See pkgcore/ferringb#41 for an example.

Signed-off-by: Brian Harring <ferringb@gmail.com>
Signed-off-by: Arthur Zamarin <arthurzam@gentoo.org>

4 files changed, 123 insertions, 74 deletions

diff --git a/src/pkgcore/config/__init__.py b/src/pkgcore/config/__init__.py
index 50ac571b..83e58a56 100644
--- a/src/pkgcore/config/__init__.py
+++ b/src/pkgcore/config/__init__.py
@@ -7,21 +7,22 @@ __all__ = ("load_config",)
 # actually needed
 
 import os
+import typing
 
 from .. import const
 from . import central, cparser
 
 
 def load_config(
-    user_conf_file=const.USER_CONF_FILE,
-    system_conf_file=const.SYSTEM_CONF_FILE,
-    debug=False,
+    user_conf_file: typing.Optional[str] = const.USER_CONF_FILE,
+    system_conf_file: typing.Optional[str] = const.SYSTEM_CONF_FILE,
+    debug: bool = False,
     prepend_sources=(),
-    skip_config_files=False,
-    profile_override=None,
-    location=None,
+    skip_config_files: bool = False,
+    profile_override: typing.Optional[str] = None,
+    location: typing.Optional[str] = None,
     **kwargs
-):
+) -> central.CompatConfigManager:
     """The main entry point for any code looking to use pkgcore.
 
     Args:
@@ -29,7 +30,7 @@ def load_config(
         system_conf_file (optional[str]): system pkgcore config file path
         profile_override (optional[str]): targeted profile instead of system setting
         location (optional[str]): path to pkgcore config file or portage config directory
-        skip_config_files (optional[str]): don't attempt to load any config files
+        skip_config_files (bool): don't attempt to load any config files
 
     Returns:
         :obj:`pkgcore.config.central.ConfigManager` instance: system config
diff --git a/src/pkgcore/config/basics.py b/src/pkgcore/config/basics.py
index 0ff78c78..7576e718 100644
--- a/src/pkgcore/config/basics.py
+++ b/src/pkgcore/config/basics.py
@@ -22,6 +22,7 @@ __all__ = (
     "parse_config_file",
 )
 
+import typing
 from functools import partial
 
 from snakeoil import modules
@@ -50,7 +51,13 @@ class ConfigType:
     :ivar allow_unknowns: controls whether unknown settings should error.
     """
 
-    def __init__(self, func_obj):
+    callable: typing.Callable
+    types: dict[str, str]
+    positional: tuple[str]
+    required: tuple[str]
+    allow_unknowns: bool
+
+    def __init__(self, func_obj: typing.Callable) -> None:
         """Create from a callable (function, member function, class).
 
         It uses the defaults to determine type:
@@ -77,6 +84,8 @@ class ConfigType:
         # need without it. Most of the code in its getargs function
         # deals with tuples inside argument definitions, which we do
         # not support anyway.
+        #
+        # TODO: use the inspect module, speed is less of an issue in 2023.
         self.types = {}
 
         varargs, args, defaults, varkw = (), (), (), ()
@@ -165,7 +174,9 @@ class ConfigType:
 class LazySectionRef:
     """Abstract base class for lazy-loaded section references."""
 
-    def __init__(self, central, typename):
+    typename: str
+
+    def __init__(self, central, typename: str) -> None:
         self.central = central
         self.typename = typename.split(":", 1)[1]
         self.cached_config = None
@@ -185,13 +196,15 @@ class LazySectionRef:
                 )
         return self.cached_config
 
-    def instantiate(self):
+    def instantiate(self) -> typing.Any:
         """Convenience method returning the instantiated section."""
         return self.collapse().instantiate()
 
 
 class LazyNamedSectionRef(LazySectionRef):
-    def __init__(self, central, typename, name):
+    name: str
+
+    def __init__(self, central, typename: str, name: str) -> None:
         super().__init__(central, typename)
         self.name = name
 
@@ -200,7 +213,7 @@ class LazyNamedSectionRef(LazySectionRef):
 
 
 class LazyUnnamedSectionRef(LazySectionRef):
-    def __init__(self, central, typename, section):
+    def __init__(self, central, typename: str, section) -> None:
         super().__init__(central, typename)
         self.section = section
 
@@ -215,15 +228,15 @@ class ConfigSection:
     be an Interface.
     """
 
-    def __contains__(self, name):
+    def __contains__(self, name: str) -> bool:
         """Check if a key is in this section."""
         raise NotImplementedError(self.__contains__)
 
-    def keys(self):
+    def keys(self) -> list[str]:
         """Return a list of keys."""
         raise NotImplementedError(self.keys)
 
-    def render_value(self, central, name, arg_type):
+    def render_value(self, central, name: str, arg_type):
         """Return a setting, converted to the requested type."""
         raise NotImplementedError(self, "render_value")
 
@@ -231,7 +244,10 @@ class ConfigSection:
 class DictConfigSection(ConfigSection):
     """Turns a dict and a conversion function into a ConfigSection."""
 
-    def __init__(self, conversion_func, source_dict):
+    func: typing.Callable
+    dict: dict[str, typing.Any]
+
+    def __init__(self, conversion_func, source_dict: dict[str, typing.Any]) -> None:
         """Initialize.
 
         :type conversion_func: callable.
@@ -243,13 +259,13 @@ class DictConfigSection(ConfigSection):
         self.func = conversion_func
         self.dict = source_dict
 
-    def __contains__(self, name):
+    def __contains__(self, name: str) -> bool:
         return name in self.dict
 
-    def keys(self):
+    def keys(self) -> list[str]:
         return list(self.dict.keys())
 
-    def render_value(self, central, name, arg_type):
+    def render_value(self, central, name: str, arg_type: str):
         try:
             return self.func(central, self.dict[name], arg_type)
         except IGNORED_EXCEPTIONS:
@@ -263,7 +279,12 @@ class DictConfigSection(ConfigSection):
 class FakeIncrementalDictConfigSection(ConfigSection):
     """Turns a dict and a conversion function into a ConfigSection."""
 
-    def __init__(self, conversion_func, source_dict):
+    func: typing.Callable
+    dict: dict[str, typing.Any]
+
+    def __init__(
+        self, conversion_func: typing.Callable, source_dict: dict[str, typing.Any]
+    ) -> None:
         """Initialize.
 
         A request for a section of a list type will look for
@@ -281,14 +302,14 @@ class FakeIncrementalDictConfigSection(ConfigSection):
         self.func = conversion_func
         self.dict = source_dict
 
-    def __contains__(self, name):
+    def __contains__(self, name: str) -> bool:
         return (
             name in self.dict
             or name + ".append" in self.dict
             or name + ".prepend" in self.dict
         )
 
-    def keys(self):
+    def keys(self) -> list[str]:
         keys = set()
         for key in self.dict:
             if key.endswith(".append"):
@@ -298,7 +319,7 @@ class FakeIncrementalDictConfigSection(ConfigSection):
             keys.add(key)
         return list(keys)
 
-    def render_value(self, central, name, arg_type):
+    def render_value(self, central, name: str, arg_type: str):
         # Check if we need our special incremental magic.
         if arg_type in ("list", "str", "repr") or arg_type.startswith("refs:"):
             result = []
@@ -418,8 +439,9 @@ class FakeIncrementalDictConfigSection(ConfigSection):
             ) from e
 
 
-def str_to_list(string):
+def str_to_list(string: str) -> list[str]:
     """Split on whitespace honoring quoting for new tokens."""
+    # TODO: replace this with shlex or equivalent parsing.
     l = []
     i = 0
     e = len(string)
@@ -455,15 +477,16 @@ def str_to_list(string):
     return l
 
 
-def str_to_str(string):
+def str_to_str(string: str) -> str:
     """Yank leading/trailing whitespace and quotation, along with newlines."""
+    # TODO: replace these with shlex
     s = string.strip()
     if len(s) > 1 and s[0] in "\"'" and s[0] == s[-1]:
         s = s[1:-1]
     return s.replace("\n", " ").replace("\t", " ")
 
 
-def str_to_bool(string):
+def str_to_bool(string: str) -> bool:
     """Convert a string to a boolean."""
     s = str_to_str(string).lower()
     if s in ("no", "false", "0"):
@@ -473,7 +496,7 @@ def str_to_bool(string):
     raise errors.ConfigurationError(f"{s!r} is not a boolean")
 
 
-def str_to_int(string):
+def str_to_int(string: str) -> int:
     """Convert a string to a integer."""
     string = str_to_str(string)
     try:
@@ -490,7 +513,7 @@ _str_converters = {
 }
 
 
-def convert_string(central, value, arg_type):
+def convert_string(central, value, arg_type: str):
     """Conversion func for a string-based DictConfigSection."""
     if not isinstance(value, str):
         raise ValueError(
@@ -519,7 +542,7 @@ def convert_string(central, value, arg_type):
     return func(value)
 
 
-def convert_asis(central, value, arg_type):
+def convert_asis(central, value, arg_type: str):
     """ "Conversion" func assuming the types are already correct."""
     if arg_type == "callable":
         if not callable(value):
@@ -558,7 +581,7 @@ def convert_asis(central, value, arg_type):
     return value
 
 
-def convert_hybrid(central, value, arg_type):
+def convert_hybrid(central, value, arg_type: str):
     """Automagically switch between :obj:`convert_string` and :obj:`convert_asis`.
 
     :obj:`convert_asis` is used for arg_type str and if value is not a string.
@@ -579,7 +602,7 @@ ConfigSectionFromStringDict = partial(FakeIncrementalDictConfigSection, convert_
 AutoConfigSection = partial(FakeIncrementalDictConfigSection, convert_hybrid)
 
 
-def section_alias(target, typename):
+def section_alias(target, typename: str) -> AutoConfigSection:
     """Build a ConfigSection that instantiates a named reference.
 
     Because of central's caching our instantiated value will be
@@ -594,11 +617,11 @@ def section_alias(target, typename):
 
 
 @configurable(types={"path": "str", "parser": "callable"}, typename="configsection")
-def parse_config_file(path, parser):
+def parse_config_file(path: str, parser):
     try:
         f = open(path, "r")
-    except (IOError, OSError):
-        raise errors.InstantiationError(f"failed opening {path!r}")
+    except (IOError, OSError) as e:
+        raise errors.InstantiationError(f"failed opening {path!r}") from e
     try:
         return parser(f)
     finally:
@@ -614,7 +637,7 @@ class ConfigSource:
 
 
 class GeneratedConfigSource(ConfigSource):
-    def __init__(self, section_data, description):
+    def __init__(self, section_data, description: str) -> None:
         self.description = description
         self.section_data = section_data
 
diff --git a/src/pkgcore/config/central.py b/src/pkgcore/config/central.py
index 12853836..53609221 100644
--- a/src/pkgcore/config/central.py
+++ b/src/pkgcore/config/central.py
@@ -33,18 +33,20 @@ class _ConfigMapping(mappings.DictMixin):
     any of them are remote!
     """
 
-    def __init__(self, manager, typename):
+    typename: str
+
+    def __init__(self, manager, typename: str) -> None:
         super().__init__()
         self.manager = manager
         self.typename = typename
 
-    def __getitem__(self, key):
+    def __getitem__(self, key: str) -> typing.Any:
         conf = self.manager.collapse_named_section(key, raise_on_missing=False)
         if conf is None or conf.type.name != self.typename:
             raise KeyError(key)
         return conf.instantiate()
 
-    def keys(self):
+    def keys(self) -> typing.Iterator[str]:
         for name in self.manager.sections():
             try:
                 collapsed = self.manager.collapse_named_section(name)
@@ -57,26 +59,31 @@ class _ConfigMapping(mappings.DictMixin):
                 if collapsed.type.name == self.typename:
                     yield name
 
-    def __contains__(self, key):
+    def __contains__(self, key: str) -> bool:
         conf = self.manager.collapse_named_section(key, raise_on_missing=False)
         return conf is not None and conf.type.name == self.typename
 
 
-class _ConfigStack(defaultdict):
-    def __init__(self):
+class _ConfigStack(defaultdict[str, list[typing.Any]]):
+    def __init__(self) -> None:
         super().__init__(list)
 
-    def render_vals(self, manager, key, type_name):
+    def render_vals(
+        self, manager, key: str, type_name: str
+    ) -> typing.Iterator[typing.Any]:
         for data in self.get(key, ()):
             if key in data.section:
                 yield data.section.render_value(manager, key, type_name)
 
-    def render_val(self, manager, key, type_name):
+    def render_val(
+        self, manager, key: str, type_name: str
+    ) -> typing.Optional[typing.Any]:
         for val in self.render_vals(manager, key, type_name):
             return val
         return None
 
     def render_prepends(self, manager, key: str, type_name: str) -> list[typing.Any]:
+
         results = []
         # keep in mind that the sequence we get is a top -> bottom walk of the config
         # as such for this operation we have to reverse it when building the content-
@@ -96,8 +103,8 @@ class _ConfigStack(defaultdict):
                 results += [append]
 
         if type_name != "str":
-            results = chain.from_iterable(results)
-        return list(results)
+            results = list(chain.from_iterable(results))
+        return results
 
 
 class CollapsedConfig:
@@ -114,7 +121,19 @@ class CollapsedConfig:
     :ivar name: our section name or C{None} for an anonymous section.
     """
 
-    def __init__(self, type_obj, config, manager, debug=False, default=False):
+    type_obj: basics.ConfigType
+    config: dict[str, typing.Any]
+    debug: bool
+    default: bool
+
+    def __init__(
+        self,
+        type_obj: basics.ConfigType,
+        config: dict[str, typing.Any],
+        manager,
+        debug: bool = False,
+        default: bool = False,
+    ) -> None:
         """Initialize instance vars."""
         # Check if we got all values required to instantiate.
         missing = set(type_obj.required) - set(config)
@@ -137,7 +156,7 @@ class CollapsedConfig:
             manager = weakref.ref(manager)
         self.manager = manager
 
-    def instantiate(self):
+    def instantiate(self) -> typing.Any:
         if self._instance is None:
             try:
                 self._instance = self._instantiate()
@@ -147,7 +166,7 @@ class CollapsedConfig:
                 raise errors.InstantiationError(self.name) from e
         return self._instance
 
-    def _instantiate(self):
+    def _instantiate(self) -> typing.Any:
         """Call our type's callable, cache and return the result.
 
         Calling instantiate more than once will return the cached value.
@@ -232,13 +251,13 @@ class CollapsedConfig:
 
         return self._instance
 
-    def __getstate__(self):
+    def __getstate__(self) -> dict[str, typing.Any]:
         d = self.__dict__.copy()
         # pull actual value from weakref
         d["manager"] = d["manager"]()
         return d
 
-    def __setstate__(self, state):
+    def __setstate__(self, state: dict[str, typing.Any]) -> None:
         self.__dict__ = state.copy()
         # reset weakref
         self.__dict__["manager"] = weakref.ref(self.__dict__["manager"])
@@ -248,29 +267,29 @@ class _ConfigObjMap:
     def __init__(self, manager):
         self._manager = manager
 
-    def __getattr__(self, attr):
+    def __getattr__(self, attr: str) -> typing.Any:
         return _ConfigMapping(self._manager, attr)
 
-    def __getitem__(self, key):
+    def __getitem__(self, key: str) -> typing.Any:
         val = getattr(self._manager.objects, key, klass.sentinel)
         if val is None:
             raise KeyError(key)
         return val
 
-    def __getstate__(self):
+    def __getstate__(self) -> dict[str, typing.Any]:
         # Explicitly defined to force pickling to work as expected without
         # trying to pull __getstate__ from _ConfigMapping due to __getattr__.
         return self.__dict__.copy()
 
-    def __setstate__(self, state):
+    def __setstate__(self, state: dict[str, typing.Any]) -> None:
         self.__dict__.update(state)
 
 
 class CompatConfigManager:
-    def __init__(self, manager):
+    def __init__(self, manager) -> None:
         self._manager = manager
 
-    def __getattr__(self, attr):
+    def __getattr__(self, attr: str) -> typing.Any:
         if attr == "_manager":
             return object.__getattribute__(self, "_manager")
         obj = getattr(self._manager, attr, klass.sentinel)
@@ -296,7 +315,7 @@ class ConfigManager:
     section with a name starting with "autoload".
     """
 
-    def __init__(self, configs=(), debug=False):
+    def __init__(self, configs=(), debug: bool = False):
         """Initialize.
 
         :type configs: sequence of mappings of string to ConfigSection.
@@ -319,7 +338,7 @@ class ConfigManager:
             return config
         return basics.GeneratedConfigSource(config, "unknown")
 
-    def reload(self):
+    def reload(self) -> None:
         """Reinitialize from the config sources originally passed in.
 
         This throws away all cached instances and re-executes autoloads.
@@ -336,15 +355,15 @@ class ConfigManager:
         for config in self.original_config_sources:
             self.add_config_source(config)
 
-    def update(self, config):
+    def update(self, config) -> None:
         """Reinitialize using an additional supplied config."""
         self.original_config_sources += (config,)
         self.reload()
 
-    def add_config_source(self, config):
-        return self._add_config_source(self._compat_mangle_config(config))
+    def add_config_source(self, config) -> None:
+        self._add_config_source(self._compat_mangle_config(config))
 
-    def _add_config_source(self, config):
+    def _add_config_source(self, config) -> None:
         """Pull extra type and config sections from configs and use them.
 
         Things loaded this way are added after already loaded things
@@ -398,11 +417,11 @@ class ConfigManager:
             if collapsed.type.name == "configsection":
                 self.add_config_source(instance)
 
-    def sections(self):
+    def sections(self) -> typing.Iterator[str]:
         """Return an iterator of all section names."""
         return iter(self.sections_lookup.keys())
 
-    def collapse_named_section(self, name, raise_on_missing=True):
+    def collapse_named_section(self, name: str, raise_on_missing: bool = True):
         """Collapse a config by name, possibly returning a cached instance.
 
         @returns: :obj:`CollapsedConfig`.
@@ -436,7 +455,7 @@ class ConfigManager:
         finally:
             self._refs.remove(name)
 
-    def _get_inherited_sections(self, name, sections):
+    def _get_inherited_sections(self, name: str, sections):
         # List of (name, ConfigSection, index) tuples, most specific first.
         slist = [(name, sections)]
 
@@ -479,13 +498,13 @@ class ConfigManager:
                     slist.append((inherit, target))
         return [_section_data(name, stack[0]) for (name, stack) in slist]
 
-    def _section_is_inherit_only(self, section):
+    def _section_is_inherit_only(self, section) -> bool:
         if "inherit-only" in section:
             if section.render_value(self, "inherit-only", "bool"):
                 return True
         return False
 
-    def collapse_section(self, sections, _name=None):
+    def collapse_section(self, sections, _name: typing.Optional[str] = None):
         """Collapse a ConfigSection to a :obj:`CollapsedConfig`."""
 
         if self._section_is_inherit_only(sections[0]):
@@ -518,7 +537,7 @@ class ConfigManager:
         return collapsed
 
     @klass.jit_attr
-    def types(self):
+    def types(self) -> dict[str, dict[str, typing.Any]]:
         type_map = defaultdict(dict)
         for name, sections in self.sections_lookup.items():
             if self._section_is_inherit_only(sections[0]):
@@ -529,7 +548,7 @@ class ConfigManager:
             (k, mappings.ImmutableDict(v)) for k, v in type_map.items()
         )
 
-    def _render_config_stack(self, type_obj, config_stack):
+    def _render_config_stack(self, type_obj, config_stack) -> dict[str, typing.Any]:
         conf = {}
         for key in config_stack:
             typename = type_obj.types.get(key)
@@ -547,6 +566,8 @@ class ConfigManager:
             if typename.startswith("refs:") or typename in ("list", "str"):
                 result = config_stack.render_prepends(self, key, typename)
                 if typename == "str":
+                    # TODO: figure out why this is needed and likely remove it.
+                    # it's likely just doing ' '.join([item])
                     result = " ".join(result)
             else:
                 result = config_stack.render_val(self, key, typename)
@@ -581,7 +602,7 @@ class ConfigManager:
 
         return mappings.ImmutableDict(conf)
 
-    def get_default(self, type_name):
+    def get_default(self, type_name: str) -> typing.Optional[typing.Any]:
         """Finds the configuration specified default obj of type_name.
 
         Returns C{None} if no defaults.
diff --git a/src/pkgcore/config/cparser.py b/src/pkgcore/config/cparser.py
index 7c49cc14..b578f4ec 100644
--- a/src/pkgcore/config/cparser.py
+++ b/src/pkgcore/config/cparser.py
@@ -5,6 +5,7 @@ ini based configuration format
 __all__ = ("config_from_file",)
 
 import configparser
+import typing
 
 from snakeoil import mappings
 
@@ -12,11 +13,14 @@ from . import basics, errors
 
 
 class CaseSensitiveConfigParser(configparser.ConfigParser):
-    def optionxform(self, val):
-        return val
+    """Parse to enforce case sensitivity for configparser"""
 
+    def optionxform(self, optionstr: str) -> str:
+        """preserve case sensitivity"""
+        return optionstr
 
-def config_from_file(file_obj):
+
+def config_from_file(file_obj: typing.Iterable[str]) -> mappings.LazyValDict:
     """
     generate a config dict