"""
API arguments and calls

>>> Setup.from_command_line("--update all --archives-from-proxy --sources bullseye --sources-from-origin Debian:lts --repositories-from-distributions test --chroots-from-sources").command_line()
'--update all --archives-from-proxy --sources bullseye --sources-from-origin Debian:lts --repositories-from-distributions test --chroots-from-sources'
"""

import abc
import os
import sys
import inspect
import collections
import logging
import argparse
import re
import glob
import shlex
import hashlib
import json
import threading
from contextlib import closing

import mini_buildd.misc
import mini_buildd.net
import mini_buildd.changes
import mini_buildd.package

from mini_buildd.dist import SETUP, Codename, Dist
from mini_buildd.values import OnOff, get_value
from mini_buildd.values import Name as Value

LOG = logging.getLogger(__name__)


class Argument():
    """
    Generic Argument Class

    ``value()`` always provides a non-``None`` value of the specific
    type, either the ``default`` or a ``given`` value.

    ``strvalue()`` always provides a non-``None`` ``str`` value.

    The ``default`` value is given in the constructor. For
    server-specific defaults, this may be function -- then the default
    value will be computed only at run time on the server.

    A ``given`` value can be provided via special ``set()`` method:
      * Empty ``str``, ``list`` or false ``bool`` will yield ``None``.
      * Non-empty ``str`` will be converted to the specific type.
      * Other given values will be used as is.

    ============  ============ ============ =========== =================
    Type          value() type svalue() ex. HTML GET    argparse
    ============  ============ ============ =========== =================
    Str           str          "string"     key=string  --key "string"
    Url           str          "http://.."  key=string  --key "string"
    MultilineStr  str          "long"       key=string  --key "string"
    Choice        str          "c0"         key=string  --key "string"
    Int           int          "17"         key=string  --key "int"
    Bool          bool         "True"       key=True    --key
    List          list of str  "v0,v1,.."   key=v0,v1.. --key "v0" "v1"..
    ============  ============ ============ =========== =================
    """

    #: Validate that values are actually of that type
    VALUE_TYPE = str

    #: Magic string value to use as value when a default callable on the server should be used.
    SERVER_DEFAULT = "<server_default>"

    def __init__(self, id_list, doc="Undocumented", default=None, choices=None, header=None):
        """
        :param id_list: List like '['--with-rollbacks', '-R']' for option or '['distributions']' for positionals; 1st entry always denotes the id.

        >>> Argument(["--long-id", "-s"]).identity
        'long_id'
        >>> Argument(["posi-tional"]).identity
        'posi_tional'
        """
        # Identifiers && doc
        self.id_list = id_list
        self.doc = doc
        self.is_positional = not id_list[0].startswith("--")
        # identity: 1st of id_list with leading '--' removed and hyphens turned to snake case
        self.identity = mini_buildd.misc.Snake(id_list[0][0 if self.is_positional else 2:]).from_kebab()

        # Values
        self._default = default
        self.given = None

        # Choices helper
        self._choices = choices
        self.header = header

    def __str__(self):
        """Be sure not to use value() here"""
        return f"{self.identity}: given={self.given}, _default={self._default}"

    @classmethod
    def s2v(cls, str_value):
        """Convert string to value"""
        return cls.VALUE_TYPE(str_value) if str_value else None

    @classmethod
    def v2s(cls, value):
        """Convert value to string"""
        return "" if value is None else str(value)

    def required(self):
        return self._default is None

    def needs_value(self):
        """If user input is no_value"""
        return self.given is None and self.required()

    def choices(self):
        return [] if self._choices is None else get_value(self._choices)

    def default(self):
        return get_value(self._default)

    def strdefault(self):
        return self.v2s(self.default())

    def value(self):
        if self.needs_value():
            raise mini_buildd.HTTPBadRequest(f"Missing required argument: {self}")
        return self.default() if self.given is None else self.given

    def strvalue(self):
        return self.v2s(self.value())

    def strgiven(self):
        return self.v2s(self.given)

    def icommand_line_given(self):
        yield self.strgiven()

    def set(self, given):
        if given == self.SERVER_DEFAULT:
            self.given = None
        elif isinstance(given, str):
            self.given = self.s2v(given)
        elif isinstance(given, (list, bool)):
            self.given = given if given else None
        elif isinstance(given, self.VALUE_TYPE):
            self.given = given
        else:
            raise Exception(f"API argument '{self.identity}': Invalid value {given} (type given {type(given)}, needs {self.VALUE_TYPE})")

    def argparse_kvsargs(self):
        """Python 'argparse' support"""
        kvsargs = {"help": self.doc}
        if self._default is not None:
            if isinstance(self._default, Value):
                kvsargs["default"] = self.SERVER_DEFAULT
                kvsargs["help"] += f"\nServer Default: {self._default.name}"
            else:
                kvsargs["default"] = self._default
        return kvsargs


class StrArgument(Argument):
    HTML_TYPE = "text"

    def argparse_kvsargs(self):
        return {**super().argparse_kvsargs(), **{"action": "store"}}


class UrlArgument(StrArgument):
    HTML_TYPE = "url"


class MultilineStrArgument(StrArgument):
    HTML_TYPE = "textarea"


class ChoiceArgument(Argument):
    HTML_TYPE = "select"

    def value(self):
        value = super().value()
        choices = self.choices()
        if value not in choices:
            raise mini_buildd.HTTPBadRequest(f"{self.identity}: Wrong choice argument: '{value}' not in {choices}")
        return value

    def argparse_kvsargs(self):
        kvsargs = super().argparse_kvsargs()
        kvsargs["choices"] = self._choices
        if self.is_positional and self._default is not None:
            kvsargs["nargs"] = "?"  # Allow positional args to be actually optional, if we have a default.
        return kvsargs


class IntArgument(StrArgument):
    VALUE_TYPE = int
    HTML_TYPE = "number"

    def argparse_kvsargs(self):
        return {**super().argparse_kvsargs(), **{"type": int}}


class BoolArgument(ChoiceArgument):
    VALUE_TYPE = bool
    HTML_TYPE = "checkbox"

    def __init__(self, *args, **kwargs):
        super().__init__(*args, choices=[True, False], default=False, **kwargs)

    @classmethod
    def s2v(cls, str_value):
        return str_value == str(True)

    def icommand_line_given(self):
        """Empty generator -- bools are just command line options like ``--with-foo``"""
        yield from []

    def argparse_kvsargs(self):
        kvsargs = Argument.argparse_kvsargs(self)
        kvsargs["action"] = "store_true"
        return kvsargs


class _ExtendAction(argparse._AppendAction):  # pylint: disable=protected-access
    """
    Customized argparse ``extend`` action from ``python 3.8``

    Allows "-L a -L b d -L c" to be combined to one flat list "a b c d".
    """

    def __call__(self, parser, namespace, values, option_string=None):
        items = getattr(namespace, self.dest, None)
        items = argparse._copy_items(items)
        if items == Argument.SERVER_DEFAULT:
            items = values
        else:
            items.extend(values)
        setattr(namespace, self.dest, items)


class ListArgument(StrArgument):
    VALUE_TYPE = list
    SEPARATOR = ","

    @classmethod
    def s2v(cls, str_value):
        return str_value.split(cls.SEPARATOR) if str_value else None

    @classmethod
    def v2s(cls, value):
        return "" if value is None else cls.SEPARATOR.join(value)

    def icommand_line_given(self):
        if self.given is not None:
            yield from self.given

    def argparse_kvsargs(self):
        kvsargs = super().argparse_kvsargs()
        kvsargs["nargs"] = "+"
        kvsargs["action"] = _ExtendAction
        if self._choices is not None and not isinstance(self._choices, Value):
            kvsargs["choices"] = self._choices
        kvsargs["metavar"] = mini_buildd.misc.singularize(self.identity.upper())
        return kvsargs


#: Specialized argument classes
class Repository(StrArgument):
    def __init__(self, id_list, **kwargs):
        super().__init__(id_list, choices=Value.ALL_REPOSITORIES, doc="Repository identity (like 'test', the default sandbox repo identity)", **kwargs)

    def object(self):
        return mini_buildd.mdls().repository.get(self.value())


class Repositories(ListArgument):
    def __init__(self, id_list, **kwargs):
        super().__init__(
            id_list,
            choices=Value.ALL_REPOSITORIES,
            doc="Repository identities (like 'test', the default sandbox repo identity)",
            **kwargs,
        )

    def objects(self):
        return mini_buildd.mdls().repository.Repository.objects.filter(identity__in=self.value())


class Codenames(ListArgument):
    def __init__(self, id_list, **kwargs):
        super().__init__(
            id_list,
            choices=Value.ALL_CODENAMES,
            doc="Codenames (like 'buster', 'bullseye')",
            **kwargs,
        )


def diststr2repository(diststr):
    return mini_buildd.mdls().repository.get(Dist(diststr).repository)


class Distribution(StrArgument):
    def __init__(self, id_list, choices=Value.ACTIVE_DISTRIBUTIONS, extra_doc="", **kwargs):
        super().__init__(id_list, choices=choices, doc="Distribution name (<codename>-<id>-<suite>[-rollback<n>])" + extra_doc, **kwargs)

    def dist(self):
        return mini_buildd.dist.Dist(self.value())


class Distributions(ListArgument):
    def __init__(self, id_list, choices=Value.ACTIVE_DISTRIBUTIONS, **kwargs):
        super().__init__(id_list, choices=choices, doc="Distribution names", **kwargs)


class Source(StrArgument):
    def __init__(self, id_list, extra_doc="", **kwargs):
        super().__init__(id_list, choices=Value.LAST_SOURCES, doc="Source package name" + extra_doc, **kwargs)


class UploadOptions(StrArgument):
    def __init__(self, id_list, **kwargs):
        super().__init__(id_list,
                         default="lintian-mode=ignore",
                         doc=("List of upload options, separated by '|':\n"
                              "\n"
                              f"{mini_buildd.changes.Upload.Options.usage()}"
                              ),
                         **kwargs)


class Output(ChoiceArgument):
    """Meta API call option 'output'"""

    def __init__(self):
        super().__init__(["output"], choices=["html", "json"], default="html",
                         doc=("Desired result format\n"
                              "\n"
                              "html: Return result as HTML (popup with JS, else full HTML page)\n"
                              "json: Return raw json result\n"
                              ))


class Rollbacks(IntArgument):
    def __init__(self, id_list, extra_doc="", **kwargs):
        super().__init__(id_list, default=0, doc="Select (maximal) <n> rollback sources (``0``: no rollbacks, ``< 0``: all rollbacks)" + extra_doc, **kwargs)

    def range(self, suite):
        """Get valid range for this suite"""
        rollbacks = self.value()
        if rollbacks < 0:
            rollbacks = suite.rollback
        return range(0, min(rollbacks, suite.rollback))


class Call():
    AUTH = mini_buildd.config.AUTH_NONE
    NEEDS_RUNNING_DAEMON = False
    CONFIRM = False
    CUSTOM_HTML_SNIPPET = None

    @classmethod
    def name(cls):
        return mini_buildd.misc.Snake(cls.__name__).from_camel()

    @classmethod
    def doc(cls):
        return inspect.getdoc(cls)

    @classmethod
    def doc_title(cls):
        return cls.doc().partition("\n")[0]

    @classmethod
    def doc_body(cls):
        return cls.doc().partition("\n")[2][1:]

    # Category support -- purely for structuring documentation.
    CATEGORIES = ["Consumer", "Developer", "Administrator"]

    @classmethod
    def category(cls):
        if cls.AUTH in [mini_buildd.config.AUTH_NONE, mini_buildd.config.AUTH_LOGIN]:
            return cls.CATEGORIES[0]
        if cls.AUTH in [mini_buildd.config.AUTH_STAFF]:
            return cls.CATEGORIES[1]
        return cls.CATEGORIES[2]

    @classmethod
    def uri(cls):
        return mini_buildd.config.URIS["api"]["view"].join(cls.name() + "/")

    @classmethod
    def iarguments(cls):
        yield from []

    @classmethod
    def from_sloppy_args(cls, **kwargs):
        """Construct ignoring any unknown arguments given"""
        keys = [a.identity for a in cls.iarguments()]
        return cls(**{key: value for key, value in kwargs.items() if key in keys})

    def set_args(self, **kwargs):
        for key, value in kwargs.items():
            if key not in self.args:
                raise mini_buildd.HTTPBadRequest(f"API call '{self.name()}': Unknown argument '{key}'")
            self.args[key].set(value)

        for key in (key for key in self.args if key in kwargs):
            self.args[key].set(kwargs.get(key))

    def __init__(self, **kwargs):
        self.args = {arg.identity: arg for arg in self.iarguments()}
        self.set_args(**kwargs)
        self.request = None
        self.result = {}  # json result

    @classmethod
    def from_command_line(cls, command_line):
        return cls(**cls.parse_command_line(command_line))

    def set_request(self, request):
        self.request = request

    @classmethod
    def get_plain(cls, result, force_json=False):
        """Get ``str`` result (non-``str`` results get json pretty-formatted)"""
        return mini_buildd.misc.json_pretty(result) if (force_json or not isinstance(result, str)) else result

    def plain(self, force_json=False):
        return self.get_plain(self.result, force_json=force_json)

    def json_pretty(self):
        """For (arg-less) use in templates only"""
        return self.plain(force_json=True)

    @classmethod
    def parse_command_line(cls, command_line):
        class RaisingArgumentParser(argparse.ArgumentParser):
            """Make argparse raise only (not exit) on error. See https://bugs.python.org/issue41255"""

            def error(self, message):
                raise Exception(message)

        parser = RaisingArgumentParser()
        for argument in cls.iarguments():
            parser.add_argument(*argument.id_list, **argument.argparse_kvsargs())
        return vars(parser.parse_args(args=shlex.split(command_line)))

    def icommand_line(self, full=False, with_user=False, user=None, exclude=None):
        _exclude = [] if exclude is None else exclude

        if full:
            yield "mini-buildd-api"
            yield self.name()
            yield mini_buildd.http_endpoint().geturl(with_user=with_user, user=user)

        for arg in (a for a in self.args.values() if a.identity not in _exclude and a.given is not None):
            if arg.is_positional:
                yield from arg.icommand_line_given()
            else:
                yield arg.id_list[0]
                yield from arg.icommand_line_given()

    def command_line(self, full=False, with_user=False, user=None, exclude=None):
        return mini_buildd.PyCompat.shlex_join(self.icommand_line(full=full, with_user=with_user, user=user, exclude=exclude))

    def command_line_full(self):
        user = self.request.user if self.AUTH is not mini_buildd.config.AUTH_NONE and self.request is not None and self.request.user.is_authenticated else None
        return self.command_line(full=True, with_user=user is not None, user=user)

    def http_args(self, with_confirm=False, with_output=None):
        http_args = {}
        for arg in self.args.values():
            if arg.given is not None:
                http_args[arg.identity] = arg.strgiven()

        if with_confirm:
            http_args["confirm"] = self.name()
        if with_output is not None:
            http_args["output"] = with_output

        return http_args

    def url(self):
        return mini_buildd.http_endpoint().geturl(path=mini_buildd.config.URIS["api"]["view"].join(self.name() + "/"), query=self.http_args(with_output="html"))

    @abc.abstractmethod
    def _run(self):
        pass

    def run(self):
        self._run()


def _pimpdoc(result_doc=None):
    def pimp(cls):
        indent = "    "  # Be sure to keep the same indent like a top level class doc (4 chars) for all lines, so inspect.getdoc() will properly unindent.
        if result_doc:
            cls.__doc__ += "\n" + "\n".join([f"{indent}{line}" for line in result_doc]) + "\n"
        cls.__doc__ += f"\n{indent}Authorization: {cls.AUTH}\n"
        return cls
    return pimp


#: Option shortcuts (use as mixin)
class _Admin:
    AUTH = mini_buildd.config.AUTH_ADMIN


class _Staff:
    AUTH = mini_buildd.config.AUTH_STAFF


class _Login:
    AUTH = mini_buildd.config.AUTH_LOGIN


class _Running:
    NEEDS_RUNNING_DAEMON = True


class _Confirm:
    CONFIRM = True


class _Maintenance:
    _LOCK = threading.Lock()

    def run(self):
        if self._LOCK.acquire(blocking=False):  # pylint: disable=consider-using-with
            try:
                with mini_buildd.daemon.Stopped():
                    self._run()
            finally:
                self._LOCK.release()
        else:
            raise mini_buildd.HTTPUnavailable("Another maintenance API call currently running")


#: API calls
@_pimpdoc()
class Status(Call):
    """
    Get the status of this instance

    JSON Result:

    * version     : mini-buildd's version.
    * identity    : Instance identity.
    * url         : Instance URL (HTTP).
    * incoming_url: Incoming URL (currently FTP).
    * load        : Instance's (0 =< load <= 1). If negative, the instance is not powered on.
    * chroots     : Active chroots.
    * remotes     : Active or auto-reactivatable remotes.
    * [repositories: Simplified structural representation of all repositories]
    """

    @classmethod
    def iarguments(cls):
        yield BoolArgument(["--with-repositories", "-R"], doc="Also list all repositories (may be longish)")

    def _run(self):
        self.result = {
            "version": mini_buildd.__version__,
            "identity": mini_buildd.get_daemon().model.identity,
            "url": mini_buildd.http_endpoint().geturl(),
            "incoming_url": mini_buildd.get_daemon().model.mbd_get_ftp_endpoint().geturl(),
            "load": mini_buildd.get_daemon().builder.load() if mini_buildd.get_daemon().is_alive() else -1.0,
            "chroots": [c.mbd_key() for c in mini_buildd.mdls().chroot.Chroot.mbd_get_active()],
            "remotes": [r.mbd_url() for r in mini_buildd.mdls().gnupg.Remote.mbd_get_active_or_auto_reactivate()],
        }
        if self.args["with_repositories"].value():
            self.result["repositories"] = {r.identity: r.mbd_json() for r in mini_buildd.mdls().repository.Repository.objects.all()}


@_pimpdoc()
class PubKey(Call):
    """
    Get public key

    Get ASCII-armored GnuPG public key of this instance.

    Used to sign the apt repositories (apt key) and for authorization across instances.
    """

    def _run(self):
        self.result = mini_buildd.get_daemon().gnupg.pub_key


@_pimpdoc()
class DputConf(Call):
    """
    Get recommended dput config snippet

    Usually, this is for integration in your personal ~/.dput.cf.

    .. tip:: Can I add a custom dput config snippet?

      Just put your custom snippet in ``~mini-buildd/etc/dput.cf``. If that file exists, it will be added to the output of :apicall:`dput_conf`.
    """

    def _run(self):
        self.result = mini_buildd.get_daemon().model.mbd_get_dput_conf()

        # Optionally add local custom config
        try:
            with mini_buildd.fopen(mini_buildd.config.ROUTES["etc"].path.join("dput.cf")) as extra_conf:
                self.result += extra_conf.read()
        except FileNotFoundError:
            pass


@_pimpdoc()
class SourcesList(Call):
    """
    Get sources.list (apt lines)

    Usually, this output is put to a file like '/etc/sources.list.d/mini-buildd-xyz.list'.
    """

    @classmethod
    def iarguments(cls):
        yield Codenames(["--codenames", "-C"], default=Value.ALL_CODENAMES)
        yield Repositories(["--repositories", "-R"], default=Value.ALL_REPOSITORIES)
        yield ListArgument(["--suites", "-S"], default=Value.ALL_SUITES, choices=Value.ALL_SUITES, doc="Suite names (like 'stable', 'unstable')")
        yield ListArgument(["--types", "-T"], default=["deb"], choices=["deb", "deb-src"], doc="Types of apt lines")
        yield StrArgument(["--options", "-O"], default="", doc="Apt line options ('deb[-src] [<options>] ...'). See 'man 5 source.list'")
        yield Rollbacks(["--rollbacks", "-r"])
        yield StrArgument(["--snapshot", "-P"], default="", doc="Select a repository snapshot")
        yield StrArgument(["--mirror", "-M"], default="", doc="URL of a mirror to use (instead of mini-buildd's native URL)")
        yield BoolArgument(["--with-extra", "-X"], doc="Also list extra sources needed")
        yield BoolArgument(["--with-comment", "-D"], doc="Add comment line above apt line")

    def _run(self):
        sources_list = mini_buildd.files.SourcesList()

        for r in self.args["repositories"].objects():
            for d in r.distributions.all().filter(base_source__codename__in=self.args["codenames"].value()):
                if self.args["with_extra"].value():
                    for e in d.extra_sources.all():
                        sources_list.append(e.source.mbd_get_apt_line())

                for s in r.layout.suiteoption_set.filter(suite__name__in=self.args["suites"].value()):
                    sources_list.append(r.mbd_get_apt_line(d, s, snapshot=self.args["snapshot"].value()))
                    for rollback in self.args["rollbacks"].range(s):
                        sources_list.append(r.mbd_get_apt_line(d, s, snapshot=self.args["snapshot"].value(), rollback=rollback))

        self.result = sources_list.get(self.args["types"].value(),
                                       mirror=self.args["mirror"].value(),
                                       options=self.args["options"].value(),
                                       with_comment=self.args["with_comment"].value())


@_pimpdoc()
class Ls(Call):
    """
    List source package
    """

    @classmethod
    def iarguments(cls):
        yield Source(["source"])
        yield Repositories(["--repositories", "-R"], default=Value.ALL_REPOSITORIES)
        yield Codenames(["--codenames", "-C"], default=Value.ALL_CODENAMES)
        yield StrArgument(["--version", "-V"], default="", doc="Limit to exactly this version")
        yield StrArgument(["--min-version", "-M"], default="", doc="Limit to this version or greater")

    def _run(self):
        for repository in self.args["repositories"].objects():
            ls = repository.mbd_reprepro.ls(self.args["source"].value(),
                                            codenames=self.args["codenames"].value(),
                                            version=self.args["version"].value(),
                                            min_version=self.args["min_version"].value())
            if ls:
                self.result[repository.identity] = ls


@_pimpdoc()
class Show(Ls):
    """
    Show source package
    """

    def _run(self):
        super()._run()
        for value in self.result.values():
            value.enrich()


MULT_VERSIONS_PER_DIST_NOTE = "Use as safeguard, or for rare cases of multiple version of the same package in one distribution (in different components)"


class Debdiff(Call):
    """
    Compare two internal source packages
    """

    @classmethod
    def iarguments(cls):
        yield Source(["source"])
        yield Repository(["repository"])
        yield StrArgument(["--versiona", "-a"], default="", doc="Version a to compare")
        yield StrArgument(["--versionb", "-b"], default="", doc="Version b to compare")

    def _run(self):
        repository = self.args["repository"].object()
        repositories_path = mini_buildd.config.ROUTES["repositories"].path
        dsca = repository.mbd_dsc_pool_path(self.args["source"].value(), self.args["versiona"].value())
        dscb = repository.mbd_dsc_pool_path(self.args["source"].value(), self.args["versionb"].value())
        self.result = mini_buildd.call.Call(["debdiff", repositories_path.join(dsca), repositories_path.join(dscb)]).stdout


@_pimpdoc()
class Migrate(_Staff, _Confirm, Call):
    """
    Migrate source package

    Migrates a source package along with all its binary packages. If
    run for a rollback distribution, this will perform a rollback
    restore.
    """

    @classmethod
    def iarguments(cls):
        yield Source(["source"])
        yield Distribution(["distribution"], choices=Value.MIGRATABLE_DISTRIBUTIONS)
        yield BoolArgument(["--full", "-F"], doc="Migrate all 'migrates_to' suites up (f.e. unstable->testing->stable)")
        yield StrArgument(["--version", "-V"], default="", doc=f"Migrate exactly this version. {MULT_VERSIONS_PER_DIST_NOTE}")

    def _run(self):
        repository, distribution, suite = mini_buildd.mdls().repository.parse_dist(self.args["distribution"].dist())
        version = self.args["version"].value()
        repository.mbd_package_migrate(self.args["source"].value(),
                                       distribution,
                                       suite,
                                       full=self.args["full"].value(),
                                       rollback=self.args["distribution"].dist().rollback_no,
                                       version=version if version else None)


@_pimpdoc()
class Remove(_Admin, _Confirm, Call):
    """
    Remove source package

    Removes a source package along with all its binary packages.
    """

    @classmethod
    def iarguments(cls):
        yield Source(["source"])
        yield Distribution(["distribution"])
        yield BoolArgument(["--without-rollback"], doc="Don't copy to rollback distribution")
        yield StrArgument(["--version", "-V"], default="", doc=f"Remove exactly this version. {MULT_VERSIONS_PER_DIST_NOTE}")

    def _run(self):
        repository, distribution, suite = mini_buildd.mdls().repository.parse_dist(self.args["distribution"].dist())
        version = self.args["version"].value()
        repository.mbd_package_remove(self.args["source"].value(),
                                      distribution,
                                      suite,
                                      rollback=self.args["distribution"].dist().rollback_no,
                                      version=version if version else None,
                                      without_rollback=self.args["without_rollback"].value())


PORT_RESULT_DOC = (
    "JSON Result (dict):",
    "  list of dicts of all items uploaded",
)


@_pimpdoc(PORT_RESULT_DOC)
class Port(_Staff, _Running, _Confirm, Call):
    """
    Port internal source package

    An internal 'port' is a no-changes (i.e., only the changelog
    will be adapted) rebuild of the given locally-installed
    package.

    When from-distribution equals to_distribution, a rebuild will be done.
    """

    @classmethod
    def iarguments(cls):
        yield Source(["source"])
        yield Distribution(["from_distribution"])
        yield Distributions(["to_distributions"], choices=Value.ACTIVE_UPLOADABLE_DISTRIBUTIONS)
        yield StrArgument(["--version", "-V"], default="", doc=f"Port exactly this version. {MULT_VERSIONS_PER_DIST_NOTE}")
        yield UploadOptions(["--options", "-O"])

    def _run(self):
        self.result = []
        version = self.args["version"].value()
        for to_diststr in self.args["to_distributions"].value():
            self.result.append(mini_buildd.package.port(self.args["source"].value(),
                                                        self.args["from_distribution"].value(),
                                                        to_diststr,
                                                        version=version if version else None,
                                                        options=self.args["options"].value().split("|")))


@_pimpdoc(PORT_RESULT_DOC)
class PortExt(_Staff, _Running, _Confirm, Call):
    """
    Port external source package

    An external 'port' is a no-changes (i.e., only the changelog
    will be adapted) rebuild of any given source package.
    """

    @classmethod
    def iarguments(cls):
        yield UrlArgument(["dsc"], doc="URL of any Debian source package (dsc) to port")
        yield BoolArgument(["--allow-unauthenticated", "-u"], doc="Don't verify downloaded DSC against Debian keyrings (see ``man dget``)")
        yield Distributions(["distributions"], choices=Value.ACTIVE_UPLOADABLE_DISTRIBUTIONS)
        yield UploadOptions(["--options", "-O"])

    def _run(self):
        self.result = []
        for diststr in self.args["distributions"].value():
            self.result.append(mini_buildd.package.port_ext(self.args["dsc"].value(),
                                                            diststr,
                                                            options=self.args["options"].value().split("|"),
                                                            allow_unauthenticated=self.args["allow_unauthenticated"].value()))


@_pimpdoc()
class Retry(_Staff, _Running, _Confirm, Call):
    """
    Retry a previously failed source package

    JSON Result:

    * changes: Changes file that has been re-uploaded.
    """

    BKEY_FORMAT = "<source>/<version>/<timecode>/source[ <arch>]"
    BKEY_REGEX = re.compile(r"[^/]+/[^/]+/[^/]+/[^/]+")

    @classmethod
    def iarguments(cls):
        yield StrArgument(["bkey"], choices=Value.LAST_FAILED_BKEYS, doc=f"Package bkey ('{cls.BKEY_FORMAT}') of a past packaging try; 'extra.bkey' of a FAILED packaging event")

    def _run(self):
        bkey = self.args["bkey"].value()
        if not self.BKEY_REGEX.match(bkey):
            raise mini_buildd.HTTPBadRequest(f"Wrong bkey format. Should be '{self.BKEY_FORMAT}'")

        events_path = mini_buildd.config.ROUTES["events"].path.new_sub([bkey])
        if not os.path.exists(events_path.full):
            raise mini_buildd.HTTPBadRequest(f"No such event path: {bkey}")

        # Check that there is exactly one failed event in the given path
        failed_events = glob.glob(events_path.join("*_FAILED.json"))
        if len(failed_events) != 1:
            raise mini_buildd.HTTPBadRequest(f"{len(failed_events)} failed events found (we need exactly 1). Check your bkey")
        failed_event = mini_buildd.events.Event.load(failed_events[0])
        LOG.debug(f"Retry: Found valid FAILED event: {failed_event}")

        # Find all changes in that path
        changes_files = glob.glob(events_path.join("*.changes"))
        if len(changes_files) != 1:
            raise mini_buildd.HTTPBadRequest(f"{len(changes_files)} changes found (we need exactly 1). Check your bkey")
        changes = changes_files[0]
        LOG.debug(f"Retry: Found solitaire changes file: {changes}")

        # Should be save now to re-upload the changes files found in path
        mini_buildd.changes.Base(changes).upload(mini_buildd.get_daemon().model.mbd_get_ftp_endpoint(), force=True)
        self.result = os.path.basename(changes)


@_pimpdoc()
class Cancel(_Staff, _Running, _Confirm, Call):
    """Cancel an ongoing package build"""

    @classmethod
    def iarguments(cls):
        yield StrArgument(["bkey"], choices=Value.CURRENT_BUILDS, doc="Build key ('<source>/<version>/<timecode>/<arch>' of ongoing builds; 'extra.bkey' in events)")

    def _run(self):
        mini_buildd.get_daemon().builder.cancel(self.args["bkey"].value(), f"{self.request.user}")
        self.result = self.args["bkey"].value()


@_pimpdoc()
class SetUserKey(_Login, _Confirm, Call):
    """
    Set a user's GnuPG public key
    """

    @classmethod
    def category(cls):
        return cls.CATEGORIES[1]

    @classmethod
    def iarguments(cls):
        yield MultilineStrArgument(["key"], doc="GnuPG public key; multiline inputs will be handled as ascii armored full key, one-liners as key ids")

    def _run(self):
        uploader = self.request.user.uploader
        uploader.Admin.mbd_remove(uploader)
        key = self.args["key"].value()

        if "\n" in key:
            LOG.debug("Using given key argument as full ascii-armored GPG key")
            uploader.key_id = ""
            uploader.key = key
        else:
            LOG.debug("Using given key argument as key ID")
            uploader.key_id = key
            uploader.key = ""

        uploader.Admin.mbd_prepare(uploader)
        uploader.Admin.mbd_check(uploader)
        LOG.warning(f"Uploader profile changed: {uploader} (must be (re-)activated by mini-buildd staff before you can actually use it)")


@_pimpdoc()
class Subscribe(_Login, Call):
    """
    Subscribe to (email) notifications
    """

    @classmethod
    def iarguments(cls):
        yield Source(["source"], default="", extra_doc="\n\nLeave empty for all source packages")
        yield Distribution(["distribution"], default="", choices=Value.PREPARED_DISTRIBUTIONS, extra_doc="\n\nLeave empty for all distributions")

    def __init__(self, **kwargs):
        super().__init__(**kwargs)

        self.source = self.args["source"].value()
        self.sourcestr = "" if self.source is None else self.source
        self.distribution = self.args["distribution"].value()
        self.distributionstr = "" if self.distribution is None else self.distribution

    def _run(self):
        s, created = mini_buildd.mdls().subscription.Subscription.objects.get_or_create(
            subscriber=self.request.user,
            package=self.sourcestr,
            distribution=self.distributionstr,
        )
        self.result = {"created": created, "source": s.package, "distribution": s.distribution}


@_pimpdoc()
class Unsubscribe(Subscribe):
    """
    Unsubscribe from (email) notifications
    """

    def _run(self):
        self.result = []
        for s in mini_buildd.mdls().subscription.Subscription.objects.filter(subscriber=self.request.user):
            if (self.sourcestr == s.package) and (self.distributionstr == s.distribution):
                s.delete()
                self.result.append({"source": s.package, "distribution": s.distribution})


@_pimpdoc()
class RemakeChroots(_Admin, _Confirm, _Maintenance, Call):
    """
    Remake chroots

    Run actions 'remove', 'prepare', 'check' and 'activate'.

    Note that Daemon will be stopped before running, cancelling ongoing events (``BUILDING``, ``PACKAGING``).
    """

    @classmethod
    def iarguments(cls):
        yield ListArgument(["--keys"], default=Value.ALL_CHROOTS, choices=Value.ALL_CHROOTS, doc="Chroot keys (<codename>:<arch>)")

    def _run(self):
        chroots = mini_buildd.mdls().chroot.Chroot.objects.none()
        for key in self.args["keys"].value():
            codename, arch = key.split(":")
            chroots |= mini_buildd.mdls().chroot.Chroot.objects.filter(source__codename=codename, architecture__name=arch)
        mini_buildd.mdls().chroot.Chroot.Admin.mbd_actions(self.request, chroots, ["remove", "prepare", "check", "activate"])


@_pimpdoc()
class Power(_Admin, _Confirm, Call):
    """
    Power Daemon on or off (toggles by default)

    This essentially stops accepting incoming, and forcibly stops any possibly running builds.

    This state is *not persisted*. Please *deactivate* the Daemon instance via :mbdpage:`setup` to make the state persist over *mini-buildd service* restarts.
    """

    @classmethod
    def iarguments(cls):
        yield ChoiceArgument(["to_state"], choices=[o.name for o in OnOff] + [Argument.SERVER_DEFAULT], default=Value.POWER_TOGGLE, doc="Power state to set to")

    def _run(self):
        self.result["state_pre"] = OnOff(mini_buildd.get_daemon().is_alive()).name
        {OnOff.ON.name: mini_buildd.get_daemon().mbd_start, OnOff.OFF.name: mini_buildd.get_daemon().mbd_stop}[self.args["to_state"].value()]()
        self.result["state"] = OnOff(mini_buildd.get_daemon().is_alive()).name


@_pimpdoc()
class Wake(_Staff, _Confirm, Call):
    """
    Wake a remote instance
    """

    @classmethod
    def iarguments(cls):
        yield StrArgument(["--remote", "-r"], choices=Value.ALL_REMOTES)
        yield IntArgument(["--sleep", "-s"], default=5, doc="Sleep between wake attempts")
        yield IntArgument(["--attempts", "-a"], default=3, doc="Max number attempts")

    def _run(self):
        remote = mini_buildd.mdls().gnupg.Remote.objects.get(http=self.args["remote"].value())
        self.result = remote.mbd_get_status(wake=True, wake_sleep=self.args["sleep"].value(), wake_attempts=self.args["attempts"].value())


@_pimpdoc()
class Handshake(Call):
    """
    Check if signed message matches a remote, reply our signed message on success

    This is for internal use only.
    """

    @classmethod
    def category(cls):
        return cls.CATEGORIES[2]

    @classmethod
    def iarguments(cls):
        yield MultilineStrArgument(["--signed-message", "-S"])

    def _run(self):
        signed_message = self.args["signed_message"].value()
        for r in mini_buildd.mdls().gnupg.Remote.objects.all():
            try:
                r.mbd_verify(signed_message)
                self.result = mini_buildd.get_daemon().handshake_message()
                LOG.debug(f"Remote handshake ok: '{r}': {r.key_long_id}: {r.key_name}")
                return
            except Exception as e:
                mini_buildd.log_exception(LOG, f"Remote handshake failed for '{r}'", e)
        raise mini_buildd.HTTPBadRequest(f"GnuPG handshake failed: No remote for public key on {mini_buildd.http_endpoint()}")


@_pimpdoc()
class Cronjob(_Admin, _Confirm, Call):
    """
    Run a cron job now (out of schedule)
    """

    @classmethod
    def iarguments(cls):
        yield StrArgument(["id"], choices=Value.ALL_CRONJOBS)

    def _run(self):
        job = mini_buildd.get_daemon().crontab.get(self.args["id"].value())
        self.result[job.id()] = job.run()


@_pimpdoc()
class Uploaders(_Admin, _Running, Call):
    """
    Get upload permissions for repositories
    """

    @classmethod
    def iarguments(cls):
        yield Repositories(["--repositories", "-R"], default=Value.ALL_REPOSITORIES)

    def _run(self):
        for r in self.args["repositories"].objects():
            self.result[r.identity] = {"allow_unauthenticated_uploads": r.allow_unauthenticated_uploads}
            with closing(mini_buildd.daemon.UploadersKeyring(r.identity)) as gpg:
                self.result[r.identity]["uploaders"] = gpg.get_pub_keys_infos()


@_pimpdoc()
class SnapshotLs(_Running, Call):
    """
    Get list of repository snapshots for a distribution

    JSON Result (dict):
      dict: <distribution>: [snasphot,...]: List of snapshots for the given distribution.
    """

    @classmethod
    def iarguments(cls):
        yield Distribution(["distribution"])

    def _run(self):
        diststr = self.args["distribution"].value()
        self.result = diststr2repository(diststr).mbd_reprepro.get_snapshots(diststr)


@_pimpdoc()
class SnapshotCreate(_Admin, _Confirm, SnapshotLs):
    """
    Create a repository snapshot
    """

    @classmethod
    def iarguments(cls):
        yield from SnapshotLs.iarguments()
        yield StrArgument(["name"], doc="Snapshot name")

    def _run(self):
        diststr = self.args["distribution"].value()
        diststr2repository(diststr).mbd_reprepro.gen_snapshot(diststr, self.args["name"].value())


class SnapshotDelete(SnapshotCreate):
    """
    Delete a repository snapshot
    """

    def _run(self):
        diststr = self.args["distribution"].value()
        diststr2repository(diststr).mbd_reprepro.del_snapshot(diststr, self.args["name"].value())


@_pimpdoc()
class Debmirror(_Admin, _Confirm, Call):
    """
    Make local partial repository mirror via :debpkg:`debmirror`

    This may be useful if you plan on publishing a stripped-down (f.e.,
    only certain repos, only ``stable``, omit rollbacks) variant of your
    repo somewhere remote.

    .. error:: debmirror (:debbug:`819925`): ``apt update`` fails on ``experimental`` suites (contents not mirrored)

      This happens only on systems where APT is configured to download contents, most likely just because
      ``apt-file`` is installed. So, the easiest workaround is::

        apt purge apt-file

      Closest to an actual fix is to install debmirror variant '+abfixes' from
      ``Hellfield Archive``, where the "bug" has been fixed (Dec 2022: at least
      available for bullseye).
    """

    PROGRAM = "/usr/bin/debmirror"
    DEB = "debmirror"

    @classmethod
    def iarguments(cls):
        yield ListArgument(["--suites", "-S"], default=Value.ALL_SUITES, choices=Value.ALL_SUITES, doc="Suite names (like 'stable', 'unstable')")
        yield Rollbacks(["--rollbacks", "-r"])
        yield Repositories(["--repositories", "-R"], default=Value.ALL_REPOSITORIES)
        yield StrArgument(["--architectures", "-A"],
                          default="amd64,i386,arm64,armhf,armel,s390x,ppc64el,mipsel,mips64el",
                          doc=("Architectures to mirror (``--arch`` in debmirror)\n\n"
                               "Usually, you just want all, so the default already lists all currently supported Debian/Ubuntu architectures (but will only mirror the architectures actually found)"))
        yield StrArgument(["--components", "-C"],
                          default="main,contrib,non-free,main/debian-installer,multiverse,restricted,universe",
                          doc=("Components to mirror (``--section`` in debmirror)\n\n"
                               "Usually, you just want all, so the default already lists all known Debian/Ubuntu components (but will only mirror the components actually found)"))
        yield StrArgument(["--destination", "-D"],
                          default=Value.DEFAULT_DEBMIRROR_DESTINATION,
                          doc=("Mirror destination dir (``mirrordir`` in debmirror)\n"
                               "\n"
                               "* ``{}`` will be replaced by repository identity\n"
                               "* Directory must be accessible by user ``mini-buildd``\n"
                               "\n"
                               "*BE CAREFUL* with this value -- anything in this directory will be *HAPPILY REPLACED* by the mirror only\n"))

    def _run(self):
        mini_buildd.check_program(self.PROGRAM, self.DEB)
        for r in self.args["repositories"].objects():
            debmirror = [
                self.PROGRAM,
                "--verbose",
                "--ignore-release-gpg",  # change
                "--diff", "none",
                "--getcontents",
                "--rsync-extra", "none",
                "--host", mini_buildd.http_endpoint().hopo(),
                "--method", mini_buildd.http_endpoint().scheme(),
                "--root", f"/repositories/{r.identity}",
                "--dist", ",".join(r.mbd_get_diststrs(frollbacks=self.args["rollbacks"].range, suite__name__in=self.args["suites"].value())),
                "--arch", self.args["architectures"].value(),
                "--section", self.args["components"].value(),
                self.args["destination"].value().format(r.identity),
            ]
            LOG.debug(debmirror)
            mini_buildd.call.Call(debmirror).check()


@_pimpdoc(PORT_RESULT_DOC)
class KeyringPackages(_Admin, _Running, _Confirm, Call):
    """
    Build keyring packages

    .. tip:: **keyring-packages**: No compat for urold (``apt-key add``)

      Since ``2.x``, keyring packages will use ``/etc/apt/trusted.gpg.d/<foo>.gpg``, not deprecated ``apt-key add <foo>``.

      In Debian, this is supported since ``wheezy (2013)``.

      For distributions ``<= squeeze`` (apt versions ``~<= 0.8.x``), you would manually have to run ``apt-key add /etc/apt/trusted.gpg.d/<foo>.gpg`` after installation of the keyring package.
    """

    @classmethod
    def iarguments(cls):
        yield Distributions(["--distributions", "-D"], default=Value.ACTIVE_KEYRING_DISTRIBUTIONS)
        yield BoolArgument(["--without-migration", "-M"], doc="Don't migrate packages")

    def _run(self):
        self.result = []
        events = mini_buildd.events.Attach(mini_buildd.get_daemon().events)

        for diststr in self.args["distributions"].value():
            repository, distribution, suite = mini_buildd.mdls().repository.parse_diststr(diststr)
            if not suite.build_keyring_package:
                raise mini_buildd.HTTPBadRequest(f"Keyring package to non-keyring suite requested (see 'build_keyring_package' flag): '{diststr}'")
            self.result.append(mini_buildd.package.upload_template_package(mini_buildd.package.KeyringPackage(), diststr))

        def unfinished():
            return [upload for upload in self.result if "event" not in upload]

        while unfinished():
            event = events.get()
            for upload in unfinished():
                if event.match(types=[mini_buildd.events.Type.INSTALLED, mini_buildd.events.Type.FAILED, mini_buildd.events.Type.REJECTED], distribution=upload["distribution"], source=upload["source"], version=upload["version"]):
                    LOG.debug(f"Keyring package result: {event}")
                    upload["event"] = event.type.name
                    if (event.type == mini_buildd.events.Type.INSTALLED) and not self.args["without_migration"].value():
                        repository, distribution, suite = mini_buildd.mdls().repository.parse_diststr(upload["distribution"])
                        repository.mbd_package_migrate(upload["source"], distribution, suite, full=True, version=upload["version"])


@_pimpdoc(PORT_RESULT_DOC)
class TestPackages(_Admin, _Running, _Confirm, Call):
    """
    Build test packages
    """

    __TEMPLATES = ["mbd-test-archall", "mbd-test-cpp", "mbd-test-ftbfs"]

    @classmethod
    def iarguments(cls):
        yield ListArgument(["--sources", "-S"], default=cls.__TEMPLATES, choices=cls.__TEMPLATES, doc="Test source packages to use")
        yield Distributions(["--distributions", "-D"], default=Value.ACTIVE_EXPERIMENTAL_DISTRIBUTIONS, choices=Value.ACTIVE_UPLOADABLE_DISTRIBUTIONS)
        yield Distributions(["--auto-ports", "-A"], default=[], choices=Value.ACTIVE_UPLOADABLE_DISTRIBUTIONS)
        yield BoolArgument(["--with-check", "-c"], doc="Check for correct packager results")

    def _run(self):
        self.result = []
        events = mini_buildd.events.Attach(mini_buildd.get_daemon().events)

        for source in self.args["sources"].value():
            for diststr in self.args["distributions"].value():
                self.result.append(mini_buildd.package.upload_template_package(mini_buildd.package.TestPackage(source, auto_ports=self.args["auto_ports"].value()), diststr))

        if self.args["with_check"].value():
            def unfinished():
                return [upload for upload in self.result if "event" not in upload]

            while unfinished():
                event = events.get()
                for upload in unfinished():
                    if event.match(types=[mini_buildd.events.Type.INSTALLED, mini_buildd.events.Type.FAILED, mini_buildd.events.Type.REJECTED], distribution=upload["distribution"], source=upload["source"], version=upload["version"]):
                        upload["event"] = event.type.name
                        if (event.type == mini_buildd.events.Type.FAILED and upload["source"] in ["mbd-test-archall", "mbd-test-cpp"]) or \
                           (event.type == mini_buildd.events.Type.INSTALLED and upload["source"] in ["mbd-test-ftbfs"]):
                            raise mini_buildd.HTTPBadRequest(f"Test package failed: {event}")
                        LOG.debug(f"Test package result: {event}")


@_pimpdoc()
class Setup(_Admin, _Confirm, _Maintenance, Call):
    """
    Create, update or inspect your setup

    Note that Daemon will be stopped before running, cancelling ongoing events (``BUILDING``, ``PACKAGING``).
    """

    @classmethod
    def iarguments(cls):
        yield StrArgument(["--update"], default="", doc="Update existing instances; 'all' to update all, empty string to never update, <hash_id>,.. (see a previous run) to update selected instances", header="Run Options")
        yield StrArgument(["--pca"], default="", doc="Prepare, check and create instances; 'all' to pca all, empty string to never pca, <hash_id>,.. (see a previous run) to pca selected instances")

        yield StrArgument(["--identity"], default=Value.DEFAULT_IDENTITY, doc="Instance identity (for keyring package names, dput config, ...)", header="Daemon")
        yield StrArgument(["--ftp-endpoint"], default=Value.DEFAULT_FTP_ENDPOINT, doc="FTP (incoming) network endpoint")

        yield ListArgument(["--archives", "-A"], default=[], doc="Add arbitrary archives", header="Archives")
        yield ListArgument(["--archives-from-origin"], default=[], choices=SETUP["origin"].keys(), doc="Add original archives from these origins")
        yield BoolArgument(["--archives-from-apt-sources"], doc="Add archives guessed from your local sources.list")
        yield BoolArgument(["--archives-from-proxy"], doc="Add archives guessed from a locally running apt-cacher-ng")

        yield ListArgument(["--sources"], default=[], doc="Manually select codenames to setup sources for", header="Sources")
        yield ListArgument(["--sources-from-origin"], default=[], doc=(
            "Add predefined codenames from origin (vendor) (as per ``distro-info``, ``di``): <origin>[:lts|all],...\n"
            "\n"
            "Per default, all 'di-supported' codenames are added.\n"
            "\n"
            "lts: Add 'di-supported' plus 'di-lts' (Note: adds 'lts', 'elts' for Debian and 'esm' for Ubuntu)\n"
            "all: Adds all source codenames with a working setup from these origins (i.e., includes very old ones)"))

        yield ListArgument(["--distributions"],
                           default=[],
                           doc=(
                               f"Distributions to setup ``<codename>:<arch0>[+<arch1>..]``\n"
                               f"For architectures, you may use the special key word 'native' for this host's natively supported architectures: {','.join(mini_buildd.dist.Archs.native())}.\n"
                               f"Example: sid:native+arm64,bullseye:native\n"),
                           header="Distributions")
        yield BoolArgument(["--distributions-from-sources"], doc="Auto-add distributions for all given ``sources`` codenames with native architectures")

        yield ListArgument(["--repositories"],
                           default=[],
                           doc=(
                               f"Repositories to setup ``<id>:<layout>:<codename>[+<codename>..]``\n"
                               f"Setup layouts available: {'|'.join(SETUP['layout'].keys())}\n"
                               f"Example: test:Default:sid+bullseye,test2:Default:sid\n"
                               "Note that repository IDs 'test' and 'debdev' are special names with hardcoded taintings:\n"
                               "* ``test`` repo will be uploadable w/o authentication\n"
                               "* `debdev`` repo will be uploadable for Debian Developers (per installed ``debian-keyring``), and Layout 'Debian Developer' by default (``sid`` uploadable as ``unstable``)\n"),
                           header="Repositories")
        yield ListArgument(["--repositories-from-distributions"], default=[], doc="AutoSetup these repositories with all setup sources: '<id>[:<layout>]'")

        yield ListArgument(["--chroots"], default=[],
                           doc=(
                               "Chroots to setup (uses same syntax as ``--distributions``).\n\n"
                               ".. tip:: With :debpkg:`qemu-user-static` installed, you also have seamless access to foreign archs (albeit with a speed penalty)"),
                           header="Chroots")
        yield BoolArgument(["--chroots-from-sources"], doc="Auto-add chroots for all given ``sources`` codenames with native architectures")
        yield ChoiceArgument(["--chroots-backend", "-C"], default=Value.DEFAULT_CHROOT_BACKEND, choices=["Dir", "File", "LVM", "LoopLVM", "BtrfsSnapshot"], doc="Chroot backend to use")

        yield ListArgument(["--remotes"], default=[], doc=f"Remotes to add. {inspect.getdoc(mini_buildd.net.ClientEndpoint)}", header="Remotes")

    class Instance:
        def diff(self):
            """Overload/expand this for additional custom diffs."""
            diff = {
                "setup": {
                    "fields": [],
                    "diff": {},
                },
                "model": {
                    "fields": [],
                    "diff": {},
                },
            }

            setup = {**self.identifiers, **self.options}
            target = self.model(**setup)
            for field in (f for f in self.model.mbd_get_fields(exclude=["id", "status", "last_checked", "auto_reactivate", "pickled_data", "ping"]) if not f.is_relation):
                data = diff["setup"] if field.name in setup else diff["model"]

                data["fields"].append(field.name)
                current_value = None if self.obj is None else getattr(self.obj, field.name, None)
                target_value = getattr(target, field.name, None)
                if target_value not in ["", current_value]:
                    data["diff"][field.name] = {
                        "current": str(current_value),
                        "target": str(target_value),
                    }
            return diff

        @classmethod
        def _add_m2m_diff(cls, diff, field, expected, current):
            diff["setup"]["fields"].append(field)
            _expected = set(expected)
            _current = set(current)
            if _expected - _current:
                diff["setup"]["diff"].setdefault(field, {})["missing"] = list(_expected - _current)
            if _current - _expected:
                diff["setup"]["diff"].setdefault(field, {})["unknown"] = list(_current - _expected)

        def _get_obj(self):
            """Override this for custom getter."""
            return self.model.mbd_get_or_none(**self.identifiers)

        def __init__(self, call, model, options, update_args, **identifiers):
            self.call, self.model, self.options, self.update_args, self.identifiers = call, model, options, update_args, identifiers
            LOG.debug(f"Instance: {self.model}({self.identifiers})")

            self.identity = {
                "class": repr(self.model),
                "identifiers": {k: repr(v) for k, v in self.identifiers.items()},
            }
            self.identity_hash = hashlib.sha256(json.dumps(self.identity, sort_keys=True).encode(mini_buildd.config.CHAR_ENCODING)).hexdigest()

            self.obj = self._get_obj()
            self.info = {
                "identity": self.identity,
                "identity_hash": self.identity_hash,
                "identity_str": f"{self.model.__name__}({self.identifiers})",
                "diff": None,
                "status": None,
                "actions": [],
            }

            if self.call.args["update"].value() in ["all"] + self.identity_hash.split(","):
                if self.obj is None:
                    self.obj = self.model(**self.identifiers, **self.options)
                    self.obj.save()
                    self.info["actions"].append("created")
                else:
                    for k, v in {**self.identifiers, **self.options}.items():
                        setattr(self.obj, k, v)
                    self.info["actions"].append("updated")

                if self.obj is not None:
                    self.update()
                    self.obj.save()

            if self.call.args["pca"].value() in ["all"] + self.identity_hash.split(","):
                if self.obj is not None and issubclass(self.model, mini_buildd.mdls().base.StatusModel):
                    self.model.Admin.mbd_action(self.call.request, [self.obj], "prepare")
                    self.model.Admin.mbd_action(self.call.request, [self.obj], "check", force=True)
                    self.model.Admin.mbd_action(self.call.request, [self.obj], "activate")
                    self.info["actions"].append("pca")

            self.info["diff"] = self.diff()
            self.info["status"] = "None" if self.obj is None else self.obj.get_status_display() if issubclass(self.model, mini_buildd.mdls().base.StatusModel) else "Exists"
            if self.obj is not None and not self.model.__name__.endswith("Option"):
                # Help for HTML template only
                self.info["change_instance_url"] = f"admin:mini_buildd_{self.model.__name__.lower()}_change"
                self.info["change_instance_pk"] = str(self.obj.pk)
            self.call.result["instances"].append(self.info)

        def update(self):
            pass

    @classmethod
    def ilocal_archive_urls(cls):
        try:
            import aptsources.sourceslist
            for src in (src for src in aptsources.sourceslist.SourcesList() if not src.invalid and not src.disabled):
                # These URLs come from the user. 'normalize' the uri first to have exactly one trailing slash.
                yield src.uri.rstrip("/") + "/"
        except BaseException as e:
            mini_buildd.log_exception(LOG, "Failed to scan local sources.lists for default mirrors ('python-apt' not installed?)", e)

    @classmethod
    def iapt_cacher_archive_urls(cls):
        url = mini_buildd.net.detect_apt_cacher_ng(url=f"http://{mini_buildd.config.HOSTNAME_FQDN}:3142")
        if url:
            LOG.debug(f"Local apt-cacher-ng detected: {url}")
            for setup in SETUP["origin"].values():
                for path in setup.get("archive_paths", []):
                    LOG.debug(f"Local proxy archive: '{url}/{path}/'")
                    yield f"{url}/{path}/"

    class Dists(dict):
        @classmethod
        def iexpand_arch(cls, arch):
            if arch == "native":
                for native_arch in mini_buildd.dist.Archs.native():
                    yield native_arch
            else:
                yield arch

        @classmethod
        def iexpand_archs(cls, archs):
            for arch in archs:
                yield from cls.iexpand_arch(arch)

        def set(self, codename, archs):
            self.setdefault(codename, [])
            for arch in self.iexpand_archs(archs):
                if arch not in self[codename]:
                    self[codename].append(arch)

        def __init__(self, items=None):
            """
            ``<codename>:<arch0>+<arch1>...<sep>...`` to ``{codename: uniq_archlist}``
            """
            _items = [] if items is None else items
            for d in _items:
                codename, dummy, _archs = d.partition(":")
                self.set(codename, mini_buildd.misc.esplit(_archs, "+"))

        def as_argument_value(self):
            return [f"{codename}:{'+'.join(archs)}" for codename, archs in self.items()]

        def merge(self, dists):
            for codename, archs in dists.items():
                self.set(codename, archs)

    def __setup(self):
        setup = {}

        # Daemon
        setup["identity"] = self.args["identity"].value()
        setup["ftp_endpoint"] = self.args["ftp_endpoint"].value()

        # Archives
        archives = self.args["archives"].value()
        if self.args["archives_from_apt_sources"].value():
            archives.extend(self.ilocal_archive_urls())
        if self.args["archives_from_proxy"].value():
            archives.extend(self.iapt_cacher_archive_urls())
        for v in self.args["archives_from_origin"].value():
            archives.extend(SETUP["origin"][v]["archive"])
        setup["archives"] = mini_buildd.misc.uniq(archives)

        # Sources
        sources = self.args["sources"].value()
        for vendor_info in self.args["sources_from_origin"].value():
            vendor, dummy, modifier = vendor_info.partition(":")
            sources.extend(mini_buildd.dist.setup_codenames_from_origin(vendor, modifier))
        setup["sources"] = mini_buildd.misc.uniq(sources)

        # Distributions
        setup["distributions"] = Setup.Dists(self.args["distributions"].value())
        if self.args["distributions_from_sources"].value():
            setup["distributions"].merge({c: mini_buildd.dist.Archs.native() for c in setup["sources"]})

        # Repositories
        setup["repositories"] = {}
        for repo_preset in self.args["repositories"].value():
            repo, dummy, rest = repo_preset.partition(":")
            layout, dummy, _dists = rest.partition(":")
            setup["repositories"].setdefault(repo, {})
            setup["repositories"][repo]["dists"] = mini_buildd.misc.esplit(_dists, "+")
            setup["repositories"][repo]["layout"] = layout if layout else "Debian Developer" if repo == "debdev" else "Default"

        for repo_preset in self.args["repositories_from_distributions"].value():
            repo, dummy, layout = repo_preset.partition(":")
            setup["repositories"].setdefault(repo, {})
            setup["repositories"][repo]["dists"] = list(setup["distributions"].keys())
            setup["repositories"][repo]["layout"] = layout if layout else "Debian Developer" if repo == "debdev" else "Default"

        # Chroots
        setup["chroots"] = Setup.Dists(self.args["chroots"].value())
        if self.args["chroots_from_sources"].value():
            setup["chroots"].merge({c: mini_buildd.dist.Archs.native() for c in setup["sources"]})
        setup["chroots_backend"] = self.args["chroots_backend"].value()

        # Remotes
        setup["remotes"] = self.args["remotes"].value()

        return setup

    def __check(self):
        # Preliminary checks
        duplicate_distributions = []
        for distribution in mini_buildd.mdls().distribution.Distribution.objects.all():
            same_codename = mini_buildd.mdls().distribution.Distribution.objects.filter(base_source__codename=distribution.base_source.codename)
            if len(same_codename) != 1:
                duplicate_distributions.append(distribution.base_source.codename)
        if duplicate_distributions:
            raise mini_buildd.HTTPBadRequest(f"Duplicate distributions found: {','.join(duplicate_distributions)}: This is unfortunately possible, but 'unintended use'. Please check if you really need them (``setup`` can't be used if you stick with it).")

    def __run(self):
        # Preliminary checks
        self.__check()

        setup = self.__setup()
        LOG.debug(f"Setup from args: {setup}")

        self.result = {
            "setup": setup,   # Setup computed from args (info, debugging)
            "instances": [],  # Diffs in individual instances
            "report": {},     # Summary && unreported in instances
        }

        # Daemon
        Setup.Instance(self,
                       mini_buildd.mdls().daemon.Daemon,
                       {
                           "identity": setup["identity"],
                           "ftpd_bind": setup["ftp_endpoint"],
                       },
                       {})

        # Archives
        for url in setup["archives"]:
            Setup.Instance(self, mini_buildd.mdls().source.Archive, {}, {}, url=url)

        # Sources
        class AptKeyInstance(Setup.Instance):
            def _get_obj(self):
                """Also lookup traditional/deprecated (short) key (last 8 chars) -- there still might be objects having this"""
                obj = self.model.mbd_get_or_none(**self.identifiers)
                if obj is None:
                    obj = self.model.mbd_get_or_none(key_id=self.identifiers["key_id"][-8:])  # short key
                return obj

            def update(self):
                """Implicitly update key_id with long key_id from setup"""
                self.obj.key_id = self.identifiers["key_id"]

        class SourceInstance(Setup.Instance):
            def update(self):
                self.obj.apt_keys.clear()
                for apt_key in self.update_args["apt_key_instances"]:
                    if apt_key.obj is not None:
                        self.obj.apt_keys.add(apt_key.obj)

            def diff(self):
                diff = super().diff()
                if self.obj is not None:
                    self._add_m2m_diff(diff, "apt_keys", self.update_args["apt_keys"], [a.key_long_id for a in self.obj.apt_keys.all()])
                return diff

        for codename in (Codename(c) for c in setup["sources"]):
            for source_codename in codename.setup_sources():
                SourceInstance(self,
                               mini_buildd.mdls().source.Source,
                               codename.setup_options(source_codename),
                               {
                                   "apt_keys": codename.setup_apt_keys(source_codename),
                                   "apt_key_instances": [AptKeyInstance(self, mini_buildd.mdls().gnupg.AptKey, {}, {}, key_id=long_key_id) for long_key_id in codename.setup_apt_keys(source_codename)],
                               },
                               origin=codename.setup_origin(source_codename),
                               codename=source_codename)

        # Distributions
        class DistributionInstance(Setup.Instance):
            def update(self):
                codename = self.update_args["codename"]

                # Extra Sources
                for ps in self.update_args["priority_source_instances"]:
                    if ps.obj is not None:
                        self.obj.extra_sources.add(ps.obj)

                # Components
                for component in self.update_args["component_instances"]:
                    if component.obj is not None:
                        self.obj.components.add(component.obj)

                # Architectures
                for arch in self.update_args["archs"]:
                    Setup.Instance(
                        self.call,
                        mini_buildd.mdls().distribution.ArchitectureOption,
                        {
                            "optional": arch in codename.setup_arch_optional(),
                        },
                        {},
                        architecture=Setup.Instance(self.call, mini_buildd.mdls().source.Architecture, {}, {}, name=arch).obj,
                        distribution=self.obj)

                archall = self.obj.architectureoption_set.filter(build_architecture_all=True)
                if len(archall) != 1:
                    LOG.debug(f"Fixing archall (now: {archall})")
                    archall_set = False  # Use first non-optional arch to build arch "all"
                    for ao in self.obj.architectureoption_set.all():
                        ao.build_architecture_all = not archall_set and not ao.optional
                        ao.save()
                        if ao.build_architecture_all:
                            archall_set = True

            def diff(self):
                diff = super().diff()
                if self.obj is not None:
                    self._add_m2m_diff(diff, "components",
                                       SETUP["origin"][self.obj.base_source.origin]["default_components"],
                                       [c.name for c in self.obj.components.all()])
                    self._add_m2m_diff(diff, "extra_sources",
                                       [s.source.codename for s in mini_buildd.mdls().source.PrioritySource.objects.filter(source__codename__regex=fr"^{self.obj.base_source.codename}[-/]")],
                                       [s.source.codename for s in self.obj.extra_sources.all()])
                return diff

        for codename, archs in ((Codename(c), a) for c, a in setup["distributions"].items()):
            DistributionInstance(self,
                                 mini_buildd.mdls().distribution.Distribution,
                                 codename.setup_distribution_options(),
                                 {
                                     "codename": codename,
                                     "priority_source_instances": [Setup.Instance(self,
                                                                                  mini_buildd.mdls().source.PrioritySource,
                                                                                  {},
                                                                                  {},
                                                                                  source=mini_buildd.mdls().source.Source.mbd_get_or_none(codename=source_codename),
                                                                                  priority=500 if codename.setup_is_security(source_codename) else 1)  # Smartly add prio source: Security gets prio=500, others prio=1 (opt-in).
                                                                   for source_codename in (c for c in codename.setup_sources() if c != codename.codename)],
                                     "component_instances": [Setup.Instance(self, mini_buildd.mdls().source.Component, {}, {}, name=comp) for comp in codename.setup_components()],
                                     "archs": archs,
                                 },
                                 base_source=mini_buildd.mdls().source.Source.mbd_get_or_none(codename=codename.codename))

        # Repositories
        class LayoutInstance(Setup.Instance):
            def update(self):
                setup = self.update_args["setup"]
                for suite_name, suite_setup in setup["suites"].items():
                    suite_instance = Setup.Instance(self.call, mini_buildd.mdls().distribution.Suite, {}, {}, name=suite_name)
                    Setup.Instance(self.call,
                                   mini_buildd.mdls().distribution.SuiteOption,
                                   {
                                       **suite_setup.get("options", {}),
                                       "extra_options": f"Rollback: {setup['rollback'].get(suite_name, 0)}\n",
                                       "migrates_to": mini_buildd.mdls().distribution.SuiteOption.mbd_get_or_none(layout=self.obj, suite=mini_buildd.mdls().distribution.Suite.mbd_get_or_none(name=suite_setup.get("migrates_to"))),
                                   },
                                   {},
                                   layout=self.obj,
                                   suite=suite_instance.obj)

        class RepositoryInstance(Setup.Instance):
            def update(self):
                for codename in self.update_args["dists"]:
                    distribution = mini_buildd.mdls().distribution.Distribution.mbd_get_or_none(base_source__codename__exact=codename)
                    self.obj.distributions.add(distribution)

            def diff(self):
                diff = super().diff()
                if self.obj is not None:
                    self._add_m2m_diff(diff, "distributions",
                                       self.update_args["dists"],
                                       [d.base_source.codename for d in self.obj.distributions.all()])
                return diff

        for repo, values in setup["repositories"].items():
            # Layouts
            options = {}
            layout_setup = SETUP["layout"].get(values["layout"], {})
            layout_instance = LayoutInstance(self, mini_buildd.mdls().distribution.Layout, layout_setup.get("options", {}), {"setup": layout_setup}, name=values["layout"])

            if repo == "test":
                options["allow_unauthenticated_uploads"] = True
            elif repo == "debdev":
                options["extra_uploader_keyrings"] = ("# Allow Debian maintainers (must install the 'debian-keyring' package)\n"
                                                      "/usr/share/keyrings/debian-keyring.gpg")

            options["layout"] = layout_instance.obj
            RepositoryInstance(self, mini_buildd.mdls().repository.Repository, options, {"dists": values["dists"]}, identity=repo)

        # Chroots
        for codename, archs in ((Codename(c), a) for c, a in setup["chroots"].items()):
            for arch in archs:
                Setup.Instance(self,
                               getattr(mini_buildd.mdls().chroot, f"{setup['chroots_backend']}Chroot"),
                               codename.setup_chroot_options(),
                               {},
                               source=mini_buildd.mdls().source.Source.mbd_get_or_none(codename=codename.codename),
                               architecture=Setup.Instance(self, mini_buildd.mdls().source.Architecture, {}, {}, name=arch).obj)

        # Remotes
        for ep in setup["remotes"]:
            Setup.Instance(self, mini_buildd.mdls().gnupg.Remote, {}, {}, http=ep)

        # Report
        self.result["report"]["setup_diffs"] = sum((1 for i in self.result["instances"] if i["diff"]["setup"]["diff"]))
        self.result["report"]["model_diffs"] = sum((1 for i in self.result["instances"] if i["diff"]["model"]["diff"]))

    def _run(self):
        self.__run()

    @classmethod
    def from_models(cls):
        chroots = Setup.Dists()
        for c in mini_buildd.mdls().chroot.Chroot.objects.all():
            chroots.merge(Setup.Dists([c.mbd_key()]))

        return Setup(
            identity=mini_buildd.get_daemon().model.identity,
            ftp_endpoint=mini_buildd.get_daemon().model.ftpd_bind,
            archives=[a.url for a in mini_buildd.mdls().source.Archive.objects.all()],
            sources=[d.base_source.codename for d in mini_buildd.mdls().distribution.Distribution.objects.all()],
            distributions=[d.mbd_setup() for d in mini_buildd.mdls().distribution.Distribution.objects.all()],
            repositories=[r.mbd_setup() for r in mini_buildd.mdls().repository.Repository.objects.all()],
            chroots=chroots.as_argument_value(),
            remotes=[r.http for r in mini_buildd.mdls().gnupg.Remote.objects.all()],
        )


SETUP_PRESETS = {
    "Debian": Setup(archives_from_origin="Debian", sources_from_origin="Debian", distributions_from_sources=True, repositories_from_distributions="test", chroots_from_sources=True),
    "Ubuntu": Setup(archives_from_origin="Ubuntu", sources_from_origin="Ubuntu", distributions_from_sources=True, repositories_from_distributions="test", chroots_from_sources=True),
}


class Calls(collections.OrderedDict):
    """Automatically collect all calls defined in this module, and make them accessible"""

    def __init__(self):
        super().__init__({c.name(): c for c in sys.modules[__name__].__dict__.values() if inspect.isclass(c) and issubclass(c, Call) and c != Call})


CALLS = Calls()
