From 5d47c7c510d43752476c4404e5c78446cc5296bc Mon Sep 17 00:00:00 2001 From: Stefano Pigozzi Date: Tue, 10 Dec 2019 00:21:52 +0100 Subject: [PATCH] Rework constellation, somehow without breaking almost anything --- .idea/codeStyles/Project.xml | 52 ++++++++++++++++++++ .idea/dataSources.xml | 11 +++++ royalnet/constellation/constellation.py | 64 ++++++++++++++++++++----- royalnet/constellation/star.py | 22 ++++++--- royalnet/utils/log.py | 29 ++++++++++- 5 files changed, 158 insertions(+), 20 deletions(-) create mode 100644 .idea/codeStyles/Project.xml create mode 100644 .idea/dataSources.xml diff --git a/.idea/codeStyles/Project.xml b/.idea/codeStyles/Project.xml new file mode 100644 index 00000000..49f7b87b --- /dev/null +++ b/.idea/codeStyles/Project.xml @@ -0,0 +1,52 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/dataSources.xml b/.idea/dataSources.xml new file mode 100644 index 00000000..ec186820 --- /dev/null +++ b/.idea/dataSources.xml @@ -0,0 +1,11 @@ + + + + + postgresql + true + org.postgresql.Driver + jdbc:postgresql://scaleway.steffo.eu:5432/royalnet + + + \ No newline at end of file diff --git a/royalnet/constellation/constellation.py b/royalnet/constellation/constellation.py index 61fb40a7..3919c3c7 100644 --- a/royalnet/constellation/constellation.py +++ b/royalnet/constellation/constellation.py @@ -7,6 +7,7 @@ import royalnet.herald as rh import royalnet.utils as ru import royalnet.commands as rc from .star import PageStar, ExceptionStar +from ..utils import init_logging try: import uvicorn @@ -51,7 +52,8 @@ class Constellation: herald_cfg: Dict[str, Any], packs_cfg: Dict[str, Any], constellation_cfg: Dict[str, Any], - **_): + logging_cfg: Dict[str, Any] + ): if Starlette is None: raise ImportError("`constellation` extra is not installed") @@ -87,9 +89,19 @@ class Constellation: self.alchemy = ra.Alchemy(alchemy_cfg["database_url"], tables) log.info(f"Alchemy: {self.alchemy}") + # Logging + self._logging_cfg: Dict[str, Any] = logging_cfg + """The logging config for the :class:`Constellation` is stored to initialize the logger when the first page is + requested, as disabling the :mod:`uvicorn` logging also disables all logging in the process in general.""" + # Herald self.herald: Optional[rh.Link] = None - """The :class:`Link` object connecting the :class:`Constellation` to the rest of the herald network.""" + """The :class:`Link` object connecting the :class:`Constellation` to the rest of the herald network. + As is the case with the logging module, it will be started on the first request received by the + :class:`Constellation`, as the event loop won't be available before that.""" + + self._herald_cfg: Dict[str, Any] = herald_cfg + """The herald config for the :class:`Constellation` is stored to initialize the :class:`rh.Herald` later.""" self.herald_task: Optional[aio.Task] = None """A reference to the :class:`aio.Task` that runs the :class:`rh.Link`.""" @@ -120,8 +132,7 @@ class Constellation: elif not herald_cfg["enabled"]: log.info("Herald: disabled") else: - self.init_herald(herald_cfg) - log.info(f"Herald: enabled") + log.info(f"Herald: will be enabled on first request") # Register PageStars and ExceptionStars for pack_name in packs: @@ -151,6 +162,11 @@ class Constellation: self.port: int = constellation_cfg["port"] """The port on which the :class:`Constellation` will listen for connection on.""" + self.loop: Optional[aio.AbstractEventLoop] = None + """The event loop of the :class:`Constellation`. + + Because of how :mod:`uvicorn` runs, it will stay :const:`None` until the first page is requested.""" + # TODO: is this a good idea? def interface_factory(self) -> Type[rc.CommandInterface]: """Create the :class:`rc.CommandInterface` class for the :class:`Constellation`.""" @@ -241,29 +257,52 @@ class Constellation: log.debug(f"Registering: {SelectedEvent.__qualname__} -> {SelectedEvent.name}") self.events[SelectedEvent.name] = event + def _first_page_check(self): + if self.loop is None: + self.loop = aio.get_running_loop() + self.init_herald(self._herald_cfg) + self.loop.create_task(self.herald.run()) + init_logging(self._logging_cfg) + + def _page_star_wrapper(self, page_star: PageStar): + async def f(request): + self._first_page_check() + log.info(f"Running {page_star}") + return await page_star.page(request) + + return page_star.path, f, page_star.methods + + def _exc_star_wrapper(self, exc_star: ExceptionStar): + async def f(request): + self._first_page_check() + log.info(f"Running {exc_star}") + return await exc_star.page(request) + + return exc_star.error, f + def register_page_stars(self, page_stars: List[Type[PageStar]], pack_cfg: Dict[str, Any]): for SelectedPageStar in page_stars: log.debug(f"Registering: {SelectedPageStar.path} -> {SelectedPageStar.__qualname__}") try: - page_star_instance = SelectedPageStar(constellation=self, config=pack_cfg) + page_star_instance = SelectedPageStar(interface=self.Interface(pack_cfg)) except Exception as e: log.error(f"Skipping: " f"{SelectedPageStar.__qualname__} - {e.__class__.__qualname__} in the initialization.") ru.sentry_exc(e) continue - self.starlette.add_route(page_star_instance.path, page_star_instance.page, page_star_instance.methods) + self.starlette.add_route(*self._page_star_wrapper(page_star_instance)) def register_exc_stars(self, exc_stars: List[Type[ExceptionStar]], pack_cfg: Dict[str, Any]): - for SelectedPageStar in exc_stars: - log.debug(f"Registering: {SelectedPageStar.error} -> {SelectedPageStar.__qualname__}") + for SelectedExcStar in exc_stars: + log.debug(f"Registering: {SelectedExcStar.error} -> {SelectedExcStar.__qualname__}") try: - page_star_instance = SelectedPageStar(constellation=self, config=pack_cfg) + exc_star_instance = SelectedExcStar(interface=self.Interface(pack_cfg)) except Exception as e: log.error(f"Skipping: " - f"{SelectedPageStar.__qualname__} - {e.__class__.__qualname__} in the initialization.") + f"{SelectedExcStar.__qualname__} - {e.__class__.__qualname__} in the initialization.") ru.sentry_exc(e) continue - self.starlette.add_exception_handler(page_star_instance.error, page_star_instance.page) + self.starlette.add_exception_handler(*self._exc_star_wrapper(exc_star_instance)) def run_blocking(self): log.info(f"Running Constellation on https://{self.address}:{self.port}/...") @@ -300,7 +339,8 @@ class Constellation: constellation = cls(alchemy_cfg=alchemy_cfg, herald_cfg=herald_cfg, packs_cfg=packs_cfg, - constellation_cfg=constellation_cfg) + constellation_cfg=constellation_cfg, + logging_cfg=logging_cfg) # Run the server constellation.run_blocking() diff --git a/royalnet/constellation/star.py b/royalnet/constellation/star.py index 6824e57f..657968dc 100644 --- a/royalnet/constellation/star.py +++ b/royalnet/constellation/star.py @@ -1,6 +1,7 @@ from typing import * from starlette.requests import Request from starlette.responses import Response +from royalnet.commands import CommandInterface if TYPE_CHECKING: from .constellation import Constellation @@ -10,9 +11,8 @@ class Star: """A Star is a class representing a part of the website. It shouldn't be used directly: please use :class:`PageStar` and :class:`ExceptionStar` instead!""" - def __init__(self, config: Dict[str, Any], constellation: "Constellation"): - self.config: Dict[str, Any] = config - self.constellation: "Constellation" = constellation + def __init__(self, interface: CommandInterface): + self.interface: CommandInterface = interface async def page(self, request: Request) -> Response: """The function generating the :class:`~starlette.Response` to a web :class:`~starlette.Request`. @@ -20,21 +20,31 @@ class Star: If it raises an error, the corresponding :class:`ExceptionStar` will be used to handle the request instead.""" raise NotImplementedError() + @property + def constellation(self) -> "Constellation": + """A shortcut for the :class:`Constellation`.""" + return self.interface.constellation + @property def alchemy(self): """A shortcut for the :class:`~royalnet.alchemy.Alchemy` of the :class:`Constellation`.""" - return self.constellation.alchemy + return self.interface.constellation.alchemy # noinspection PyPep8Naming @property def Session(self): """A shortcut for the :class:`~royalnet.alchemy.Alchemy` :class:`Session` of the :class:`Constellation`.""" - return self.constellation.alchemy.Session + return self.interface.constellation.alchemy.Session @property def session_acm(self): """A shortcut for :func:`.alchemy.session_acm` of the :class:`Constellation`.""" - return self.constellation.alchemy.session_acm + return self.interface.constellation.alchemy.session_acm + + @property + def config(self) -> Dict[str, Any]: + """A shortcut for the Pack configuration of the :class:`Constellation`.""" + return self.interface.config def __repr__(self): return f"<{self.__class__.__qualname__}>" diff --git a/royalnet/utils/log.py b/royalnet/utils/log.py index ba1ab9c5..06686518 100644 --- a/royalnet/utils/log.py +++ b/royalnet/utils/log.py @@ -9,7 +9,32 @@ except ImportError: l: logging.Logger = logging.getLogger(__name__) +# From https://stackoverflow.com/a/56810619/4334568 +def reset_logging(): + manager = logging.root.manager + manager.disabled = logging.NOTSET + for logger in manager.loggerDict.values(): + if isinstance(logger, logging.Logger): + logger.setLevel(logging.NOTSET) + logger.propagate = True + logger.disabled = False + logger.filters.clear() + handlers = logger.handlers.copy() + for handler in handlers: + # Copied from `logging.shutdown`. + try: + handler.acquire() + handler.flush() + handler.close() + except (OSError, ValueError): + pass + finally: + handler.release() + logger.removeHandler(handler) + + def init_logging(logging_cfg: Dict[str, Any]): + reset_logging() loggers_cfg = logging_cfg["Loggers"] for logger_name in loggers_cfg: if logger_name == "root": @@ -23,7 +48,7 @@ def init_logging(logging_cfg: Dict[str, Any]): stream_handler.formatter = coloredlogs.ColoredFormatter(logging_cfg["log_format"], style="{") else: stream_handler.formatter = logging.Formatter(logging_cfg["log_format"], style="{") - if len(logging.root.handlers) < 1: - logging.root.addHandler(stream_handler) + logging.root.handlers.clear() + logging.root.addHandler(stream_handler) l.debug("Logging: ready")