From 884b44302d2b857077a16fb497b278eccd55bf70 Mon Sep 17 00:00:00 2001 From: Stefano Pigozzi Date: Tue, 5 Nov 2019 14:45:59 +0100 Subject: [PATCH] Webserver is working but the db is still missing --- requirements.txt | 1 + royalnet/__main__.py | 38 +++++++++-- royalnet/bots/generic.py | 1 + royalnet/packs/common/__init__.py | 13 +++- royalnet/packs/common/stars/__init__.py | 16 +++++ royalnet/packs/common/stars/version.py | 15 ++++ royalnet/packs/royal/__init__.py | 12 +++- royalnet/packs/royal/stars/__init__.py | 15 ++++ royalnet/packs/royal/tables/wikipages.py | 2 +- royalnet/packs/rpg/__init__.py | 12 +++- royalnet/packs/rpg/stars/__init__.py | 15 ++++ royalnet/utils/__init__.py | 3 + royalnet/utils/urluuid.py | 11 +++ royalnet/web/__init__.py | 9 +++ royalnet/web/constellation.py | 87 ++++++++++++++++++++++++ royalnet/web/star.py | 24 ++++++- 16 files changed, 263 insertions(+), 11 deletions(-) create mode 100644 royalnet/packs/common/stars/__init__.py create mode 100644 royalnet/packs/common/stars/version.py create mode 100644 royalnet/packs/royal/stars/__init__.py create mode 100644 royalnet/packs/rpg/stars/__init__.py create mode 100644 royalnet/utils/urluuid.py create mode 100644 royalnet/web/constellation.py diff --git a/requirements.txt b/requirements.txt index bf33e55a..ac584170 100644 --- a/requirements.txt +++ b/requirements.txt @@ -40,4 +40,5 @@ yarl==1.3.0 youtube-dl==2019.10.16 riotwatcher==2.7.1 # discord.py is missing as we currently use the git version and we ignore the websockets<7.0 requirement +uvicorn==0.10.3 starlette==0.12.13 diff --git a/royalnet/__main__.py b/royalnet/__main__.py index 1315d42b..726c8b42 100644 --- a/royalnet/__main__.py +++ b/royalnet/__main__.py @@ -5,7 +5,7 @@ import royalnet as r import royalherald as rh import multiprocessing import keyring -import starlette +import logging @click.command() @@ -36,6 +36,14 @@ def run(telegram: typing.Optional[bool], local_network_server: bool, secrets_name: str, verbose: bool): + # Setup logging + if verbose: + core_logger = logging.root + core_logger.setLevel(logging.DEBUG) + stream_handler = logging.StreamHandler() + stream_handler.formatter = logging.Formatter("{asctime}\t{name}\t{levelname}\t{message}", style="{") + core_logger.addHandler(stream_handler) + core_logger.debug("Logging setup complete.") # Get the network password network_password = keyring.get_password(f"Royalnet/{secrets_name}", "network") @@ -92,17 +100,29 @@ def run(telegram: typing.Optional[bool], r.packs.common.tables.Discord, "discord_id") - # Import command packs + # Import command and star packs packs: typing.List[str] = list(packs) packs.append("royalnet.packs.common") # common pack is always imported enabled_commands = [] + enabled_page_stars = [] + enabled_exception_stars = [] for pack in packs: imported = importlib.import_module(pack) try: imported_commands = imported.available_commands except AttributeError: - raise click.ClickException(f"{pack} isn't a Royalnet Pack.") + raise click.ClickException(f"{pack} isn't a Royalnet Pack as it is missing available_commands.") + try: + imported_page_stars = imported.available_page_stars + except AttributeError: + raise click.ClickException(f"{pack} isn't a Royalnet Pack as it is missing available_page_stars.") + try: + imported_exception_stars = imported.available_exception_stars + except AttributeError: + raise click.ClickException(f"{pack} isn't a Royalnet Pack as it is missing available_exception_stars.") enabled_commands = [*enabled_commands, *imported_commands] + enabled_page_stars = [*enabled_page_stars, *imported_page_stars] + enabled_exception_stars = [*enabled_exception_stars, *imported_exception_stars] telegram_process: typing.Optional[multiprocessing.Process] = None if interfaces["telegram"]: @@ -134,8 +154,16 @@ def run(telegram: typing.Optional[bool], daemon=True) discord_process.start() + webserver_process: typing.Optional[multiprocessing.Process] = None if interfaces["webserver"]: - ... + constellation = r.web.Constellation(page_stars=enabled_page_stars, + exc_stars=enabled_exception_stars, + secrets_name=secrets_name) + webserver_process = multiprocessing.Process(name="Constellation Webserver", + target=constellation.run_blocking, + args=(verbose,), + daemon=True) + webserver_process.start() click.echo("Royalnet processes have been started. You can force-quit by pressing Ctrl+C.") if server_process is not None: @@ -144,6 +172,8 @@ def run(telegram: typing.Optional[bool], telegram_process.join() if discord_process is not None: discord_process.join() + if webserver_process is not None: + webserver_process.join() if __name__ == "__main__": diff --git a/royalnet/bots/generic.py b/royalnet/bots/generic.py index f5628055..4014e9fa 100644 --- a/royalnet/bots/generic.py +++ b/royalnet/bots/generic.py @@ -37,6 +37,7 @@ class GenericBot: command = SelectedCommand(interface) except Exception as e: log.error(f"{e.__class__.__qualname__} during the registration of {SelectedCommand.__qualname__}") + sentry_sdk.capture_exception(e) continue # Linking the command to the interface interface.command = command diff --git a/royalnet/packs/common/__init__.py b/royalnet/packs/common/__init__.py index 4b96ad2c..feb329ab 100644 --- a/royalnet/packs/common/__init__.py +++ b/royalnet/packs/common/__init__.py @@ -1,7 +1,16 @@ # This is a template Pack __init__. You can use this without changing anything in other packages too! -from . import commands, tables +from . import commands, tables, stars from .commands import available_commands from .tables import available_tables +from .stars import available_page_stars, available_exception_stars -__all__ = ["commands", "tables", "available_commands", "available_tables"] +__all__ = [ + "commands", + "tables", + "stars", + "available_commands", + "available_tables", + "available_page_stars", + "available_exception_stars", +] diff --git a/royalnet/packs/common/stars/__init__.py b/royalnet/packs/common/stars/__init__.py new file mode 100644 index 00000000..2393b7e5 --- /dev/null +++ b/royalnet/packs/common/stars/__init__.py @@ -0,0 +1,16 @@ +# Imports go here! +from .version import VersionStar + + +# Enter the PageStars of your Pack here! +available_page_stars = [ + VersionStar, +] + +# Enter the ExceptionStars of your Pack here! +available_exception_stars = [ + +] + +# Don't change this, it should automatically generate __all__ +__all__ = [star.__name__ for star in [*available_page_stars, *available_exception_stars]] diff --git a/royalnet/packs/common/stars/version.py b/royalnet/packs/common/stars/version.py new file mode 100644 index 00000000..af16dd2b --- /dev/null +++ b/royalnet/packs/common/stars/version.py @@ -0,0 +1,15 @@ +import royalnet +from starlette.requests import Request +from starlette.responses import * +from royalnet.web import PageStar + + +class VersionStar(PageStar): + path = "/api/royalnet/version" + + async def page(self, request: Request, **kwargs) -> JSONResponse: + return JSONResponse({ + "version": { + "semantic": royalnet.version.semantic + } + }) diff --git a/royalnet/packs/royal/__init__.py b/royalnet/packs/royal/__init__.py index 8cb183d1..feb329ab 100644 --- a/royalnet/packs/royal/__init__.py +++ b/royalnet/packs/royal/__init__.py @@ -1,6 +1,16 @@ # This is a template Pack __init__. You can use this without changing anything in other packages too! +from . import commands, tables, stars from .commands import available_commands from .tables import available_tables +from .stars import available_page_stars, available_exception_stars -__all__ = ["commands", "tables", "available_commands", "available_tables"] +__all__ = [ + "commands", + "tables", + "stars", + "available_commands", + "available_tables", + "available_page_stars", + "available_exception_stars", +] diff --git a/royalnet/packs/royal/stars/__init__.py b/royalnet/packs/royal/stars/__init__.py new file mode 100644 index 00000000..10748e81 --- /dev/null +++ b/royalnet/packs/royal/stars/__init__.py @@ -0,0 +1,15 @@ +# Imports go here! + + +# Enter the PageStars of your Pack here! +available_page_stars = [ + +] + +# Enter the ExceptionStars of your Pack here! +available_exception_stars = [ + +] + +# Don't change this, it should automatically generate __all__ +__all__ = [star.__name__ for star in [*available_page_stars, *available_exception_stars]] diff --git a/royalnet/packs/royal/tables/wikipages.py b/royalnet/packs/royal/tables/wikipages.py index f3b0bc1a..91719fc8 100644 --- a/royalnet/packs/royal/tables/wikipages.py +++ b/royalnet/packs/royal/tables/wikipages.py @@ -3,7 +3,7 @@ from sqlalchemy import Column, \ String from sqlalchemy.dialects.postgresql import UUID from sqlalchemy.ext.declarative import declared_attr -from royalnet.web.shortcuts import to_urluuid +from royalnet.utils import to_urluuid class WikiPage: diff --git a/royalnet/packs/rpg/__init__.py b/royalnet/packs/rpg/__init__.py index 8cb183d1..feb329ab 100644 --- a/royalnet/packs/rpg/__init__.py +++ b/royalnet/packs/rpg/__init__.py @@ -1,6 +1,16 @@ # This is a template Pack __init__. You can use this without changing anything in other packages too! +from . import commands, tables, stars from .commands import available_commands from .tables import available_tables +from .stars import available_page_stars, available_exception_stars -__all__ = ["commands", "tables", "available_commands", "available_tables"] +__all__ = [ + "commands", + "tables", + "stars", + "available_commands", + "available_tables", + "available_page_stars", + "available_exception_stars", +] diff --git a/royalnet/packs/rpg/stars/__init__.py b/royalnet/packs/rpg/stars/__init__.py new file mode 100644 index 00000000..3aa1c42f --- /dev/null +++ b/royalnet/packs/rpg/stars/__init__.py @@ -0,0 +1,15 @@ +# Imports go here! + + +# Enter the PageStars of your Pack here! +available_page_stars = [ + +] + +# Enter the ExceptionStars of your Pack here! +available_exception_stars = [ + +] + +# Don't change this, it should automatically generate __all__ +__all__ = [star.__name__ for star in [*available_page_stars, *available_exception_stars]] diff --git a/royalnet/utils/__init__.py b/royalnet/utils/__init__.py index b5cbbdfa..9627780d 100644 --- a/royalnet/utils/__init__.py +++ b/royalnet/utils/__init__.py @@ -6,6 +6,7 @@ from .safeformat import safeformat from .classdictjanitor import cdj from .sleepuntil import sleep_until from .formatters import andformat, plusformat, fileformat, ytdldateformat, numberemojiformat, splitstring, ordinalformat +from .urluuid import to_urluuid, from_urluuid __all__ = [ "asyncify", @@ -22,4 +23,6 @@ __all__ = [ "discord_escape", "splitstring", "ordinalformat", + "to_urluuid", + "from_urluuid", ] diff --git a/royalnet/utils/urluuid.py b/royalnet/utils/urluuid.py new file mode 100644 index 00000000..e16fb41c --- /dev/null +++ b/royalnet/utils/urluuid.py @@ -0,0 +1,11 @@ +import uuid as _uuid +import base64 + + +def to_urluuid(uuid: _uuid.UUID) -> str: + """Return a base64 url-friendly short UUID.""" + return str(base64.urlsafe_b64encode(uuid.bytes), encoding="ascii").rstrip("=") + + +def from_urluuid(b: str) -> _uuid.UUID: + return _uuid.UUID(bytes=base64.urlsafe_b64decode(bytes(b + "==", encoding="ascii"))) diff --git a/royalnet/web/__init__.py b/royalnet/web/__init__.py index e69de29b..0f2fbe11 100644 --- a/royalnet/web/__init__.py +++ b/royalnet/web/__init__.py @@ -0,0 +1,9 @@ +from .constellation import Constellation +from .star import Star, PageStar, ExceptionStar + +__all__ = [ + "Constellation", + "Star", + "PageStar", + "ExceptionStar", +] diff --git a/royalnet/web/constellation.py b/royalnet/web/constellation.py new file mode 100644 index 00000000..b26fdfa8 --- /dev/null +++ b/royalnet/web/constellation.py @@ -0,0 +1,87 @@ +import typing +import uvicorn +import logging +import sentry_sdk +import royalnet +import keyring +from starlette.applications import Starlette +from .star import PageStar, ExceptionStar + + +log = logging.getLogger(__name__) + + +class Constellation: + def __init__(self, + secrets_name: str, + page_stars: typing.List[typing.Type[PageStar]] = None, + exc_stars: typing.List[typing.Type[ExceptionStar]] = None, + *, + debug: bool = __debug__,): + if page_stars is None: + page_stars = [] + + if exc_stars is None: + exc_stars = [] + + self.secrets_name: str = secrets_name + + log.info("Creating starlette app...") + self.starlette = Starlette(debug=debug) + + log.info("Registering page_stars...") + for SelectedPageStar in page_stars: + try: + page_star_instance = SelectedPageStar(constellation=self) + except Exception as e: + log.error(f"{e.__class__.__qualname__} during the registration of {SelectedPageStar.__qualname__}") + sentry_sdk.capture_exception(e) + continue + log.info(f"Registering: {page_star_instance.path} -> {page_star_instance.__class__.__name__}") + self.starlette.add_route(page_star_instance.path, page_star_instance.page, page_star_instance.methods) + + log.info("Registering exc_stars...") + for SelectedExcStar in exc_stars: + try: + exc_star_instance = SelectedExcStar(constellation=self) + except Exception as e: + log.error(f"{e.__class__.__qualname__} during the registration of {SelectedExcStar.__qualname__}") + sentry_sdk.capture_exception(e) + continue + log.info(f"Registering: {exc_star_instance.error} -> {exc_star_instance.__class__.__name__}") + self.starlette.add_exception_handler(exc_star_instance.error, exc_star_instance.page) + + def _init_sentry(self): + sentry_dsn = self.get_secret("sentry") + if sentry_dsn: + # noinspection PyUnreachableCode + if __debug__: + release = "DEV" + else: + release = royalnet.version.semantic + log.info(f"Sentry: enabled (Royalnet {release})") + self.sentry = sentry_sdk.init(sentry_dsn, + integrations=[AioHttpIntegration(), + SqlalchemyIntegration(), + LoggingIntegration(event_level=None)], + release=release) + else: + log.info("Sentry: disabled") + + def get_secret(self, username: str): + return keyring.get_password(f"Royalnet/{self.secrets_name}", username) + + def set_secret(self, username: str, password: str): + return keyring.set_password(f"Royalnet/{self.secrets_name}", username, password) + + def run_blocking(self, verbose): + if verbose: + core_logger = logging.root + core_logger.setLevel(logging.DEBUG) + stream_handler = logging.StreamHandler() + stream_handler.formatter = logging.Formatter("{asctime}\t{name}\t{levelname}\t{message}", style="{") + core_logger.addHandler(stream_handler) + core_logger.debug("Logging setup complete.") + self._init_sentry() + log.info("Running constellation server...") + uvicorn.run(self.starlette) diff --git a/royalnet/web/star.py b/royalnet/web/star.py index 62cf8e6e..8e8c00bb 100644 --- a/royalnet/web/star.py +++ b/royalnet/web/star.py @@ -1,5 +1,25 @@ -import starlette +import typing +from starlette.requests import Request +from starlette.responses import Response +if typing.TYPE_CHECKING: + from .constellation import Constellation class Star: - ... + tables: set = {} + + def __init__(self, constellation: "Constellation"): + self.constellation: "Constellation" = constellation + + async def page(self, request: Request, **kwargs) -> Response: + raise NotImplementedError() + + +class PageStar(Star): + path: str = NotImplemented + + methods: typing.List[str] = ["GET"] + + +class ExceptionStar(Star): + error: typing.Union[typing.Type[Exception], int]