From 192b4cf3d6533cb99380f147898fa18dc83f3c3c Mon Sep 17 00:00:00 2001 From: Stefano Pigozzi Date: Fri, 15 Nov 2019 20:47:32 +0100 Subject: [PATCH] Make the bot runnable --- .idea/royalnet.iml | 1 + poetry.lock | 39 ++-- pyproject.toml | 6 +- royalnet/__init__.py | 13 ++ royalnet/__main__.py | 213 +++++++++--------- royalnet/backpack/README.md | 3 + .../{packs/default => backpack}/__init__.py | 2 +- .../default => backpack}/commands/__init__.py | 0 .../default => backpack}/commands/ping.py | 0 .../default => backpack}/commands/version.py | 4 +- .../default => backpack}/stars/__init__.py | 0 .../stars/api_royalnet_version.py | 7 +- .../default => backpack}/tables/__init__.py | 0 .../default => backpack}/tables/discord.py | 0 .../default => backpack}/tables/telegram.py | 0 .../default => backpack}/tables/users.py | 0 royalnet/bard/ytdlmp3.py | 2 +- royalnet/commands/commanddata.py | 4 +- royalnet/configurator.py | 7 +- royalnet/constellation/constellation.py | 32 ++- royalnet/herald/config.py | 18 ++ royalnet/serf/__init__.py | 5 +- royalnet/serf/alchemyconfig.py | 12 +- royalnet/serf/discord/discordserf.py | 12 +- royalnet/serf/serf.py | 33 +-- royalnet/serf/telegram/telegramserf.py | 11 +- royalnet/utils/__init__.py | 6 +- royalnet/web/constellation.py | 95 -------- royalnet/web/star.py | 37 --- 29 files changed, 247 insertions(+), 315 deletions(-) create mode 100644 royalnet/backpack/README.md rename royalnet/{packs/default => backpack}/__init__.py (77%) rename royalnet/{packs/default => backpack}/commands/__init__.py (100%) rename royalnet/{packs/default => backpack}/commands/ping.py (100%) rename royalnet/{packs/default => backpack}/commands/version.py (78%) rename royalnet/{packs/default => backpack}/stars/__init__.py (100%) rename royalnet/{packs/default => backpack}/stars/api_royalnet_version.py (64%) rename royalnet/{packs/default => backpack}/tables/__init__.py (100%) rename royalnet/{packs/default => backpack}/tables/discord.py (100%) rename royalnet/{packs/default => backpack}/tables/telegram.py (100%) rename royalnet/{packs/default => backpack}/tables/users.py (100%) delete mode 100644 royalnet/web/constellation.py delete mode 100644 royalnet/web/star.py diff --git a/.idea/royalnet.iml b/.idea/royalnet.iml index f344fe9e..4832e066 100644 --- a/.idea/royalnet.iml +++ b/.idea/royalnet.iml @@ -3,6 +3,7 @@ + diff --git a/poetry.lock b/poetry.lock index 63d52a63..b41fb9a6 100644 --- a/poetry.lock +++ b/poetry.lock @@ -82,7 +82,7 @@ python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" version = "7.0" [[package]] -category = "dev" +category = "main" description = "Cross-platform colored terminal text." marker = "sys_platform == \"win32\"" name = "colorama" @@ -90,6 +90,17 @@ optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" version = "0.4.1" +[[package]] +category = "main" +description = "Log formatting with colors!" +name = "colorlog" +optional = true +python-versions = "*" +version = "4.0.2" + +[package.dependencies] +colorama = "*" + [[package]] category = "main" description = "cryptography is a package which provides cryptographic recipes and primitives to Python developers." @@ -142,7 +153,7 @@ voice = ["PyNaCl (1.3.0)"] [package.source] reference = "09a08f9a9f126aa1f55c2444eb70508d1d52f8d9" type = "git" -url = "https://github.com/Steffo99/discord.py.git" +url = "https://github.com/Steffo99/discord.py" [[package]] category = "main" description = "Discover and load entry points from installed packages." @@ -328,7 +339,7 @@ description = "pytest: simple powerful testing with Python" name = "pytest" optional = false python-versions = ">=3.5" -version = "5.2.2" +version = "5.2.3" [package.dependencies] atomicwrites = ">=1.0" @@ -562,6 +573,7 @@ version = "2019.11.5" alchemy_easy = ["sqlalchemy", "psycopg2_binary"] alchemy_hard = ["sqlalchemy", "psycopg2"] bard = ["ffmpeg_python", "youtube_dl"] +colorlog = ["colorlog"] constellation = ["starlette", "uvicorn"] discord = ["discord.py", "pynacl"] herald = ["websockets"] @@ -569,7 +581,7 @@ sentry = ["sentry_sdk"] telegram = ["python_telegram_bot"] [metadata] -content-hash = "a51bc903341dd7fb6c2923e4f0a45654ccc9c011182ad9a67c2c80d24f62fb26" +content-hash = "48fd4f6a0a25ffaf89999db4a999774b8e22794e07488b032ac0f244049caa5f" python-versions = "^3.8" [metadata.files] @@ -660,6 +672,10 @@ colorama = [ {file = "colorama-0.4.1-py2.py3-none-any.whl", hash = "sha256:f8ac84de7840f5b9c4e3347b3c1eaa50f7e49c2b07596221daec5edaabbd7c48"}, {file = "colorama-0.4.1.tar.gz", hash = "sha256:05eed71e2e327246ad6b38c540c4a3117230b19679b875190486ddd2d721422d"}, ] +colorlog = [ + {file = "colorlog-4.0.2-py2.py3-none-any.whl", hash = "sha256:450f52ea2a2b6ebb308f034ea9a9b15cea51e65650593dca1da3eb792e4e4981"}, + {file = "colorlog-4.0.2.tar.gz", hash = "sha256:3cf31b25cbc8f86ec01fef582ef3b840950dea414084ed19ab922c8b493f9b42"}, +] cryptography = [ {file = "cryptography-2.8-cp27-cp27m-macosx_10_6_intel.whl", hash = "sha256:fb81c17e0ebe3358486cd8cc3ad78adbae58af12fc2bf2bc0bb84e8090fa5ce8"}, {file = "cryptography-2.8-cp27-cp27m-manylinux1_x86_64.whl", hash = "sha256:44ff04138935882fef7c686878e1c8fd80a723161ad6a98da31e14b7553170c2"}, @@ -845,8 +861,8 @@ pyparsing = [ {file = "pyparsing-2.4.5.tar.gz", hash = "sha256:4ca62001be367f01bd3e92ecbb79070272a9d4964dce6a48a82ff0b8bc7e683a"}, ] pytest = [ - {file = "pytest-5.2.2-py3-none-any.whl", hash = "sha256:58cee9e09242937e136dbb3dab466116ba20d6b7828c7620f23947f37eb4dae4"}, - {file = "pytest-5.2.2.tar.gz", hash = "sha256:27abc3fef618a01bebb1f0d6d303d2816a99aa87a5968ebc32fe971be91eb1e6"}, + {file = "pytest-5.2.3-py3-none-any.whl", hash = "sha256:b6cf7ad9064049ee486586b3a0ddd70dc5136c40e1147e7d286efd77ba66c5eb"}, + {file = "pytest-5.2.3.tar.gz", hash = "sha256:15837d2880cb94821087bc07476892ea740696b20e90288fd6c19e44b435abdb"}, ] python-dateutil = [ {file = "python-dateutil-2.8.1.tar.gz", hash = "sha256:73ebfe9dbf22e832286dafa60473e4cd239f8592f699aa5adaf10050e6e1823c"}, @@ -936,24 +952,13 @@ websockets = [ {file = "websockets-8.1-cp36-cp36m-macosx_10_6_intel.whl", hash = "sha256:3762791ab8b38948f0c4d281c8b2ddfa99b7e510e46bd8dfa942a5fff621068c"}, {file = "websockets-8.1-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:3db87421956f1b0779a7564915875ba774295cc86e81bc671631379371af1170"}, {file = "websockets-8.1-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:4f9f7d28ce1d8f1295717c2c25b732c2bc0645db3215cf757551c392177d7cb8"}, - {file = "websockets-8.1-cp36-cp36m-manylinux2010_i686.whl", hash = "sha256:295359a2cc78736737dd88c343cd0747546b2174b5e1adc223824bcaf3e164cb"}, - {file = "websockets-8.1-cp36-cp36m-manylinux2010_x86_64.whl", hash = "sha256:1d3f1bf059d04a4e0eb4985a887d49195e15ebabc42364f4eb564b1d065793f5"}, {file = "websockets-8.1-cp36-cp36m-win32.whl", hash = "sha256:2db62a9142e88535038a6bcfea70ef9447696ea77891aebb730a333a51ed559a"}, {file = "websockets-8.1-cp36-cp36m-win_amd64.whl", hash = "sha256:0e4fb4de42701340bd2353bb2eee45314651caa6ccee80dbd5f5d5978888fed5"}, {file = "websockets-8.1-cp37-cp37m-macosx_10_6_intel.whl", hash = "sha256:9b248ba3dd8a03b1a10b19efe7d4f7fa41d158fdaa95e2cf65af5a7b95a4f989"}, {file = "websockets-8.1-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:ce85b06a10fc65e6143518b96d3dca27b081a740bae261c2fb20375801a9d56d"}, {file = "websockets-8.1-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:965889d9f0e2a75edd81a07592d0ced54daa5b0785f57dc429c378edbcffe779"}, - {file = "websockets-8.1-cp37-cp37m-manylinux2010_i686.whl", hash = "sha256:751a556205d8245ff94aeef23546a1113b1dd4f6e4d102ded66c39b99c2ce6c8"}, - {file = "websockets-8.1-cp37-cp37m-manylinux2010_x86_64.whl", hash = "sha256:3ef56fcc7b1ff90de46ccd5a687bbd13a3180132268c4254fc0fa44ecf4fc422"}, {file = "websockets-8.1-cp37-cp37m-win32.whl", hash = "sha256:7ff46d441db78241f4c6c27b3868c9ae71473fe03341340d2dfdbe8d79310acc"}, {file = "websockets-8.1-cp37-cp37m-win_amd64.whl", hash = "sha256:20891f0dddade307ffddf593c733a3fdb6b83e6f9eef85908113e628fa5a8308"}, - {file = "websockets-8.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:c1ec8db4fac31850286b7cd3b9c0e1b944204668b8eb721674916d4e28744092"}, - {file = "websockets-8.1-cp38-cp38-manylinux1_i686.whl", hash = "sha256:5c01fd846263a75bc8a2b9542606927cfad57e7282965d96b93c387622487485"}, - {file = "websockets-8.1-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:9bef37ee224e104a413f0780e29adb3e514a5b698aabe0d969a6ba426b8435d1"}, - {file = "websockets-8.1-cp38-cp38-manylinux2010_i686.whl", hash = "sha256:d705f8aeecdf3262379644e4b55107a3b55860eb812b673b28d0fbc347a60c55"}, - {file = "websockets-8.1-cp38-cp38-manylinux2010_x86_64.whl", hash = "sha256:c8a116feafdb1f84607cb3b14aa1418424ae71fee131642fc568d21423b51824"}, - {file = "websockets-8.1-cp38-cp38-win32.whl", hash = "sha256:e898a0863421650f0bebac8ba40840fc02258ef4714cb7e1fd76b6a6354bda36"}, - {file = "websockets-8.1-cp38-cp38-win_amd64.whl", hash = "sha256:f8a7bff6e8664afc4e6c28b983845c5bc14965030e3fb98789734d416af77c4b"}, {file = "websockets-8.1.tar.gz", hash = "sha256:5c65d2da8c6bce0fca2528f69f44b2f977e06954c8512a952222cea50dad430f"}, ] yarl = [ diff --git a/pyproject.toml b/pyproject.toml index 59a0a18f..1b443279 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -24,7 +24,7 @@ # telegram python_telegram_bot = {version="^12.2.0", optional=true} # discord - "discord.py" = {git="https://github.com/Steffo99/discord.py.git", optional=true} # discord.py 1.2.4 is missing Go Live related methods + "discord.py" = {git="https://github.com/Steffo99/discord.py", optional=true} # discord.py 1.2.4 is missing Go Live related methods pynacl = {version="^1.3.0", optional=true} # This requires libffi-dev and python3.*-dev to be installed on Linux systems # bard ffmpeg_python = {version="~0.2.0", optional=true} @@ -40,6 +40,8 @@ sentry_sdk = {version="~0.13.2", optional=true} # herald websockets = {version="^8.1", optional=true} + # colorlog + colorlog = {version="^4.0.2", optional=true} # Development dependencies [tool.poetry.dev-dependencies] @@ -55,7 +57,7 @@ constellation = ["starlette", "uvicorn"] sentry = ["sentry_sdk"] herald = ["websockets"] - + colorlog = ["colorlog"] [build-system] requires = ["poetry>=0.12"] diff --git a/royalnet/__init__.py b/royalnet/__init__.py index f4046d7e..50f6132d 100644 --- a/royalnet/__init__.py +++ b/royalnet/__init__.py @@ -1 +1,14 @@ __version__ = "5.1a1" + +from . import alchemy, bard, commands, constellation, herald, backpack, serf, utils + +__all__ = [ + "alchemy", + "bard", + "commands", + "constellation", + "herald", + "serf", + "utils", + "backpack", +] diff --git a/royalnet/__main__.py b/royalnet/__main__.py index 55c1ee15..1f9f761d 100644 --- a/royalnet/__main__.py +++ b/royalnet/__main__.py @@ -2,10 +2,9 @@ import click import typing import importlib import royalnet as r -import royalherald as rh import multiprocessing import keyring -import logging +from logging import Formatter, StreamHandler, getLogger, Logger @click.command() @@ -13,55 +12,53 @@ import logging help="Enable/disable the Telegram bot.") @click.option("--discord/--no-discord", default=None, help="Enable/disable the Discord bot.") -@click.option("--webserver/--no-webserver", default=None, - help="Enable/disable the Web server.") -@click.option("--webserver-port", default=8001, - help="The port on which the web server will listen on.") -@click.option("-d", "--database", type=str, default=None, - help="The PostgreSQL database path.") -@click.option("-p", "--packs", type=str, multiple=True, default=[], - help="The names of the Packs that should be used.") -@click.option("-n", "--network-address", type=str, default=None, - help="The Network server URL to connect to.") -@click.option("-l", "--local-network-server", is_flag=True, default=False, - help="Locally run a Network server and bind it to port 44444. Overrides -n.") -@click.option("--local-network-server-port", type=int, default=44444, - help="The port on which the local network will be ran.") +@click.option("--constellation/--no-constellation", default=None, + help="Enable/disable the Constellation web server.") +@click.option("--herald/--no-herald", default=None, + help="Enable/disable the integrated Herald server." + " If turned off, Royalnet will try to connect to another server.") +@click.option("--remote-herald-address", type=str, default=None, + help="If --no-herald is specified, connect to the Herald server at this URL instead.") +@click.option("-c", "--constellation-port", default=44445, + help="The port on which the Constellation will serve webpages on.") +@click.option("-a", "--alchemy-url", type=str, default=None, + help="The Alchemy database path.") +@click.option("-h", "--herald-port", type=int, default=44444, + help="The port on which the Herald should be running.") +@click.option("-p", "--pack", type=str, multiple=True, default=tuple(), + help="Import the pack with the specified name and use it in the Royalnet instance.") @click.option("-s", "--secrets-name", type=str, default="__default__", help="The name in the keyring that the secrets are stored with.") -@click.option("-v", "--verbose", is_flag=True, default=False, - help="Print all possible debug information.") +@click.option("-l", "--log-level", type=str, default="INFO", + help="Select how much information you want to be printed on the console." + " Valid log levels are: FATAL/ERROR/WARNING/INFO/DEBUG") def run(telegram: typing.Optional[bool], discord: typing.Optional[bool], - webserver: typing.Optional[bool], - webserver_port: typing.Optional[int], - database: typing.Optional[str], - packs: typing.Tuple[str], - network_address: typing.Optional[str], - local_network_server: bool, - local_network_server_port: int, + constellation: typing.Optional[bool], + herald: typing.Optional[bool], + remote_herald_address: typing.Optional[str], + constellation_port: int, + alchemy_url: typing.Optional[str], + herald_port: int, + pack: typing.Tuple[str], secrets_name: str, - verbose: bool): - # Setup logging - if verbose: - core_logger = logging.root - core_logger.setLevel(logging.DEBUG) - stream_handler = logging.StreamHandler() - stream_handler.formatter = logging.Formatter("{asctime}\t{name}\t{levelname}\t{message}", style="{") - core_logger.addHandler(stream_handler) - core_logger.debug("Logging setup complete.") + log_level: str): + # Initialize logging + royalnet_log: Logger = getLogger("royalnet") + royalnet_log.setLevel(log_level) + stream_handler = StreamHandler() + stream_handler.formatter = Formatter("{asctime}\t{name}\t{levelname}\t{message}", style="{") + royalnet_log.addHandler(stream_handler) - # Get the network password - network_password = keyring.get_password(f"Royalnet/{secrets_name}", "network") - - # Get the sentry dsn - sentry_dsn = keyring.get_password(f"Royalnet/{secrets_name}", "sentry") + def get_secret(username: str): + return keyring.get_password(f"Royalnet/{secrets_name}", username) # Enable / Disable interfaces interfaces = { "telegram": telegram, "discord": discord, - "webserver": webserver + "herald": herald, + "constellation": constellation, } # If any interface is True, then the undefined ones should be False if any(interfaces[name] is True for name in interfaces): @@ -79,36 +76,30 @@ def run(telegram: typing.Optional[bool], for name in interfaces: interfaces[name] = True - server_process: typing.Optional[multiprocessing.Process] = None - # Start the network server - if local_network_server: - server_process = multiprocessing.Process(name="Network Server", - target=rh.Server("0.0.0.0", local_network_server_port, network_password).run_blocking, + herald_process: typing.Optional[multiprocessing.Process] = None + # Start the Herald server + if interfaces["herald"]: + herald_config = r.herald.Config(name="", + address="127.0.0.1", + port=herald_port, + secret=get_secret("herald"), + secure=False, + path="/") + herald_process = multiprocessing.Process(name="Herald", + target=r.herald.Server(config=herald_config).run_blocking, daemon=True) - server_process.start() - network_address = f"ws://127.0.0.1:{local_network_server_port}/" - - # Create a Royalnet configuration - network_config: typing.Optional[rh.Config] = None - if network_address is not None: - network_config = rh.Config(network_address, network_password) - - # Create a Alchemy configuration - telegram_db_config: typing.Optional[r.alchemy.DatabaseConfig] = None - discord_db_config: typing.Optional[r.alchemy.DatabaseConfig] = None - if database is not None: - telegram_db_config = r.alchemy.DatabaseConfig(database, - r.packs.common.tables.User, - r.packs.common.tables.Telegram, - "tg_id") - discord_db_config = r.alchemy.DatabaseConfig(database, - r.packs.common.tables.User, - r.packs.common.tables.Discord, - "discord_id") + herald_process.start() + else: + herald_config = r.herald.Config(name=..., + address=remote_herald_address, + port=herald_port, + secret=get_secret("herald"), + secure=False, + path="/") # Import command and star packs - packs: typing.List[str] = list(packs) - packs.append("royalnet.packs.common") # common pack is always imported + packs: typing.List[str] = list(pack) + packs.append("royalnet.backpack") # backpack is always imported enabled_commands = [] enabled_page_stars = [] enabled_exception_stars = [] @@ -132,62 +123,66 @@ def run(telegram: typing.Optional[bool], telegram_process: typing.Optional[multiprocessing.Process] = None if interfaces["telegram"]: - click.echo("\n@BotFather Commands String") - for command in enabled_commands: - click.echo(f"{command.name} - {command.description}") - click.echo("") - telegram_bot = r.interfaces.TelegramBot(network_config=network_config, - database_config=telegram_db_config, - sentry_dsn=sentry_dsn, - commands=enabled_commands, - secrets_name=secrets_name) - telegram_process = multiprocessing.Process(name="Telegram Interface", - target=telegram_bot.run_blocking, - args=(verbose,), + telegram_db_config = r.serf.AlchemyConfig(database_url=alchemy_url, + master_table=r.backpack.tables.User, + identity_table=r.backpack.tables.Telegram, + identity_column="tg_id") + telegram_serf_kwargs = { + 'alchemy_config': telegram_db_config, + 'commands': enabled_commands, + 'network_config': herald_config.copy(name="telegram"), + 'secrets_name': secrets_name + } + telegram_process = multiprocessing.Process(name="Telegram Serf", + target=r.serf.telegram.TelegramSerf.run_process, + kwargs=telegram_serf_kwargs, daemon=True) telegram_process.start() discord_process: typing.Optional[multiprocessing.Process] = None if interfaces["discord"]: - discord_bot = r.interfaces.DiscordBot(network_config=network_config, - database_config=discord_db_config, - sentry_dsn=sentry_dsn, - commands=enabled_commands, - secrets_name=secrets_name) - discord_process = multiprocessing.Process(name="Discord Interface", - target=discord_bot.run_blocking, - args=(verbose,), + discord_db_config = r.serf.AlchemyConfig(database_url=alchemy_url, + master_table=r.backpack.tables.User, + identity_table=r.backpack.tables.Discord, + identity_column="discord_id") + discord_serf_kwargs = { + 'alchemy_config': discord_db_config, + 'commands': enabled_commands, + 'network_config': herald_config.copy(name="discord"), + 'secrets_name': secrets_name + } + discord_process = multiprocessing.Process(name="Discord Serf", + target=r.serf.discord.DiscordSerf.run_process, + kwargs=discord_serf_kwargs, daemon=True) discord_process.start() - webserver_process: typing.Optional[multiprocessing.Process] = None - if interfaces["webserver"]: - # Common tables are always included - constellation_tables = set(r.packs.common.available_tables) - # Find the required tables - for star in [*enabled_page_stars, *enabled_exception_stars]: - constellation_tables = constellation_tables.union(star.tables) + constellation_process: typing.Optional[multiprocessing.Process] = None + if interfaces["constellation"]: # Create the Constellation - constellation = r.web.Constellation(page_stars=enabled_page_stars, - exc_stars=enabled_exception_stars, - secrets_name=secrets_name, - database_uri=database, - tables=constellation_tables) - webserver_process = multiprocessing.Process(name="Constellation Webserver", - target=constellation.run_blocking, - args=("0.0.0.0", webserver_port, verbose,), - daemon=True) - webserver_process.start() + constellation_kwargs = { + 'address': "127.0.0.1", + 'port': constellation_port, + 'secrets_name': secrets_name, + 'database_uri': alchemy_url, + 'page_stars': enabled_page_stars, + 'exc_stars': enabled_exception_stars, + } + constellation_process = multiprocessing.Process(name="Constellation", + target=r.constellation.Constellation.run_process, + kwargs=constellation_kwargs, + daemon=True) + constellation_process.start() - click.echo("Royalnet processes have been started. You can force-quit by pressing Ctrl+C.") - if server_process is not None: - server_process.join() + click.echo("Royalnet is now running! You can stop its execution by pressing Ctrl+C at any time.") + if herald_process is not None: + herald_process.join() if telegram_process is not None: telegram_process.join() if discord_process is not None: discord_process.join() - if webserver_process is not None: - webserver_process.join() + if constellation_process is not None: + constellation_process.join() if __name__ == "__main__": diff --git a/royalnet/backpack/README.md b/royalnet/backpack/README.md new file mode 100644 index 00000000..0121370f --- /dev/null +++ b/royalnet/backpack/README.md @@ -0,0 +1,3 @@ +# `backpack` + +A Pack that is imported by default by all `royalnet` instances. diff --git a/royalnet/packs/default/__init__.py b/royalnet/backpack/__init__.py similarity index 77% rename from royalnet/packs/default/__init__.py rename to royalnet/backpack/__init__.py index feb329ab..651cbe4e 100644 --- a/royalnet/packs/default/__init__.py +++ b/royalnet/backpack/__init__.py @@ -1,4 +1,4 @@ -# This is a template Pack __init__. You can use this without changing anything in other packages too! +"""A Pack that is imported by default by all :mod:`royalnet` instances.""" from . import commands, tables, stars from .commands import available_commands diff --git a/royalnet/packs/default/commands/__init__.py b/royalnet/backpack/commands/__init__.py similarity index 100% rename from royalnet/packs/default/commands/__init__.py rename to royalnet/backpack/commands/__init__.py diff --git a/royalnet/packs/default/commands/ping.py b/royalnet/backpack/commands/ping.py similarity index 100% rename from royalnet/packs/default/commands/ping.py rename to royalnet/backpack/commands/ping.py diff --git a/royalnet/packs/default/commands/version.py b/royalnet/backpack/commands/version.py similarity index 78% rename from royalnet/packs/default/commands/version.py rename to royalnet/backpack/commands/version.py index 1a66f9e3..7d7cf56b 100644 --- a/royalnet/packs/default/commands/version.py +++ b/royalnet/backpack/commands/version.py @@ -1,5 +1,5 @@ +import royalnet from royalnet.commands import * -from royalnet.version import semantic class VersionCommand(Command): @@ -8,7 +8,7 @@ class VersionCommand(Command): description: str = "Get the current Royalnet version." async def run(self, args: CommandArgs, data: CommandData) -> None: - message = f"ℹ️ Royalnet {semantic}\n" + message = f"ℹ️ Royalnet {royalnet.__version__}\n" if "69" in message: message += "(Nice.)" await data.reply(message) diff --git a/royalnet/packs/default/stars/__init__.py b/royalnet/backpack/stars/__init__.py similarity index 100% rename from royalnet/packs/default/stars/__init__.py rename to royalnet/backpack/stars/__init__.py diff --git a/royalnet/packs/default/stars/api_royalnet_version.py b/royalnet/backpack/stars/api_royalnet_version.py similarity index 64% rename from royalnet/packs/default/stars/api_royalnet_version.py rename to royalnet/backpack/stars/api_royalnet_version.py index 37e9e487..d9a3f4f7 100644 --- a/royalnet/packs/default/stars/api_royalnet_version.py +++ b/royalnet/backpack/stars/api_royalnet_version.py @@ -1,15 +1,18 @@ import royalnet from starlette.requests import Request from starlette.responses import * -from royalnet.web import PageStar +from royalnet.constellation import PageStar +from ..tables import available_tables class ApiRoyalnetVersionStar(PageStar): path = "/api/royalnet/version" + tables = set(available_tables) + async def page(self, request: Request) -> JSONResponse: return JSONResponse({ "version": { - "semantic": royalnet.version.semantic + "semantic": royalnet.__version__, } }) diff --git a/royalnet/packs/default/tables/__init__.py b/royalnet/backpack/tables/__init__.py similarity index 100% rename from royalnet/packs/default/tables/__init__.py rename to royalnet/backpack/tables/__init__.py diff --git a/royalnet/packs/default/tables/discord.py b/royalnet/backpack/tables/discord.py similarity index 100% rename from royalnet/packs/default/tables/discord.py rename to royalnet/backpack/tables/discord.py diff --git a/royalnet/packs/default/tables/telegram.py b/royalnet/backpack/tables/telegram.py similarity index 100% rename from royalnet/packs/default/tables/telegram.py rename to royalnet/backpack/tables/telegram.py diff --git a/royalnet/packs/default/tables/users.py b/royalnet/backpack/tables/users.py similarity index 100% rename from royalnet/packs/default/tables/users.py rename to royalnet/backpack/tables/users.py diff --git a/royalnet/bard/ytdlmp3.py b/royalnet/bard/ytdlmp3.py index c06cb04e..5db0d867 100644 --- a/royalnet/bard/ytdlmp3.py +++ b/royalnet/bard/ytdlmp3.py @@ -34,7 +34,7 @@ class YtdlMp3: ) self.mp3_filename = destination_filename - def delete_asap(self) -> None: + async def delete_asap(self) -> None: """Delete the mp3 file.""" if self.is_converted: async with self.lock.exclusive(): diff --git a/royalnet/commands/commanddata.py b/royalnet/commands/commanddata.py index 32b09203..6dd48c78 100644 --- a/royalnet/commands/commanddata.py +++ b/royalnet/commands/commanddata.py @@ -1,3 +1,4 @@ +from asyncio import AbstractEventLoop from typing import Optional, TYPE_CHECKING from .errors import UnsupportedError from .commandinterface import CommandInterface @@ -8,9 +9,10 @@ if TYPE_CHECKING: class CommandData: - def __init__(self, interface: CommandInterface, session: Optional["Session"]): + def __init__(self, interface: CommandInterface, session: Optional["Session"], loop: AbstractEventLoop): self._interface: CommandInterface = interface self._session: Optional["Session"] = session + self.loop: AbstractEventLoop = loop @property def session(self) -> "Session": diff --git a/royalnet/configurator.py b/royalnet/configurator.py index fc78e573..2bc70d38 100644 --- a/royalnet/configurator.py +++ b/royalnet/configurator.py @@ -6,9 +6,9 @@ import keyring def run(): click.echo("Welcome to the Royalnet configuration creator!") secrets_name = click.prompt("Desired secrets name", default="__default__") - network = click.prompt("Network password", default="") + network = click.prompt("Herald password", default="") if network: - keyring.set_password(f"Royalnet/{secrets_name}", "network", network) + keyring.set_password(f"Royalnet/{secrets_name}", "herald", network) telegram = click.prompt("Telegram Bot API token", default="") if telegram: keyring.set_password(f"Royalnet/{secrets_name}", "telegram", telegram) @@ -21,9 +21,6 @@ def run(): sentry = click.prompt("Sentry DSN", default="") if sentry: keyring.set_password(f"Royalnet/{secrets_name}", "sentry", sentry) - leagueoflegends = click.prompt("League of Legends API Token", default="") - if leagueoflegends: - keyring.set_password(f"Royalnet/{secrets_name}", "leagueoflegends", leagueoflegends) if __name__ == "__main__": diff --git a/royalnet/constellation/constellation.py b/royalnet/constellation/constellation.py index eedd6b70..2b9e9aca 100644 --- a/royalnet/constellation/constellation.py +++ b/royalnet/constellation/constellation.py @@ -2,7 +2,6 @@ import typing import logging import royalnet import keyring -from royalnet.alchemy import Alchemy from .star import PageStar, ExceptionStar try: @@ -60,7 +59,7 @@ class Constellation: """The :class:`Starlette` app.""" log.debug("Finding required Tables...") - tables = set() + tables = set(royalnet.backpack.available_tables) for SelectedPageStar in page_stars: tables = tables.union(SelectedPageStar.tables) for SelectedExcStar in exc_stars: @@ -68,7 +67,7 @@ class Constellation: log.debug(f"Found Tables: {' '.join([table.__name__ for table in tables])}") log.info(f"Creating Alchemy...") - self.alchemy: Alchemy = Alchemy(database_uri=database_uri, tables=tables) + self.alchemy: royalnet.alchemy.Alchemy = royalnet.alchemy.Alchemy(database_uri=database_uri, tables=tables) """The :class:`Alchemy: of this Constellation.""" log.info("Registering PageStars...") @@ -98,19 +97,34 @@ class Constellation: username: the name of the secret that should be retrieved.""" return keyring.get_password(f"Royalnet/{self.secrets_name}", username) - def run_blocking(self, address: str, port: int): - """Blockingly run the Constellation. + @classmethod + def run_process(cls, + address: str, + port: int, + secrets_name: str, + database_uri: str, + page_stars: typing.List[typing.Type[PageStar]] = None, + exc_stars: typing.List[typing.Type[ExceptionStar]] = None, + *, + debug: bool = __debug__,): + """Blockingly create and run the Constellation. This should be used as the target of a :class:`multiprocessing.Process`. Args: address: The IP address this Constellation should bind to. port: The port this Constellation should listen for requests on.""" + constellation = cls(secrets_name=secrets_name, + database_uri=database_uri, + page_stars=page_stars, + exc_stars=exc_stars, + debug=debug) + # Initialize Sentry on the process if sentry_sdk is None: log.info("Sentry: not installed") else: - sentry_dsn = self.get_secret("sentry") + sentry_dsn = constellation.get_secret("sentry") if not sentry_dsn: log.info("Sentry: disabled") else: @@ -128,11 +142,11 @@ class Constellation: log.info(f"Sentry: enabled (Royalnet {release})") # Run the server log.info(f"Running Constellation on {address}:{port}...") - self.running = True + constellation.running = True try: - uvicorn.run(self.starlette, host=address, port=port) + uvicorn.run(constellation.starlette, host=address, port=port) finally: - self.running = False + constellation.running = False def __repr__(self): return f"<{self.__class__.__qualname__}: {'running' if self.running else 'inactive'}>" diff --git a/royalnet/herald/config.py b/royalnet/herald/config.py index 1b9b1a44..52095dd8 100644 --- a/royalnet/herald/config.py +++ b/royalnet/herald/config.py @@ -1,3 +1,6 @@ +from typing import Optional + + class Config: def __init__(self, name: str, @@ -30,5 +33,20 @@ class Config: def url(self): return f"ws{'s' if self.secure else ''}://{self.address}:{self.port}{self.path}" + def copy(self, + name: Optional[str] = None, + address: Optional[str] = None, + port: Optional[int] = None, + secret: Optional[str] = None, + secure: Optional[bool] = None, + path: Optional[str] = None): + """Create an exact copy of this configuration, but with different parameters.""" + return self.__class__(name=name if name else self.name, + address=address if address else self.address, + port=port if port else self.port, + secret=secret if secret else self.secret, + secure=secure if secure else self.secure, + path=path if path else self.path) + def __repr__(self): return f"" diff --git a/royalnet/serf/__init__.py b/royalnet/serf/__init__.py index d62e46f2..a82f8f5a 100644 --- a/royalnet/serf/__init__.py +++ b/royalnet/serf/__init__.py @@ -1,7 +1,10 @@ from .serf import Serf from .alchemyconfig import AlchemyConfig +from . import telegram, discord __all__ = [ "Serf", - "AlchemyConfig" + "AlchemyConfig", + "telegram", + "discord", ] diff --git a/royalnet/serf/alchemyconfig.py b/royalnet/serf/alchemyconfig.py index af67b008..6be4774d 100644 --- a/royalnet/serf/alchemyconfig.py +++ b/royalnet/serf/alchemyconfig.py @@ -1,18 +1,16 @@ -from typing import Type, TYPE_CHECKING -if TYPE_CHECKING: - from sqlalchemy.schema import Table +from typing import TYPE_CHECKING class AlchemyConfig: """A helper class to configure :class:`Alchemy` in a :class:`Serf`.""" def __init__(self, database_url: str, - master_table: Type[Table], - identity_table: Type[Table], + master_table: type, + identity_table: type, identity_column: str): self.database_url: str = database_url - self.master_table: Type[Table] = master_table - self.identity_table: Type[Table] = identity_table + self.master_table: type = master_table + self.identity_table: type = identity_table self.identity_column: str = identity_column def __repr__(self): diff --git a/royalnet/serf/discord/discordserf.py b/royalnet/serf/discord/discordserf.py index f40350db..d03dab23 100644 --- a/royalnet/serf/discord/discordserf.py +++ b/royalnet/serf/discord/discordserf.py @@ -1,3 +1,4 @@ +import asyncio import logging from typing import Type, Optional, List, Union from royalnet.commands import Command, CommandInterface, CommandData, CommandArgs, CommandError, InvalidInputError, \ @@ -63,8 +64,12 @@ class DiscordSerf(Serf): def data_factory(self) -> Type[CommandData]: # noinspection PyMethodParameters,PyAbstractClass class DiscordData(CommandData): - def __init__(data, interface: CommandInterface, session, message: discord.Message): - super().__init__(interface=interface, session=session) + def __init__(data, + interface: CommandInterface, + session, + loop: asyncio.AbstractEventLoop, + message: discord.Message): + super().__init__(interface=interface, session=session, loop=loop) data.message = message async def reply(data, text: str): @@ -118,7 +123,7 @@ class DiscordSerf(Serf): else: session = None # Prepare data - data = self.Data(interface=command.interface, session=session, message=message) + data = self.Data(interface=command.interface, session=session, loop=self.loop, message=message) try: # Run the command await command.run(CommandArgs(parameters), data) @@ -196,6 +201,7 @@ class DiscordSerf(Serf): return DiscordClient async def run(self): + await super().run() token = self.get_secret("discord") await self.client.login(token) await self.client.connect() diff --git a/royalnet/serf/serf.py b/royalnet/serf/serf.py index 8cbd6015..3529664f 100644 --- a/royalnet/serf/serf.py +++ b/royalnet/serf/serf.py @@ -1,5 +1,5 @@ import logging -from asyncio import Task, AbstractEventLoop +from asyncio import Task, AbstractEventLoop, get_event_loop from typing import Type, Optional, Awaitable, Dict, List, Any, Callable, Union, Set from keyring import get_password from sqlalchemy.schema import Table @@ -130,10 +130,10 @@ class Serf: self.alchemy = Alchemy(alchemy_config.database_url, tables) self._master_table = self.alchemy.get(alchemy_config.master_table) self._identity_table = self.alchemy.get(alchemy_config.identity_table) - # FIXME: this MAY break - self._identity_column = self._identity_table.__getattribute__(alchemy_config.identity_column) - # self._identity_column = self._identity_table.__getattribute__(self._identity_table, - # alchemy_config.identity_column) + # This is fine, as Pycharm doesn't know that identity_table is a class and not an object + # noinspection PyArgumentList + self._identity_column = self._identity_table.__getattribute__(self._identity_table, + alchemy_config.identity_column) @property def _identity_chain(self) -> tuple: @@ -147,7 +147,6 @@ class Serf: class GenericInterface(CommandInterface): alchemy: Alchemy = self.alchemy bot: "Serf" = self - loop: AbstractEventLoop = self.loop def register_herald_action(ci, event_name: str, @@ -232,10 +231,9 @@ class Serf: def init_network(self, config: HeraldConfig): """Create a :py:class:`Link`, and run it as a :py:class:`asyncio.Task`.""" log.debug(f"Initializing herald...") - self.herald: Link = Link(config, self._network_handler) - self.herald_task = self.loop.create_task(self.herald.run()) + self.herald: Link = Link(config, self.network_handler) - async def _network_handler(self, message: Union[Request, Broadcast]) -> Response: + async def network_handler(self, message: Union[Request, Broadcast]) -> Response: try: network_handler = self.herald_handlers[message.handler] except KeyError: @@ -288,19 +286,24 @@ class Serf: async def run(self): """A coroutine that starts the event loop and handles command calls.""" - raise NotImplementedError() + self.herald_task = self.loop.create_task(self.herald.run()) + # OVERRIDE THIS METHOD! - def run_blocking(self): - """Blockingly run the Serf. + @classmethod + def run_process(cls, *args, **kwargs): + """Blockingly create and run the Serf. This should be used as the target of a :class:`multiprocessing.Process`.""" + serf = cls(*args, **kwargs) + if sentry_sdk is None: log.info("Sentry: not installed") else: - sentry_dsn = self.get_secret("sentry") + sentry_dsn = serf.get_secret("sentry") if sentry_dsn is None: log.info("Sentry: disabled") else: - self.init_sentry(sentry_dsn) + serf.init_sentry(sentry_dsn) - self.loop.run_until_complete(self.run()) + serf.loop = get_event_loop() + serf.loop.run_until_complete(serf.run()) diff --git a/royalnet/serf/telegram/telegramserf.py b/royalnet/serf/telegram/telegramserf.py index c6db695c..84908346 100644 --- a/royalnet/serf/telegram/telegramserf.py +++ b/royalnet/serf/telegram/telegramserf.py @@ -99,8 +99,12 @@ class TelegramSerf(Serf): def data_factory(self) -> Type[CommandData]: # noinspection PyMethodParameters class TelegramData(CommandData): - def __init__(data, interface: CommandInterface, session, update: telegram.Update): - super().__init__(interface=interface, session=session) + def __init__(data, + interface: CommandInterface, + session, + loop: asyncio.AbstractEventLoop, + update: telegram.Update): + super().__init__(interface=interface, session=session, loop=loop) data.update = update async def reply(data, text: str): @@ -192,7 +196,7 @@ class TelegramSerf(Serf): session = None try: # Create the command data - data = self.Data(interface=command.interface, session=session, update=update) + data = self.Data(interface=command.interface, session=session, loop=self.loop, update=update) try: # Run the command await command.run(CommandArgs(parameters), data) @@ -240,6 +244,7 @@ class TelegramSerf(Serf): pass async def run(self): + await super().run() while True: # Get the latest 100 updates last_updates: List[telegram.Update] = await self.api_call(self.client.get_updates, diff --git a/royalnet/utils/__init__.py b/royalnet/utils/__init__.py index a8c18e40..81a618a4 100644 --- a/royalnet/utils/__init__.py +++ b/royalnet/utils/__init__.py @@ -1,11 +1,7 @@ -"""Miscellaneous useful functions and classes.""" - from .asyncify import asyncify -from .escaping import telegram_escape, discord_escape from .safeformat import safeformat -from .classdictjanitor import cdj from .sleep_until import sleep_until -from .formatters import andformat, plusformat, underscorize, ytdldateformat, numberemojiformat, splitstring, ordinalformat +from .formatters import andformat, underscorize, ytdldateformat, numberemojiformat, splitstring, ordinalformat from .urluuid import to_urluuid, from_urluuid from .multilock import MultiLock diff --git a/royalnet/web/constellation.py b/royalnet/web/constellation.py deleted file mode 100644 index 9eed5786..00000000 --- a/royalnet/web/constellation.py +++ /dev/null @@ -1,95 +0,0 @@ -import typing -import uvicorn -import logging -import sentry_sdk -from sentry_sdk.integrations.aiohttp import AioHttpIntegration -from sentry_sdk.integrations.sqlalchemy import SqlalchemyIntegration -from sentry_sdk.integrations.logging import LoggingIntegration -import royalnet -import keyring -from starlette.applications import Starlette -from .star import PageStar, ExceptionStar - - -log = logging.getLogger(__name__) - - -class Constellation: - def __init__(self, - secrets_name: str, - database_uri: str, - tables: set, - page_stars: typing.List[typing.Type[PageStar]] = None, - exc_stars: typing.List[typing.Type[ExceptionStar]] = None, - *, - debug: bool = __debug__,): - if page_stars is None: - page_stars = [] - - if exc_stars is None: - exc_stars = [] - - self.secrets_name: str = secrets_name - - log.info("Creating starlette app...") - self.starlette = Starlette(debug=debug) - - log.info(f"Creating alchemy with tables: {' '.join([table.__name__ for table in tables])}") - self.alchemy: royalnet.alchemy.Alchemy = royalnet.alchemy.Alchemy(database_uri=database_uri, tables=tables) - - log.info("Registering page_stars...") - for SelectedPageStar in page_stars: - try: - page_star_instance = SelectedPageStar(constellation=self) - except Exception as e: - log.error(f"{e.__class__.__qualname__} during the registration of {SelectedPageStar.__qualname__}") - sentry_sdk.capture_exception(e) - continue - log.info(f"Registering: {page_star_instance.path} -> {page_star_instance.__class__.__name__}") - self.starlette.add_route(page_star_instance.path, page_star_instance.page, page_star_instance.methods) - - log.info("Registering exc_stars...") - for SelectedExcStar in exc_stars: - try: - exc_star_instance = SelectedExcStar(constellation=self) - except Exception as e: - log.error(f"{e.__class__.__qualname__} during the registration of {SelectedExcStar.__qualname__}") - sentry_sdk.capture_exception(e) - continue - log.info(f"Registering: {exc_star_instance.error} -> {exc_star_instance.__class__.__name__}") - self.starlette.add_exception_handler(exc_star_instance.error, exc_star_instance.page) - - def _init_sentry(self): - sentry_dsn = self.get_secret("sentry") - if sentry_dsn: - # noinspection PyUnreachableCode - if __debug__: - release = "DEV" - else: - release = royalnet.version.semantic - log.info(f"Sentry: enabled (Royalnet {release})") - self.sentry = sentry_sdk.init(sentry_dsn, - integrations=[AioHttpIntegration(), - SqlalchemyIntegration(), - LoggingIntegration(event_level=None)], - release=release) - else: - log.info("Sentry: disabled") - - def get_secret(self, username: str): - return keyring.get_password(f"Royalnet/{self.secrets_name}", username) - - def set_secret(self, username: str, password: str): - return keyring.set_password(f"Royalnet/{self.secrets_name}", username, password) - - def run_blocking(self, address: str, port: int, verbose: bool): - if verbose: - core_logger = logging.root - core_logger.setLevel(logging.DEBUG) - stream_handler = logging.StreamHandler() - stream_handler.formatter = logging.Formatter("{asctime}\t{name}\t{levelname}\t{message}", style="{") - core_logger.addHandler(stream_handler) - core_logger.debug("Logging setup complete.") - self._init_sentry() - log.info(f"Running constellation server on {address}:{port}...") - uvicorn.run(self.starlette, host=address, port=port) diff --git a/royalnet/web/star.py b/royalnet/web/star.py deleted file mode 100644 index eb5e40d1..00000000 --- a/royalnet/web/star.py +++ /dev/null @@ -1,37 +0,0 @@ -import typing -from starlette.requests import Request -from starlette.responses import Response -if typing.TYPE_CHECKING: - from .constellation import Constellation - - -class Star: - tables: set = {} - - def __init__(self, constellation: "Constellation"): - self.constellation: "Constellation" = constellation - - async def page(self, request: Request) -> Response: - raise NotImplementedError() - - @property - def alchemy(self): - return self.constellation.alchemy - - @property - def Session(self): - return self.constellation.alchemy._Session - - @property - def session_acm(self): - return self.constellation.alchemy.session_acm - - -class PageStar(Star): - path: str = NotImplemented - - methods: typing.List[str] = ["GET"] - - -class ExceptionStar(Star): - error: typing.Union[typing.Type[Exception], int]