From 5a30fa85482f5fe6ad67ca5fbc37319b490a46e5 Mon Sep 17 00:00:00 2001 From: Stefano Pigozzi Date: Tue, 19 Oct 2021 19:39:45 +0200 Subject: [PATCH] =?UTF-8?q?=F0=9F=92=A5=20Prepare=20backend=20for=20docker?= =?UTF-8?q?=20deployment?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/sophon/notebooks/admin.py | 2 +- backend/sophon/notebooks/apache.py | 2 +- backend/sophon/notebooks/docker.py | 9 ++- .../migrations/0010_auto_20211019_1738.py | 30 ++++++++ backend/sophon/notebooks/models.py | 75 ++++++++++++------- backend/sophon/settings.py | 34 ++++++++- 6 files changed, 122 insertions(+), 30 deletions(-) create mode 100644 backend/sophon/notebooks/migrations/0010_auto_20211019_1738.py diff --git a/backend/sophon/notebooks/admin.py b/backend/sophon/notebooks/admin.py index 3e4c9c6..add45a7 100644 --- a/backend/sophon/notebooks/admin.py +++ b/backend/sophon/notebooks/admin.py @@ -13,7 +13,6 @@ class NotebookAdmin(SophonAdmin): "locked_by", "container_image", "container_id", - "port", ) list_filter = ( @@ -48,6 +47,7 @@ class NotebookAdmin(SophonAdmin): "Proxy", { "fields": ( "port", + "internal_url", ), }, ), diff --git a/backend/sophon/notebooks/apache.py b/backend/sophon/notebooks/apache.py index 7ac41b2..b01b37d 100644 --- a/backend/sophon/notebooks/apache.py +++ b/backend/sophon/notebooks/apache.py @@ -52,7 +52,7 @@ class ApacheDB: del adb[key] -db = lazy_object_proxy.Proxy(lambda: ApacheDB(settings.PROXY_FILE)) +db: ApacheDB = lazy_object_proxy.Proxy(lambda: ApacheDB(settings.PROXY_FILE)) def get_ephemeral_port() -> int: diff --git a/backend/sophon/notebooks/docker.py b/backend/sophon/notebooks/docker.py index a8f84e1..77be0d7 100644 --- a/backend/sophon/notebooks/docker.py +++ b/backend/sophon/notebooks/docker.py @@ -21,7 +21,7 @@ def get_docker_client() -> docker.DockerClient: return result -client = lazy_object_proxy.Proxy(get_docker_client) +client: docker.DockerClient = lazy_object_proxy.Proxy(get_docker_client) class HealthState(enum.IntEnum): @@ -56,6 +56,13 @@ def get_health(container: docker.models.containers.Container) -> HealthState: return state +def get_proxy_container() -> docker.models.containers.Container: + """ + :return: The container of the proxy, having the name specified in `settings.PROXY_CONTAINER_NAME`. + """ + return client.containers.get(settings.PROXY_CONTAINER_NAME) + + def sleep_until_container_has_started(container: docker.models.containers.Container) -> HealthState: """ Sleep until the specified container is not anymore in the ``starting`` state. diff --git a/backend/sophon/notebooks/migrations/0010_auto_20211019_1738.py b/backend/sophon/notebooks/migrations/0010_auto_20211019_1738.py new file mode 100644 index 0000000..ff04150 --- /dev/null +++ b/backend/sophon/notebooks/migrations/0010_auto_20211019_1738.py @@ -0,0 +1,30 @@ +# Generated by Django 3.2.7 on 2021-10-19 17:38 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ('notebooks', '0009_alter_notebook_slug'), + ] + + operations = [ + migrations.RemoveField( + model_name='notebook', + name='internet_access', + ), + migrations.AddField( + model_name='notebook', + name='internal_url', + field=models.IntegerField(blank=True, + help_text='The URL reachable from the proxy where the container is available. Can be null if the notebook is not running.', + null=True, verbose_name='Internal URL'), + ), + migrations.AlterField( + model_name='notebook', + name='port', + field=models.IntegerField(blank=True, + help_text='The port number of the local machine at which the container is available. Can be null if the notebook is not running, or if the proxy itself is running in a Docker container.', + null=True, verbose_name='Local port number'), + ), + ] diff --git a/backend/sophon/notebooks/models.py b/backend/sophon/notebooks/models.py index 797e1e0..d740652 100644 --- a/backend/sophon/notebooks/models.py +++ b/backend/sophon/notebooks/models.py @@ -1,7 +1,6 @@ from __future__ import annotations import logging -import os import typing as t import docker.errors @@ -17,7 +16,7 @@ from sophon.core.models import SophonGroupModel, ResearchGroup from sophon.notebooks.apache import db as apache_db from sophon.notebooks.apache import get_ephemeral_port from sophon.notebooks.docker import client as docker_client -from sophon.notebooks.docker import sleep_until_container_has_started +from sophon.notebooks.docker import sleep_until_container_has_started, get_proxy_container from sophon.notebooks.jupyter import generate_secure_token from sophon.notebooks.validators import DisallowedValuesValidator from sophon.projects.models import ResearchProject @@ -94,14 +93,6 @@ class Notebook(SophonGroupModel): default="steffo45/jupyterlab-docker-sophon", ) - # TODO: Find a way to prevent Internet access without using the --internal flag, as it doesn't allow to expose ports - internet_access = models.BooleanField( - "Allow internet access", - help_text="If true, the notebook will be able to access the Internet as the host machine. Can only be set by a superuser via the admin interface. " - "Does not currently do anything.", - default=True, - ) - jupyter_token = models.CharField( "Jupyter Access Token", help_text="The token to allow access to the JupyterLab editor.", @@ -118,7 +109,13 @@ class Notebook(SophonGroupModel): port = models.IntegerField( "Local port number", - help_text="The port number of the local machine at which the container is available. Can be null if the notebook is not running.", + help_text="The port number of the local machine at which the container is available. Can be null if the notebook is not running, or if the proxy itself is running in a Docker container.", + blank=True, null=True, + ) + + internal_url = models.IntegerField( + "Internal URL", + help_text="The URL reachable from the proxy where the container is available. Can be null if the notebook is not running.", blank=True, null=True, ) @@ -176,21 +173,21 @@ class Notebook(SophonGroupModel): """ :return: The name given to the container associated with this :class:`Notebook`. """ - return f"{os.environ.get('SOPHON_CONTAINER_PREFIX', 'sophon-container')}-{self.slug}" + return f"{settings.CONTAINER_PREFIX}-{self.slug}" @property def volume_name(self) -> str: """ :return: The name given to the volume associated with this :class:`Notebook`. """ - return f"{os.environ.get('SOPHON_VOLUME_PREFIX', 'sophon-volume')}-{self.slug}" + return f"{settings.VOLUME_PREFIX}-{self.slug}" @property def network_name(self) -> str: """ :return: The name given to the network associated with this :class:`Notebook`. """ - return f"{os.environ.get('SOPHON_NETWORK_PREFIX', 'sophon-network')}-{self.slug}" + return f"{settings.NETWORK_PREFIX}-{self.slug}" @property def external_domain(self) -> str: @@ -222,10 +219,10 @@ class Notebook(SophonGroupModel): return f"{settings.PROXY_PROTOCOL}://{self.external_domain}/tree?token={self.jupyter_token}" @property - def internal_domain(self) -> t.Optional[str]: + def local_domain(self) -> t.Optional[str]: """ :return: The domain name where this :class:`Notebook` is accessible on the local machine, or :data:`None` if no port has been assigned to this - container yet. + container yet or if the proxy itself is running in a container. """ if self.port is None: return None @@ -307,7 +304,7 @@ class Notebook(SophonGroupModel): self.log.debug(f"Network does not exist, creating it now...") network = docker_client.networks.create( name=self.network_name, - internal=not self.internet_access, + internal=True, ) self.log.info(f"Created {network!r}") return network @@ -316,9 +313,19 @@ class Notebook(SophonGroupModel): """ Disable the proxying of this :class:`Notebook` by removing its URLs from the :data:`apache_db` and its port from :attr:`.port`. """ + if settings.PROXY_CONTAINER_NAME: + self.log.debug("Getting the notebook network...") + network = self.get_network() - self.log.debug("Unassigning port...") - self.port = None + self.log.debug("Getting proxy container...") + proxy = get_proxy_container() + + self.log.debug("Disconnecting proxy container from the network...") + network.disconnect(proxy) + + else: + self.log.debug("Unassigning port...") + self.port = None self.log.debug("Removing entry from the apache_db...") try: @@ -326,7 +333,7 @@ class Notebook(SophonGroupModel): except KeyError: pass - self.log.debug("Clearing port from the SQL database...") + self.log.debug("Clearing proxy data from the SQL database...") self.save() def enable_proxying(self) -> None: @@ -334,13 +341,30 @@ class Notebook(SophonGroupModel): Enable the proxying of this :class:`Notebook` by adding its URLs to the :data:`apache_db` and its port to :attr:`.port`. """ - self.log.debug("Getting free port...") - self.port: int = get_ephemeral_port() + if settings.PROXY_CONTAINER_NAME: + self.log.debug("Getting the notebook network...") + network = self.get_network() - self.log.debug("Adding entry to the apache_db...") - apache_db[self.external_domain] = f"{self.internal_domain}" + self.log.debug("Getting proxy container...") + proxy = get_proxy_container() - self.log.debug("Saving port to the SQL database...") + self.log.debug("Connecting proxy container to the network...") + network.connect(proxy) + + self.log.debug("Setting internal_url...") + self.internal_url = f"http://{self.container_name}:8888" + + self.log.debug("Adding entry to the apache_db...") + apache_db[bytes(self.external_domain, encoding="ascii")] = bytes(self.internal_url, encoding="ascii") + + else: + self.log.debug("Getting free port...") + self.port: int = get_ephemeral_port() + + self.log.debug("Adding entry to the apache_db...") + apache_db[bytes(self.external_domain, encoding="ascii")] = bytes(f"{self.local_domain}", encoding="ascii") + + self.log.debug("Saving proxy data to the SQL database...") self.save() def get_container(self) -> t.Optional[docker.models.containers.Container]: @@ -473,6 +497,7 @@ class Notebook(SophonGroupModel): "mode": "rw", } }, + network=network, ) self.log.debug("Storing container_id in the SQL database...") diff --git a/backend/sophon/settings.py b/backend/sophon/settings.py index 52d9203..c75375f 100644 --- a/backend/sophon/settings.py +++ b/backend/sophon/settings.py @@ -73,20 +73,22 @@ except KeyError: SECRET_KEY = secrets.token_urlsafe(24) log.debug(f"SECRET_KEY = {'*' * len(SECRET_KEY)}") + # Set debug mode DEBUG = __debug__ if DEBUG: log.warning("DEBUG mode is on, run the Python interpreter with the -O option to disable.") log.debug(f"{DEBUG = }") + # Set the hosts from which the admin page can be accessed, separated by pipes -ALLOWED_HOSTS = os.environ.get("DJANGO_ALLOWED_HOSTS", "").split("|") +ALLOWED_HOSTS = os.environ.get("DJANGO_ALLOWED_HOSTS", "").split("|") if "DJANGO_ALLOWED_HOSTS" in os.environ else [] if len(ALLOWED_HOSTS) == 0: log.warning(f"No DJANGO_ALLOWED_HOSTS are set, the admin page may not be accessible.") log.debug(f"{ALLOWED_HOSTS = }") # Set the origins from which the API can be called, separated by pipes -CORS_ALLOWED_ORIGINS = os.environ.get("DJANGO_ALLOWED_ORIGINS", "").split("|") +CORS_ALLOWED_ORIGINS = os.environ.get("DJANGO_ALLOWED_ORIGINS", "").split("|") if "DJANGO_ALLOWED_ORIGINS" in os.environ else [] if len(CORS_ALLOWED_ORIGINS) == 0: log.warning(f"No DJANGO_ALLOWED_ORIGINS are set, the API may not be usable.") log.debug(f"{CORS_ALLOWED_ORIGINS = }") @@ -205,6 +207,34 @@ except KeyError: PROXY_BASE_DOMAIN = "dev.sophon.steffo.eu" log.debug(f"{PROXY_BASE_DOMAIN = }") +# Set the name of the proxy docker container +PROXY_CONTAINER_NAME = os.environ.get("DJANGO_PROXY_CONTAINER_NAME") +log.debug(f"{PROXY_CONTAINER_NAME =}") + +# Set the prefix to add to all instantiated notebook container names +try: + DOCKER_CONTAINER_PREFIX = os.environ["DJANGO_DOCKER_CONTAINER_PREFIX"] +except KeyError: + log.warning("DOCKER_CONTAINER_PREFIX was not set, defaulting to `sophon-container`") + DOCKER_CONTAINER_PREFIX = "sophon-container" +log.debug(f"{DOCKER_CONTAINER_PREFIX = }") + +# Set the prefix to add to all instantiated notebook volume names +try: + DOCKER_VOLUME_PREFIX = os.environ["DJANGO_DOCKER_VOLUME_PREFIX"] +except KeyError: + log.warning("DOCKER_VOLUME_PREFIX was not set, defaulting to `sophon-volume`") + DOCKER_VOLUME_PREFIX = "sophon-volume" +log.debug(f"{DOCKER_VOLUME_PREFIX = }") + +# Set the prefix to add to all instantiated notebook network names +try: + DOCKER_NETWORK_PREFIX = os.environ["DJANGO_DOCKER_NETWORK_PREFIX"] +except KeyError: + log.warning("DOCKER_VOLUME_PREFIX was not set, defaulting to `sophon-network`") + DOCKER_NETWORK_PREFIX = "sophon-network" +log.debug(f"{DOCKER_NETWORK_PREFIX = }") + try: DOCKER_HOST = os.environ["DJANGO_DOCKER_HOST"] except KeyError: