steffo/sophon
steffo/sophon
1
Fork 0
mirror of https://github.com/Steffo99/sophon.git synced 2024-12-22 06:44:21 +00:00
Code Activity

💥 Prepare backend for docker deployment

Browse source
This commit is contained in:
Steffo 2021-10-19 19:39:45 +02:00
parent 96498fddd2
commit 5a30fa8548
6 changed files with 122 additions and 30 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
backend/sophon/notebooks/admin.py
View file

@ -13,7 +13,6 @@ class NotebookAdmin(SophonAdmin):
"locked_by",
"container_image",
"container_id",
"port",
)
list_filter = (
@ -48,6 +47,7 @@ class NotebookAdmin(SophonAdmin):
"Proxy", {
"fields": (
"port",
"internal_url",
),
},
),

2
backend/sophon/notebooks/apache.py
View file

@ -52,7 +52,7 @@ class ApacheDB:
del adb[key]
db = lazy_object_proxy.Proxy(lambda: ApacheDB(settings.PROXY_FILE))
db: ApacheDB = lazy_object_proxy.Proxy(lambda: ApacheDB(settings.PROXY_FILE))
def get_ephemeral_port() -> int:

9
backend/sophon/notebooks/docker.py
View file

@ -21,7 +21,7 @@ def get_docker_client() -> docker.DockerClient:
return result
client = lazy_object_proxy.Proxy(get_docker_client)
client: docker.DockerClient = lazy_object_proxy.Proxy(get_docker_client)
class HealthState(enum.IntEnum):
@ -56,6 +56,13 @@ def get_health(container: docker.models.containers.Container) -> HealthState:
return state
def get_proxy_container() -> docker.models.containers.Container:
"""
:return: The container of the proxy, having the name specified in `settings.PROXY_CONTAINER_NAME`.
"""
return client.containers.get(settings.PROXY_CONTAINER_NAME)
def sleep_until_container_has_started(container: docker.models.containers.Container) -> HealthState:
"""
Sleep until the specified container is not anymore in the ``starting`` state.

30
backend/sophon/notebooks/migrations/0010_auto_20211019_1738.py Normal file
View file

@ -0,0 +1,30 @@
# Generated by Django 3.2.7 on 2021-10-19 17:38
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('notebooks', '0009_alter_notebook_slug'),
]
operations = [
migrations.RemoveField(
model_name='notebook',
name='internet_access',
),
migrations.AddField(
model_name='notebook',
name='internal_url',
field=models.IntegerField(blank=True,
help_text='The URL reachable from the proxy where the container is available. Can be null if the notebook is not running.',
null=True, verbose_name='Internal URL'),
),
migrations.AlterField(
model_name='notebook',
name='port',
field=models.IntegerField(blank=True,
help_text='The port number of the local machine at which the container is available. Can be null if the notebook is not running, or if the proxy itself is running in a Docker container.',
null=True, verbose_name='Local port number'),
),
]

75
backend/sophon/notebooks/models.py
View file

@ -1,7 +1,6 @@
from __future__ import annotations
import logging
import os
import typing as t
import docker.errors
@ -17,7 +16,7 @@ from sophon.core.models import SophonGroupModel, ResearchGroup
from sophon.notebooks.apache import db as apache_db
from sophon.notebooks.apache import get_ephemeral_port
from sophon.notebooks.docker import client as docker_client
from sophon.notebooks.docker import sleep_until_container_has_started
from sophon.notebooks.docker import sleep_until_container_has_started, get_proxy_container
from sophon.notebooks.jupyter import generate_secure_token
from sophon.notebooks.validators import DisallowedValuesValidator
from sophon.projects.models import ResearchProject
@ -94,14 +93,6 @@ class Notebook(SophonGroupModel):
default="steffo45/jupyterlab-docker-sophon",
)
# TODO: Find a way to prevent Internet access without using the --internal flag, as it doesn't allow to expose ports
internet_access = models.BooleanField(
"Allow internet access",
help_text="If true, the notebook will be able to access the Internet as the host machine. Can only be set by a superuser via the admin interface. "
"<em>Does not currently do anything.</em>",
default=True,
)
jupyter_token = models.CharField(
"Jupyter Access Token",
help_text="The token to allow access to the JupyterLab editor.",
@ -118,7 +109,13 @@ class Notebook(SophonGroupModel):
port = models.IntegerField(
"Local port number",
help_text="The port number of the local machine at which the container is available. Can be null if the notebook is not running.",
help_text="The port number of the local machine at which the container is available. Can be null if the notebook is not running, or if the proxy itself is running in a Docker container.",
blank=True, null=True,
)
internal_url = models.IntegerField(
"Internal URL",
help_text="The URL reachable from the proxy where the container is available. Can be null if the notebook is not running.",
blank=True, null=True,
)
@ -176,21 +173,21 @@ class Notebook(SophonGroupModel):
"""
:return: The name given to the container associated with this :class:`Notebook`.
"""
return f"{os.environ.get('SOPHON_CONTAINER_PREFIX', 'sophon-container')}-{self.slug}"
return f"{settings.CONTAINER_PREFIX}-{self.slug}"
@property
def volume_name(self) -> str:
"""
:return: The name given to the volume associated with this :class:`Notebook`.
"""
return f"{os.environ.get('SOPHON_VOLUME_PREFIX', 'sophon-volume')}-{self.slug}"
return f"{settings.VOLUME_PREFIX}-{self.slug}"
@property
def network_name(self) -> str:
"""
:return: The name given to the network associated with this :class:`Notebook`.
"""
return f"{os.environ.get('SOPHON_NETWORK_PREFIX', 'sophon-network')}-{self.slug}"
return f"{settings.NETWORK_PREFIX}-{self.slug}"
@property
def external_domain(self) -> str:
@ -222,10 +219,10 @@ class Notebook(SophonGroupModel):
return f"{settings.PROXY_PROTOCOL}://{self.external_domain}/tree?token={self.jupyter_token}"
@property
def internal_domain(self) -> t.Optional[str]:
def local_domain(self) -> t.Optional[str]:
"""
:return: The domain name where this :class:`Notebook` is accessible on the local machine, or :data:`None` if no port has been assigned to this
container yet.
container yet or if the proxy itself is running in a container.
"""
if self.port is None:
return None
@ -307,7 +304,7 @@ class Notebook(SophonGroupModel):
self.log.debug(f"Network does not exist, creating it now...")
network = docker_client.networks.create(
name=self.network_name,
internal=not self.internet_access,
internal=True,
)
self.log.info(f"Created {network!r}")
return network
@ -316,9 +313,19 @@ class Notebook(SophonGroupModel):
"""
Disable the proxying of this :class:`Notebook` by removing its URLs from the :data:`apache_db` and its port from :attr:`.port`.
"""
if settings.PROXY_CONTAINER_NAME:
self.log.debug("Getting the notebook network...")
network = self.get_network()
self.log.debug("Unassigning port...")
self.port = None
self.log.debug("Getting proxy container...")
proxy = get_proxy_container()
self.log.debug("Disconnecting proxy container from the network...")
network.disconnect(proxy)
else:
self.log.debug("Unassigning port...")
self.port = None
self.log.debug("Removing entry from the apache_db...")
try:
@ -326,7 +333,7 @@ class Notebook(SophonGroupModel):
except KeyError:
pass
self.log.debug("Clearing port from the SQL database...")
self.log.debug("Clearing proxy data from the SQL database...")
self.save()
def enable_proxying(self) -> None:
@ -334,13 +341,30 @@ class Notebook(SophonGroupModel):
Enable the proxying of this :class:`Notebook` by adding its URLs to the :data:`apache_db` and its port to :attr:`.port`.
"""
self.log.debug("Getting free port...")
self.port: int = get_ephemeral_port()
if settings.PROXY_CONTAINER_NAME:
self.log.debug("Getting the notebook network...")
network = self.get_network()
self.log.debug("Adding entry to the apache_db...")
apache_db[self.external_domain] = f"{self.internal_domain}"
self.log.debug("Getting proxy container...")
proxy = get_proxy_container()
self.log.debug("Saving port to the SQL database...")
self.log.debug("Connecting proxy container to the network...")
network.connect(proxy)
self.log.debug("Setting internal_url...")
self.internal_url = f"http://{self.container_name}:8888"
self.log.debug("Adding entry to the apache_db...")
apache_db[bytes(self.external_domain, encoding="ascii")] = bytes(self.internal_url, encoding="ascii")
else:
self.log.debug("Getting free port...")
self.port: int = get_ephemeral_port()
self.log.debug("Adding entry to the apache_db...")
apache_db[bytes(self.external_domain, encoding="ascii")] = bytes(f"{self.local_domain}", encoding="ascii")
self.log.debug("Saving proxy data to the SQL database...")
self.save()
def get_container(self) -> t.Optional[docker.models.containers.Container]:
@ -473,6 +497,7 @@ class Notebook(SophonGroupModel):
"mode": "rw",
}
},
network=network,
)
self.log.debug("Storing container_id in the SQL database...")

34
backend/sophon/settings.py
View file

@ -73,20 +73,22 @@ except KeyError:
SECRET_KEY = secrets.token_urlsafe(24)
log.debug(f"SECRET_KEY = {'*' * len(SECRET_KEY)}")
# Set debug mode
DEBUG = __debug__
if DEBUG:
log.warning("DEBUG mode is on, run the Python interpreter with the -O option to disable.")
log.debug(f"{DEBUG = }")
# Set the hosts from which the admin page can be accessed, separated by pipes
ALLOWED_HOSTS = os.environ.get("DJANGO_ALLOWED_HOSTS", "").split("|")
ALLOWED_HOSTS = os.environ.get("DJANGO_ALLOWED_HOSTS", "").split("|") if "DJANGO_ALLOWED_HOSTS" in os.environ else []
if len(ALLOWED_HOSTS) == 0:
log.warning(f"No DJANGO_ALLOWED_HOSTS are set, the admin page may not be accessible.")
log.debug(f"{ALLOWED_HOSTS = }")
# Set the origins from which the API can be called, separated by pipes
CORS_ALLOWED_ORIGINS = os.environ.get("DJANGO_ALLOWED_ORIGINS", "").split("|")
CORS_ALLOWED_ORIGINS = os.environ.get("DJANGO_ALLOWED_ORIGINS", "").split("|") if "DJANGO_ALLOWED_ORIGINS" in os.environ else []
if len(CORS_ALLOWED_ORIGINS) == 0:
log.warning(f"No DJANGO_ALLOWED_ORIGINS are set, the API may not be usable.")
log.debug(f"{CORS_ALLOWED_ORIGINS = }")
@ -205,6 +207,34 @@ except KeyError:
PROXY_BASE_DOMAIN = "dev.sophon.steffo.eu"
log.debug(f"{PROXY_BASE_DOMAIN = }")
# Set the name of the proxy docker container
PROXY_CONTAINER_NAME = os.environ.get("DJANGO_PROXY_CONTAINER_NAME")
log.debug(f"{PROXY_CONTAINER_NAME =}")
# Set the prefix to add to all instantiated notebook container names
try:
DOCKER_CONTAINER_PREFIX = os.environ["DJANGO_DOCKER_CONTAINER_PREFIX"]
except KeyError:
log.warning("DOCKER_CONTAINER_PREFIX was not set, defaulting to `sophon-container`")
DOCKER_CONTAINER_PREFIX = "sophon-container"
log.debug(f"{DOCKER_CONTAINER_PREFIX = }")
# Set the prefix to add to all instantiated notebook volume names
try:
DOCKER_VOLUME_PREFIX = os.environ["DJANGO_DOCKER_VOLUME_PREFIX"]
except KeyError:
log.warning("DOCKER_VOLUME_PREFIX was not set, defaulting to `sophon-volume`")
DOCKER_VOLUME_PREFIX = "sophon-volume"
log.debug(f"{DOCKER_VOLUME_PREFIX = }")
# Set the prefix to add to all instantiated notebook network names
try:
DOCKER_NETWORK_PREFIX = os.environ["DJANGO_DOCKER_NETWORK_PREFIX"]
except KeyError:
log.warning("DOCKER_VOLUME_PREFIX was not set, defaulting to `sophon-network`")
DOCKER_NETWORK_PREFIX = "sophon-network"
log.debug(f"{DOCKER_NETWORK_PREFIX = }")
try:
DOCKER_HOST = os.environ["DJANGO_DOCKER_HOST"]
except KeyError: