🧹 Optimize imports

backend/sophon/core/admin.py

@@ -1,4 +1,5 @@
 from django.contrib import admin
+
 from . import models
 
 

backend/sophon/core/migrations/0001_initial.py

@@ -1,8 +1,10 @@
 # Generated by Django 3.2 on 2021-04-08 14:36
 
-from django.db import migrations, models
-import django.db.models.deletion
 import importlib.resources
+
+import django.db.models.deletion
+from django.db import migrations, models
+
 from .. import models as core_models
 
 
@@ -19,7 +21,6 @@ def create_builtin_sources(apps, schema_editor):
 
 
 class Migration(migrations.Migration):
-
     initial = True
 
     dependencies = [
@@ -29,49 +30,94 @@ class Migration(migrations.Migration):
         migrations.CreateModel(
             name='DataFlow',
             fields=[
-                ('surrogate_id', models.IntegerField(help_text='Internal id used by Django to identify this DataFlow.', primary_key=True, serialize=False, verbose_name='Surrogate id')),
+                ('surrogate_id', models.IntegerField(help_text='Internal id used by Django to identify this DataFlow.',
-                ('id', models.CharField(help_text='Internal string used in SDMX communication to identify the DataFlow.', max_length=64, verbose_name='SDMX id')),
+                                                     primary_key=True, serialize=False, verbose_name='Surrogate id')),
-                ('description', models.TextField(blank=True, help_text='Natural language description of the DataFlow.', verbose_name='Description')),
+                ('id',
+                 models.CharField(help_text='Internal string used in SDMX communication to identify the DataFlow.',
+                                  max_length=64, verbose_name='SDMX id')),
+                ('description', models.TextField(blank=True, help_text='Natural language description of the DataFlow.',
+                                                 verbose_name='Description')),
             ],
         ),
         migrations.CreateModel(
             name='DataSource',
             fields=[
-                ('id', models.CharField(help_text='Internal id used by PandaSDMX to reference the source.', max_length=16, primary_key=True, serialize=False, verbose_name='PandaSDMX id')),
+                ('id',
-                ('name', models.CharField(help_text='Full length name of the data source.', max_length=512, verbose_name='Name')),
+                 models.CharField(help_text='Internal id used by PandaSDMX to reference the source.', max_length=16,
-                ('description', models.TextField(blank=True, help_text='Long description of the data source.', verbose_name='Description')),
+                                  primary_key=True, serialize=False, verbose_name='PandaSDMX id')),
-                ('url', models.URLField(help_text='The base URL of the SDMX endpoint of the data source.', verbose_name='API URL')),
+                ('name', models.CharField(help_text='Full length name of the data source.', max_length=512,
-                ('documentation', models.URLField(help_text='Documentation URL of the data source.', null=True, verbose_name='Documentation URL')),
+                                          verbose_name='Name')),
-                ('data_content_type', models.CharField(choices=[('JSON', 'JSON'), ('XML', 'XML')], default='XML', help_text='The format in which the API returns its data.', max_length=16, verbose_name='API type')),
+                ('description', models.TextField(blank=True, help_text='Long description of the data source.',
-                ('headers', models.JSONField(default=dict, help_text='HTTP headers to attach to every request, as a JSON object.', verbose_name='HTTP Headers')),
+                                                 verbose_name='Description')),
-                ('resources', models.JSONField(default=dict, help_text='Unknown and undocumented JSON object.', verbose_name='Resources')),
+                ('url', models.URLField(help_text='The base URL of the SDMX endpoint of the data source.',
-                ('supports_agencyscheme', models.BooleanField(default=True, help_text='Whether the data source supports <a href="https://pandasdmx.readthedocs.io/en/latest/api.html#pandasdmx.model.AgencyScheme">AgencyScheme </a> or not.', verbose_name='Supports AgencyScheme')),
+                                        verbose_name='API URL')),
-                ('supports_categoryscheme', models.BooleanField(default=True, help_text='Whether the data source supports <a href="https://pandasdmx.readthedocs.io/en/latest/api.html#pandasdmx.model.CategoryScheme">CategoryScheme </a> or not.', verbose_name='Supports CategoryScheme')),
+                ('documentation', models.URLField(help_text='Documentation URL of the data source.', null=True,
-                ('supports_codelist', models.BooleanField(default=True, help_text='Whether the data source supports <a href="https://pandasdmx.readthedocs.io/en/latest/api.html#pandasdmx.model.CodeList">CodeList </a> or not.', verbose_name='Supports CodeList')),
+                                                  verbose_name='Documentation URL')),
-                ('supports_conceptscheme', models.BooleanField(default=True, help_text='Whether the data source supports <a href="https://pandasdmx.readthedocs.io/en/latest/api.html#pandasdmx.model.ConceptScheme">ConceptScheme </a> or not.', verbose_name='Supports ConceptScheme')),
+                ('data_content_type', models.CharField(choices=[('JSON', 'JSON'), ('XML', 'XML')], default='XML',
-                ('supports_data', models.BooleanField(default=True, help_text='Whether the data source supports <a href="https://pandasdmx.readthedocs.io/en/latest/api.html#pandasdmx.model.DataSet">DataSet </a> or not.', verbose_name='Supports DataSet')),
+                                                       help_text='The format in which the API returns its data.',
-                ('supports_dataflow', models.BooleanField(default=True, help_text='Whether the data source supports <a href="https://pandasdmx.readthedocs.io/en/latest/api.html#pandasdmx.model.DataflowDefinition">DataflowDefinition </a> or not.', verbose_name='Supports DataflowDefinition')),
+                                                       max_length=16, verbose_name='API type')),
-                ('supports_datastructure', models.BooleanField(default=True, help_text='Whether the data source supports <a href="https://pandasdmx.readthedocs.io/en/latest/api.html#pandasdmx.model.DataStructureDefinition">CategoryScheme </a> or not.', verbose_name='Supports DataStructureDefinition')),
+                ('headers',
-                ('supports_provisionagreement', models.BooleanField(default=True, help_text='Whether the data source supports <a href="https://pandasdmx.readthedocs.io/en/latest/api.html#pandasdmx.model.ProvisionAgreement">CategoryScheme </a> or not.', verbose_name='Supports ProvisionAgreement')),
+                 models.JSONField(default=dict, help_text='HTTP headers to attach to every request, as a JSON object.',
-                ('supports_preview', models.BooleanField(default=False, help_text='Whether the data source supports <a href="https://pandasdmx.readthedocs.io/en/latest/api.html#pandasdmx.Request.preview_data">previews of data </a> or not.', verbose_name='Supports previews')),
+                                  verbose_name='HTTP Headers')),
-                ('supports_structurespecific_data', models.BooleanField(default=False, help_text='Whether the data source returns <a href="https://pandasdmx.readthedocs.io/en/latest/api.html#pandasdmx.source.Source">structure-specific data messages </a> or not.', verbose_name='Supports structure-specific data messages')),
+                ('resources', models.JSONField(default=dict, help_text='Unknown and undocumented JSON object.',
-                ('builtin', models.BooleanField(help_text='Whether the source is built-in in PandaSDMX or not.', verbose_name='Builtin')),
+                                               verbose_name='Resources')),
-                ('last_sync', models.DateTimeField(help_text='The datetime at which the data flows of this source were last syncronized.', null=True, verbose_name='Last updated')),
+                ('supports_agencyscheme', models.BooleanField(default=True,
+                                                              help_text='Whether the data source supports <a href="https://pandasdmx.readthedocs.io/en/latest/api.html#pandasdmx.model.AgencyScheme">AgencyScheme </a> or not.',
+                                                              verbose_name='Supports AgencyScheme')),
+                ('supports_categoryscheme', models.BooleanField(default=True,
+                                                                help_text='Whether the data source supports <a href="https://pandasdmx.readthedocs.io/en/latest/api.html#pandasdmx.model.CategoryScheme">CategoryScheme </a> or not.',
+                                                                verbose_name='Supports CategoryScheme')),
+                ('supports_codelist', models.BooleanField(default=True,
+                                                          help_text='Whether the data source supports <a href="https://pandasdmx.readthedocs.io/en/latest/api.html#pandasdmx.model.CodeList">CodeList </a> or not.',
+                                                          verbose_name='Supports CodeList')),
+                ('supports_conceptscheme', models.BooleanField(default=True,
+                                                               help_text='Whether the data source supports <a href="https://pandasdmx.readthedocs.io/en/latest/api.html#pandasdmx.model.ConceptScheme">ConceptScheme </a> or not.',
+                                                               verbose_name='Supports ConceptScheme')),
+                ('supports_data', models.BooleanField(default=True,
+                                                      help_text='Whether the data source supports <a href="https://pandasdmx.readthedocs.io/en/latest/api.html#pandasdmx.model.DataSet">DataSet </a> or not.',
+                                                      verbose_name='Supports DataSet')),
+                ('supports_dataflow', models.BooleanField(default=True,
+                                                          help_text='Whether the data source supports <a href="https://pandasdmx.readthedocs.io/en/latest/api.html#pandasdmx.model.DataflowDefinition">DataflowDefinition </a> or not.',
+                                                          verbose_name='Supports DataflowDefinition')),
+                ('supports_datastructure', models.BooleanField(default=True,
+                                                               help_text='Whether the data source supports <a href="https://pandasdmx.readthedocs.io/en/latest/api.html#pandasdmx.model.DataStructureDefinition">CategoryScheme </a> or not.',
+                                                               verbose_name='Supports DataStructureDefinition')),
+                ('supports_provisionagreement', models.BooleanField(default=True,
+                                                                    help_text='Whether the data source supports <a href="https://pandasdmx.readthedocs.io/en/latest/api.html#pandasdmx.model.ProvisionAgreement">CategoryScheme </a> or not.',
+                                                                    verbose_name='Supports ProvisionAgreement')),
+                ('supports_preview', models.BooleanField(default=False,
+                                                         help_text='Whether the data source supports <a href="https://pandasdmx.readthedocs.io/en/latest/api.html#pandasdmx.Request.preview_data">previews of data </a> or not.',
+                                                         verbose_name='Supports previews')),
+                ('supports_structurespecific_data', models.BooleanField(default=False,
+                                                                        help_text='Whether the data source returns <a href="https://pandasdmx.readthedocs.io/en/latest/api.html#pandasdmx.source.Source">structure-specific data messages </a> or not.',
+                                                                        verbose_name='Supports structure-specific data messages')),
+                ('builtin', models.BooleanField(help_text='Whether the source is built-in in PandaSDMX or not.',
+                                                verbose_name='Builtin')),
+                ('last_sync', models.DateTimeField(
+                    help_text='The datetime at which the data flows of this source were last syncronized.', null=True,
+                    verbose_name='Last updated')),
             ],
         ),
         migrations.CreateModel(
             name='Project',
             fields=[
-                ('slug', models.SlugField(help_text='Unique alphanumeric string which identifies the project.', max_length=64, primary_key=True, serialize=False, verbose_name='Slug')),
+                ('slug',
-                ('name', models.CharField(help_text='The display name of the project.', max_length=512, verbose_name='Name')),
+                 models.SlugField(help_text='Unique alphanumeric string which identifies the project.', max_length=64,
-                ('description', models.TextField(blank=True, help_text='A brief description of the project, to be displayed inthe overview.', verbose_name='Description')),
+                                  primary_key=True, serialize=False, verbose_name='Slug')),
-                ('flows', models.ManyToManyField(blank=True, help_text='The DataFlows used in this project.', related_name='used_in', to='core.DataFlow')),
+                ('name',
+                 models.CharField(help_text='The display name of the project.', max_length=512, verbose_name='Name')),
+                ('description', models.TextField(blank=True,
+                                                 help_text='A brief description of the project, to be displayed inthe overview.',
+                                                 verbose_name='Description')),
+                ('flows', models.ManyToManyField(blank=True, help_text='The DataFlows used in this project.',
+                                                 related_name='used_in', to='core.DataFlow')),
             ],
         ),
         migrations.AddField(
             model_name='dataflow',
             name='datasource',
-            field=models.ForeignKey(help_text='The DataSource this object belongs to.', on_delete=django.db.models.deletion.RESTRICT, to='core.datasource'),
+            field=models.ForeignKey(help_text='The DataSource this object belongs to.',
+                                    on_delete=django.db.models.deletion.RESTRICT, to='core.datasource'),
         ),
         migrations.RunPython(create_builtin_sources)
     ]

backend/sophon/core/migrations/0001_sources.json

@@ -16,7 +16,9 @@
         "url": "https://sdw-wsrest.ecb.europa.eu/service",
         "name": "European Central Bank",
         "documentation": "https://www.ecb.europa.eu/stats/ecb_statistics/co-operation_and_standards/sdmx/html/index.en.html",
-        "supports": {"preview": true}
+        "supports": {
+            "preview": true
+        }
     },
     {
         "id": "ESTAT",
@@ -72,7 +74,9 @@
                 "Accept": "application/vnd.sdmx.genericdata+xml;version=2.1"
             }
         },
-        "supports": {"provisionagreement": false}
+        "supports": {
+            "provisionagreement": false
+        }
     },
     {
         "id": "ISTAT",
@@ -103,7 +107,10 @@
         "name": "Norges Bank (NO)",
         "documentation": "https://www.norges-bank.no/en/topics/Statistics/open-data/",
         "url": "https://data.norges-bank.no/api",
-        "supports": {"categoryscheme": false, "structure-specific data": true},
+        "supports": {
+            "categoryscheme": false,
+            "structure-specific data": true
+        },
         "headers": {
             "data": {
                 "accept": "application/vnd.sdmx.genericdata+xml;version=2.1"
@@ -130,15 +137,20 @@
         "name": "Pacific Data Hub",
         "documentation": "https://stats.pacificdata.org/?locale=en",
         "url": "https://stats-nsi-stable.pacificdata.org/rest",
-        "supports": {"preview": false, "provisionagreement": false}
+        "supports": {
+            "preview": false,
+            "provisionagreement": false
+        }
     },
-
     {
         "id": "UNSD",
         "name": "United Nations Statistics Division",
         "documentation": "https://unstats.un.org/home/",
         "url": "https://data.un.org/WS/rest",
-        "supports": {"preview": true, "provisionagreement": false}
+        "supports": {
+            "preview": true,
+            "provisionagreement": false
+        }
     },
     {
         "id": "WB",

backend/sophon/core/migrations/0002_auto_20210415_1453.py

@@ -1,12 +1,11 @@
 # Generated by Django 3.2 on 2021-04-15 14:53
 
+import django.db.models.deletion
 from django.conf import settings
 from django.db import migrations, models
-import django.db.models.deletion
 
 
 class Migration(migrations.Migration):
-
     dependencies = [
         migrations.swappable_dependency(settings.AUTH_USER_MODEL),
         ('core', '0001_initial'),
@@ -16,18 +15,23 @@ class Migration(migrations.Migration):
         migrations.AddField(
             model_name='project',
             name='collaborators',
-            field=models.ManyToManyField(blank=True, help_text='The users who can edit the project.', related_name='collaborates_in', to=settings.AUTH_USER_MODEL),
+            field=models.ManyToManyField(blank=True, help_text='The users who can edit the project.',
+                                         related_name='collaborates_in', to=settings.AUTH_USER_MODEL),
         ),
         # No projects should be in the db before this migration is run, so this should be fine
         migrations.AddField(
             model_name='project',
             name='owner',
-            field=models.ForeignKey(default=0, help_text='The user who owns the project, and has full access to it.', on_delete=django.db.models.deletion.CASCADE, to='auth.user'),
+            field=models.ForeignKey(default=0, help_text='The user who owns the project, and has full access to it.',
+                                    on_delete=django.db.models.deletion.CASCADE, to='auth.user'),
             preserve_default=False,
         ),
         migrations.AddField(
             model_name='project',
             name='visibility',
-            field=models.CharField(choices=[('PUBLIC', '🌍 Public'), ('INTERNAL', '🏭 Internal'), ('PRIVATE', '🔒 Private')], default='INTERNAL', help_text='A setting specifying who can view the project.', max_length=16, verbose_name='Visibility'),
+            field=models.CharField(
+                choices=[('PUBLIC', '🌍 Public'), ('INTERNAL', '🏭 Internal'), ('PRIVATE', '🔒 Private')],
+                default='INTERNAL', help_text='A setting specifying who can view the project.', max_length=16,
+                verbose_name='Visibility'),
         ),
     ]

backend/sophon/core/migrations/0003_rename_id_dataflow_sdmx_id.py

@@ -4,7 +4,6 @@ from django.db import migrations
 
 
 class Migration(migrations.Migration):
-
     dependencies = [
         ('core', '0002_auto_20210415_1453'),
     ]

backend/sophon/core/migrations/0004_alter_dataflow_surrogate_id.py

@@ -4,7 +4,6 @@ from django.db import migrations, models
 
 
 class Migration(migrations.Migration):
-
     dependencies = [
         ('core', '0003_rename_id_dataflow_sdmx_id'),
     ]
@@ -13,6 +12,7 @@ class Migration(migrations.Migration):
         migrations.AlterField(
             model_name='dataflow',
             name='surrogate_id',
-            field=models.BigAutoField(help_text='Internal id used by Django to identify this DataFlow.', primary_key=True, serialize=False, verbose_name='Surrogate id'),
+            field=models.BigAutoField(help_text='Internal id used by Django to identify this DataFlow.',
+                                      primary_key=True, serialize=False, verbose_name='Surrogate id'),
         ),
     ]

backend/sophon/core/models.py

@@ -1,14 +1,13 @@
-from django.db import models
+import json
-from django.core import validators
+import logging
-from django.contrib.auth.models import User
+import typing as t
-from django.utils import timezone
+
 import pandas
 import pandasdmx
 import pandasdmx.message
-import typing as t
+from django.contrib.auth.models import User
-import json
+from django.db import models
-import abc
+from django.utils import timezone
-import logging
 
 log = logging.getLogger(__name__)
 
@@ -277,7 +276,6 @@ class DataSource(models.Model):
 
         log.info(f"Syncing DataFlows of {self!r}...")
         for description, sdmx_id in zip(flows, flows.index):
-
             db_flow, _created = DataFlow.objects.update_or_create(
                 **{
                     "datasource": self,

backend/sophon/core/permissions.py

@@ -1,4 +1,5 @@
 import logging
+
 from rest_framework import permissions
 
 log = logging.getLogger(__name__)

backend/sophon/core/serializers.py

@@ -1,4 +1,5 @@
 from rest_framework import serializers
+
 from . import models
 
 

backend/sophon/core/urls.py

@@ -1,14 +1,13 @@
 from django.urls import path, include
 from rest_framework.routers import DefaultRouter
-from . import views
 
+from . import views
 
 router = DefaultRouter()
 router.register("datasources", views.DataSourceViewSet)
 router.register("dataflows", views.DataFlowViewSet)
 router.register("projects", views.ProjectViewSet)
 
-
 urlpatterns = [
     path("", include(router.urls)),
 ]

backend/sophon/core/views.py

@@ -1,8 +1,9 @@
-from rest_framework import viewsets, decorators, response, permissions, mixins, generics
-from . import models, serializers, permissions as custom_permissions
-from datetime import datetime
 from logging import getLogger
 
+from rest_framework import viewsets, decorators, response, permissions
+
+from . import models, serializers, permissions as custom_permissions
+
 log = getLogger(__name__)
 
 

backend/sophon/settings.py

@@ -10,8 +10,8 @@ For the full list of settings and their values, see
 https://docs.djangoproject.com/en/3.1/ref/settings/
 """
 
-from pathlib import Path
 import os
+from pathlib import Path
 
 # Build paths inside the project like this: BASE_DIR / 'subdir'.
 BASE_DIR = Path(__file__).resolve().parent.parent