diff --git a/backend/sophon/core/admin.py b/backend/sophon/core/admin.py
index d48f6f1..f9f8eec 100644
--- a/backend/sophon/core/admin.py
+++ b/backend/sophon/core/admin.py
@@ -1,4 +1,5 @@
from django.contrib import admin
+
from . import models
diff --git a/backend/sophon/core/migrations/0001_initial.py b/backend/sophon/core/migrations/0001_initial.py
index d36e104..bb7327a 100644
--- a/backend/sophon/core/migrations/0001_initial.py
+++ b/backend/sophon/core/migrations/0001_initial.py
@@ -1,8 +1,10 @@
# Generated by Django 3.2 on 2021-04-08 14:36
-from django.db import migrations, models
-import django.db.models.deletion
import importlib.resources
+
+import django.db.models.deletion
+from django.db import migrations, models
+
from .. import models as core_models
@@ -19,7 +21,6 @@ def create_builtin_sources(apps, schema_editor):
class Migration(migrations.Migration):
-
initial = True
dependencies = [
@@ -29,49 +30,94 @@ class Migration(migrations.Migration):
migrations.CreateModel(
name='DataFlow',
fields=[
- ('surrogate_id', models.IntegerField(help_text='Internal id used by Django to identify this DataFlow.', primary_key=True, serialize=False, verbose_name='Surrogate id')),
- ('id', models.CharField(help_text='Internal string used in SDMX communication to identify the DataFlow.', max_length=64, verbose_name='SDMX id')),
- ('description', models.TextField(blank=True, help_text='Natural language description of the DataFlow.', verbose_name='Description')),
+ ('surrogate_id', models.IntegerField(help_text='Internal id used by Django to identify this DataFlow.',
+ primary_key=True, serialize=False, verbose_name='Surrogate id')),
+ ('id',
+ models.CharField(help_text='Internal string used in SDMX communication to identify the DataFlow.',
+ max_length=64, verbose_name='SDMX id')),
+ ('description', models.TextField(blank=True, help_text='Natural language description of the DataFlow.',
+ verbose_name='Description')),
],
),
migrations.CreateModel(
name='DataSource',
fields=[
- ('id', models.CharField(help_text='Internal id used by PandaSDMX to reference the source.', max_length=16, primary_key=True, serialize=False, verbose_name='PandaSDMX id')),
- ('name', models.CharField(help_text='Full length name of the data source.', max_length=512, verbose_name='Name')),
- ('description', models.TextField(blank=True, help_text='Long description of the data source.', verbose_name='Description')),
- ('url', models.URLField(help_text='The base URL of the SDMX endpoint of the data source.', verbose_name='API URL')),
- ('documentation', models.URLField(help_text='Documentation URL of the data source.', null=True, verbose_name='Documentation URL')),
- ('data_content_type', models.CharField(choices=[('JSON', 'JSON'), ('XML', 'XML')], default='XML', help_text='The format in which the API returns its data.', max_length=16, verbose_name='API type')),
- ('headers', models.JSONField(default=dict, help_text='HTTP headers to attach to every request, as a JSON object.', verbose_name='HTTP Headers')),
- ('resources', models.JSONField(default=dict, help_text='Unknown and undocumented JSON object.', verbose_name='Resources')),
- ('supports_agencyscheme', models.BooleanField(default=True, help_text='Whether the data source supports AgencyScheme or not.', verbose_name='Supports AgencyScheme')),
- ('supports_categoryscheme', models.BooleanField(default=True, help_text='Whether the data source supports CategoryScheme or not.', verbose_name='Supports CategoryScheme')),
- ('supports_codelist', models.BooleanField(default=True, help_text='Whether the data source supports CodeList or not.', verbose_name='Supports CodeList')),
- ('supports_conceptscheme', models.BooleanField(default=True, help_text='Whether the data source supports ConceptScheme or not.', verbose_name='Supports ConceptScheme')),
- ('supports_data', models.BooleanField(default=True, help_text='Whether the data source supports DataSet or not.', verbose_name='Supports DataSet')),
- ('supports_dataflow', models.BooleanField(default=True, help_text='Whether the data source supports DataflowDefinition or not.', verbose_name='Supports DataflowDefinition')),
- ('supports_datastructure', models.BooleanField(default=True, help_text='Whether the data source supports CategoryScheme or not.', verbose_name='Supports DataStructureDefinition')),
- ('supports_provisionagreement', models.BooleanField(default=True, help_text='Whether the data source supports CategoryScheme or not.', verbose_name='Supports ProvisionAgreement')),
- ('supports_preview', models.BooleanField(default=False, help_text='Whether the data source supports previews of data or not.', verbose_name='Supports previews')),
- ('supports_structurespecific_data', models.BooleanField(default=False, help_text='Whether the data source returns structure-specific data messages or not.', verbose_name='Supports structure-specific data messages')),
- ('builtin', models.BooleanField(help_text='Whether the source is built-in in PandaSDMX or not.', verbose_name='Builtin')),
- ('last_sync', models.DateTimeField(help_text='The datetime at which the data flows of this source were last syncronized.', null=True, verbose_name='Last updated')),
+ ('id',
+ models.CharField(help_text='Internal id used by PandaSDMX to reference the source.', max_length=16,
+ primary_key=True, serialize=False, verbose_name='PandaSDMX id')),
+ ('name', models.CharField(help_text='Full length name of the data source.', max_length=512,
+ verbose_name='Name')),
+ ('description', models.TextField(blank=True, help_text='Long description of the data source.',
+ verbose_name='Description')),
+ ('url', models.URLField(help_text='The base URL of the SDMX endpoint of the data source.',
+ verbose_name='API URL')),
+ ('documentation', models.URLField(help_text='Documentation URL of the data source.', null=True,
+ verbose_name='Documentation URL')),
+ ('data_content_type', models.CharField(choices=[('JSON', 'JSON'), ('XML', 'XML')], default='XML',
+ help_text='The format in which the API returns its data.',
+ max_length=16, verbose_name='API type')),
+ ('headers',
+ models.JSONField(default=dict, help_text='HTTP headers to attach to every request, as a JSON object.',
+ verbose_name='HTTP Headers')),
+ ('resources', models.JSONField(default=dict, help_text='Unknown and undocumented JSON object.',
+ verbose_name='Resources')),
+ ('supports_agencyscheme', models.BooleanField(default=True,
+ help_text='Whether the data source supports AgencyScheme or not.',
+ verbose_name='Supports AgencyScheme')),
+ ('supports_categoryscheme', models.BooleanField(default=True,
+ help_text='Whether the data source supports CategoryScheme or not.',
+ verbose_name='Supports CategoryScheme')),
+ ('supports_codelist', models.BooleanField(default=True,
+ help_text='Whether the data source supports CodeList or not.',
+ verbose_name='Supports CodeList')),
+ ('supports_conceptscheme', models.BooleanField(default=True,
+ help_text='Whether the data source supports ConceptScheme or not.',
+ verbose_name='Supports ConceptScheme')),
+ ('supports_data', models.BooleanField(default=True,
+ help_text='Whether the data source supports DataSet or not.',
+ verbose_name='Supports DataSet')),
+ ('supports_dataflow', models.BooleanField(default=True,
+ help_text='Whether the data source supports DataflowDefinition or not.',
+ verbose_name='Supports DataflowDefinition')),
+ ('supports_datastructure', models.BooleanField(default=True,
+ help_text='Whether the data source supports CategoryScheme or not.',
+ verbose_name='Supports DataStructureDefinition')),
+ ('supports_provisionagreement', models.BooleanField(default=True,
+ help_text='Whether the data source supports CategoryScheme or not.',
+ verbose_name='Supports ProvisionAgreement')),
+ ('supports_preview', models.BooleanField(default=False,
+ help_text='Whether the data source supports previews of data or not.',
+ verbose_name='Supports previews')),
+ ('supports_structurespecific_data', models.BooleanField(default=False,
+ help_text='Whether the data source returns structure-specific data messages or not.',
+ verbose_name='Supports structure-specific data messages')),
+ ('builtin', models.BooleanField(help_text='Whether the source is built-in in PandaSDMX or not.',
+ verbose_name='Builtin')),
+ ('last_sync', models.DateTimeField(
+ help_text='The datetime at which the data flows of this source were last syncronized.', null=True,
+ verbose_name='Last updated')),
],
),
migrations.CreateModel(
name='Project',
fields=[
- ('slug', models.SlugField(help_text='Unique alphanumeric string which identifies the project.', max_length=64, primary_key=True, serialize=False, verbose_name='Slug')),
- ('name', models.CharField(help_text='The display name of the project.', max_length=512, verbose_name='Name')),
- ('description', models.TextField(blank=True, help_text='A brief description of the project, to be displayed inthe overview.', verbose_name='Description')),
- ('flows', models.ManyToManyField(blank=True, help_text='The DataFlows used in this project.', related_name='used_in', to='core.DataFlow')),
+ ('slug',
+ models.SlugField(help_text='Unique alphanumeric string which identifies the project.', max_length=64,
+ primary_key=True, serialize=False, verbose_name='Slug')),
+ ('name',
+ models.CharField(help_text='The display name of the project.', max_length=512, verbose_name='Name')),
+ ('description', models.TextField(blank=True,
+ help_text='A brief description of the project, to be displayed inthe overview.',
+ verbose_name='Description')),
+ ('flows', models.ManyToManyField(blank=True, help_text='The DataFlows used in this project.',
+ related_name='used_in', to='core.DataFlow')),
],
),
migrations.AddField(
model_name='dataflow',
name='datasource',
- field=models.ForeignKey(help_text='The DataSource this object belongs to.', on_delete=django.db.models.deletion.RESTRICT, to='core.datasource'),
+ field=models.ForeignKey(help_text='The DataSource this object belongs to.',
+ on_delete=django.db.models.deletion.RESTRICT, to='core.datasource'),
),
migrations.RunPython(create_builtin_sources)
]
diff --git a/backend/sophon/core/migrations/0001_sources.json b/backend/sophon/core/migrations/0001_sources.json
index da90d86..65d7f47 100644
--- a/backend/sophon/core/migrations/0001_sources.json
+++ b/backend/sophon/core/migrations/0001_sources.json
@@ -16,7 +16,9 @@
"url": "https://sdw-wsrest.ecb.europa.eu/service",
"name": "European Central Bank",
"documentation": "https://www.ecb.europa.eu/stats/ecb_statistics/co-operation_and_standards/sdmx/html/index.en.html",
- "supports": {"preview": true}
+ "supports": {
+ "preview": true
+ }
},
{
"id": "ESTAT",
@@ -72,7 +74,9 @@
"Accept": "application/vnd.sdmx.genericdata+xml;version=2.1"
}
},
- "supports": {"provisionagreement": false}
+ "supports": {
+ "provisionagreement": false
+ }
},
{
"id": "ISTAT",
@@ -103,7 +107,10 @@
"name": "Norges Bank (NO)",
"documentation": "https://www.norges-bank.no/en/topics/Statistics/open-data/",
"url": "https://data.norges-bank.no/api",
- "supports": {"categoryscheme": false, "structure-specific data": true},
+ "supports": {
+ "categoryscheme": false,
+ "structure-specific data": true
+ },
"headers": {
"data": {
"accept": "application/vnd.sdmx.genericdata+xml;version=2.1"
@@ -128,17 +135,22 @@
{
"id": "SPC",
"name": "Pacific Data Hub",
- "documentation":"https://stats.pacificdata.org/?locale=en",
+ "documentation": "https://stats.pacificdata.org/?locale=en",
"url": "https://stats-nsi-stable.pacificdata.org/rest",
- "supports": {"preview": false, "provisionagreement": false}
+ "supports": {
+ "preview": false,
+ "provisionagreement": false
+ }
},
-
{
"id": "UNSD",
"name": "United Nations Statistics Division",
"documentation": "https://unstats.un.org/home/",
"url": "https://data.un.org/WS/rest",
- "supports": {"preview": true, "provisionagreement": false}
+ "supports": {
+ "preview": true,
+ "provisionagreement": false
+ }
},
{
"id": "WB",
diff --git a/backend/sophon/core/migrations/0002_auto_20210415_1453.py b/backend/sophon/core/migrations/0002_auto_20210415_1453.py
index 1122a91..dd88d9f 100644
--- a/backend/sophon/core/migrations/0002_auto_20210415_1453.py
+++ b/backend/sophon/core/migrations/0002_auto_20210415_1453.py
@@ -1,12 +1,11 @@
# Generated by Django 3.2 on 2021-04-15 14:53
+import django.db.models.deletion
from django.conf import settings
from django.db import migrations, models
-import django.db.models.deletion
class Migration(migrations.Migration):
-
dependencies = [
migrations.swappable_dependency(settings.AUTH_USER_MODEL),
('core', '0001_initial'),
@@ -16,18 +15,23 @@ class Migration(migrations.Migration):
migrations.AddField(
model_name='project',
name='collaborators',
- field=models.ManyToManyField(blank=True, help_text='The users who can edit the project.', related_name='collaborates_in', to=settings.AUTH_USER_MODEL),
+ field=models.ManyToManyField(blank=True, help_text='The users who can edit the project.',
+ related_name='collaborates_in', to=settings.AUTH_USER_MODEL),
),
# No projects should be in the db before this migration is run, so this should be fine
migrations.AddField(
model_name='project',
name='owner',
- field=models.ForeignKey(default=0, help_text='The user who owns the project, and has full access to it.', on_delete=django.db.models.deletion.CASCADE, to='auth.user'),
+ field=models.ForeignKey(default=0, help_text='The user who owns the project, and has full access to it.',
+ on_delete=django.db.models.deletion.CASCADE, to='auth.user'),
preserve_default=False,
),
migrations.AddField(
model_name='project',
name='visibility',
- field=models.CharField(choices=[('PUBLIC', '🌍 Public'), ('INTERNAL', '🏭 Internal'), ('PRIVATE', '🔒 Private')], default='INTERNAL', help_text='A setting specifying who can view the project.', max_length=16, verbose_name='Visibility'),
+ field=models.CharField(
+ choices=[('PUBLIC', '🌍 Public'), ('INTERNAL', '🏭 Internal'), ('PRIVATE', '🔒 Private')],
+ default='INTERNAL', help_text='A setting specifying who can view the project.', max_length=16,
+ verbose_name='Visibility'),
),
]
diff --git a/backend/sophon/core/migrations/0003_rename_id_dataflow_sdmx_id.py b/backend/sophon/core/migrations/0003_rename_id_dataflow_sdmx_id.py
index 78e0a82..40da5be 100644
--- a/backend/sophon/core/migrations/0003_rename_id_dataflow_sdmx_id.py
+++ b/backend/sophon/core/migrations/0003_rename_id_dataflow_sdmx_id.py
@@ -4,7 +4,6 @@ from django.db import migrations
class Migration(migrations.Migration):
-
dependencies = [
('core', '0002_auto_20210415_1453'),
]
diff --git a/backend/sophon/core/migrations/0004_alter_dataflow_surrogate_id.py b/backend/sophon/core/migrations/0004_alter_dataflow_surrogate_id.py
index 1d815fe..d23238a 100644
--- a/backend/sophon/core/migrations/0004_alter_dataflow_surrogate_id.py
+++ b/backend/sophon/core/migrations/0004_alter_dataflow_surrogate_id.py
@@ -4,7 +4,6 @@ from django.db import migrations, models
class Migration(migrations.Migration):
-
dependencies = [
('core', '0003_rename_id_dataflow_sdmx_id'),
]
@@ -13,6 +12,7 @@ class Migration(migrations.Migration):
migrations.AlterField(
model_name='dataflow',
name='surrogate_id',
- field=models.BigAutoField(help_text='Internal id used by Django to identify this DataFlow.', primary_key=True, serialize=False, verbose_name='Surrogate id'),
+ field=models.BigAutoField(help_text='Internal id used by Django to identify this DataFlow.',
+ primary_key=True, serialize=False, verbose_name='Surrogate id'),
),
]
diff --git a/backend/sophon/core/models.py b/backend/sophon/core/models.py
index 33cb253..7668c47 100644
--- a/backend/sophon/core/models.py
+++ b/backend/sophon/core/models.py
@@ -1,14 +1,13 @@
-from django.db import models
-from django.core import validators
-from django.contrib.auth.models import User
-from django.utils import timezone
+import json
+import logging
+import typing as t
+
import pandas
import pandasdmx
import pandasdmx.message
-import typing as t
-import json
-import abc
-import logging
+from django.contrib.auth.models import User
+from django.db import models
+from django.utils import timezone
log = logging.getLogger(__name__)
@@ -277,7 +276,6 @@ class DataSource(models.Model):
log.info(f"Syncing DataFlows of {self!r}...")
for description, sdmx_id in zip(flows, flows.index):
-
db_flow, _created = DataFlow.objects.update_or_create(
**{
"datasource": self,
diff --git a/backend/sophon/core/permissions.py b/backend/sophon/core/permissions.py
index b4e7670..b8a4ef4 100644
--- a/backend/sophon/core/permissions.py
+++ b/backend/sophon/core/permissions.py
@@ -1,4 +1,5 @@
import logging
+
from rest_framework import permissions
log = logging.getLogger(__name__)
diff --git a/backend/sophon/core/serializers.py b/backend/sophon/core/serializers.py
index 855da5f..fb479a7 100644
--- a/backend/sophon/core/serializers.py
+++ b/backend/sophon/core/serializers.py
@@ -1,4 +1,5 @@
from rest_framework import serializers
+
from . import models
diff --git a/backend/sophon/core/urls.py b/backend/sophon/core/urls.py
index 00d9fb1..3e5b42b 100644
--- a/backend/sophon/core/urls.py
+++ b/backend/sophon/core/urls.py
@@ -1,14 +1,13 @@
from django.urls import path, include
from rest_framework.routers import DefaultRouter
-from . import views
+from . import views
router = DefaultRouter()
router.register("datasources", views.DataSourceViewSet)
router.register("dataflows", views.DataFlowViewSet)
router.register("projects", views.ProjectViewSet)
-
urlpatterns = [
path("", include(router.urls)),
]
diff --git a/backend/sophon/core/views.py b/backend/sophon/core/views.py
index 6b0eaa7..fa0b868 100644
--- a/backend/sophon/core/views.py
+++ b/backend/sophon/core/views.py
@@ -1,8 +1,9 @@
-from rest_framework import viewsets, decorators, response, permissions, mixins, generics
-from . import models, serializers, permissions as custom_permissions
-from datetime import datetime
from logging import getLogger
+from rest_framework import viewsets, decorators, response, permissions
+
+from . import models, serializers, permissions as custom_permissions
+
log = getLogger(__name__)
diff --git a/backend/sophon/settings.py b/backend/sophon/settings.py
index 2d47890..959c95f 100644
--- a/backend/sophon/settings.py
+++ b/backend/sophon/settings.py
@@ -10,8 +10,8 @@ For the full list of settings and their values, see
https://docs.djangoproject.com/en/3.1/ref/settings/
"""
-from pathlib import Path
import os
+from pathlib import Path
# Build paths inside the project like this: BASE_DIR / 'subdir'.
BASE_DIR = Path(__file__).resolve().parent.parent