From f673bc0818c087fda0523f57c9c7e991e3e31051 Mon Sep 17 00:00:00 2001 From: Stefano Pigozzi Date: Sun, 18 Apr 2021 20:05:07 +0200 Subject: [PATCH] =?UTF-8?q?=F0=9F=A7=B9=20Optimize=20imports?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/sophon/core/admin.py | 1 + .../sophon/core/migrations/0001_initial.py | 108 +++++++++++++----- .../sophon/core/migrations/0001_sources.json | 26 +++-- .../migrations/0002_auto_20210415_1453.py | 14 ++- .../0003_rename_id_dataflow_sdmx_id.py | 1 - .../0004_alter_dataflow_surrogate_id.py | 4 +- backend/sophon/core/models.py | 16 ++- backend/sophon/core/permissions.py | 1 + backend/sophon/core/serializers.py | 1 + backend/sophon/core/urls.py | 3 +- backend/sophon/core/views.py | 7 +- backend/sophon/settings.py | 2 +- 12 files changed, 123 insertions(+), 61 deletions(-) diff --git a/backend/sophon/core/admin.py b/backend/sophon/core/admin.py index d48f6f1..f9f8eec 100644 --- a/backend/sophon/core/admin.py +++ b/backend/sophon/core/admin.py @@ -1,4 +1,5 @@ from django.contrib import admin + from . import models diff --git a/backend/sophon/core/migrations/0001_initial.py b/backend/sophon/core/migrations/0001_initial.py index d36e104..bb7327a 100644 --- a/backend/sophon/core/migrations/0001_initial.py +++ b/backend/sophon/core/migrations/0001_initial.py @@ -1,8 +1,10 @@ # Generated by Django 3.2 on 2021-04-08 14:36 -from django.db import migrations, models -import django.db.models.deletion import importlib.resources + +import django.db.models.deletion +from django.db import migrations, models + from .. import models as core_models @@ -19,7 +21,6 @@ def create_builtin_sources(apps, schema_editor): class Migration(migrations.Migration): - initial = True dependencies = [ @@ -29,49 +30,94 @@ class Migration(migrations.Migration): migrations.CreateModel( name='DataFlow', fields=[ - ('surrogate_id', models.IntegerField(help_text='Internal id used by Django to identify this DataFlow.', primary_key=True, serialize=False, verbose_name='Surrogate id')), - ('id', models.CharField(help_text='Internal string used in SDMX communication to identify the DataFlow.', max_length=64, verbose_name='SDMX id')), - ('description', models.TextField(blank=True, help_text='Natural language description of the DataFlow.', verbose_name='Description')), + ('surrogate_id', models.IntegerField(help_text='Internal id used by Django to identify this DataFlow.', + primary_key=True, serialize=False, verbose_name='Surrogate id')), + ('id', + models.CharField(help_text='Internal string used in SDMX communication to identify the DataFlow.', + max_length=64, verbose_name='SDMX id')), + ('description', models.TextField(blank=True, help_text='Natural language description of the DataFlow.', + verbose_name='Description')), ], ), migrations.CreateModel( name='DataSource', fields=[ - ('id', models.CharField(help_text='Internal id used by PandaSDMX to reference the source.', max_length=16, primary_key=True, serialize=False, verbose_name='PandaSDMX id')), - ('name', models.CharField(help_text='Full length name of the data source.', max_length=512, verbose_name='Name')), - ('description', models.TextField(blank=True, help_text='Long description of the data source.', verbose_name='Description')), - ('url', models.URLField(help_text='The base URL of the SDMX endpoint of the data source.', verbose_name='API URL')), - ('documentation', models.URLField(help_text='Documentation URL of the data source.', null=True, verbose_name='Documentation URL')), - ('data_content_type', models.CharField(choices=[('JSON', 'JSON'), ('XML', 'XML')], default='XML', help_text='The format in which the API returns its data.', max_length=16, verbose_name='API type')), - ('headers', models.JSONField(default=dict, help_text='HTTP headers to attach to every request, as a JSON object.', verbose_name='HTTP Headers')), - ('resources', models.JSONField(default=dict, help_text='Unknown and undocumented JSON object.', verbose_name='Resources')), - ('supports_agencyscheme', models.BooleanField(default=True, help_text='Whether the data source supports AgencyScheme or not.', verbose_name='Supports AgencyScheme')), - ('supports_categoryscheme', models.BooleanField(default=True, help_text='Whether the data source supports CategoryScheme or not.', verbose_name='Supports CategoryScheme')), - ('supports_codelist', models.BooleanField(default=True, help_text='Whether the data source supports CodeList or not.', verbose_name='Supports CodeList')), - ('supports_conceptscheme', models.BooleanField(default=True, help_text='Whether the data source supports ConceptScheme or not.', verbose_name='Supports ConceptScheme')), - ('supports_data', models.BooleanField(default=True, help_text='Whether the data source supports DataSet or not.', verbose_name='Supports DataSet')), - ('supports_dataflow', models.BooleanField(default=True, help_text='Whether the data source supports DataflowDefinition or not.', verbose_name='Supports DataflowDefinition')), - ('supports_datastructure', models.BooleanField(default=True, help_text='Whether the data source supports CategoryScheme or not.', verbose_name='Supports DataStructureDefinition')), - ('supports_provisionagreement', models.BooleanField(default=True, help_text='Whether the data source supports CategoryScheme or not.', verbose_name='Supports ProvisionAgreement')), - ('supports_preview', models.BooleanField(default=False, help_text='Whether the data source supports previews of data or not.', verbose_name='Supports previews')), - ('supports_structurespecific_data', models.BooleanField(default=False, help_text='Whether the data source returns structure-specific data messages or not.', verbose_name='Supports structure-specific data messages')), - ('builtin', models.BooleanField(help_text='Whether the source is built-in in PandaSDMX or not.', verbose_name='Builtin')), - ('last_sync', models.DateTimeField(help_text='The datetime at which the data flows of this source were last syncronized.', null=True, verbose_name='Last updated')), + ('id', + models.CharField(help_text='Internal id used by PandaSDMX to reference the source.', max_length=16, + primary_key=True, serialize=False, verbose_name='PandaSDMX id')), + ('name', models.CharField(help_text='Full length name of the data source.', max_length=512, + verbose_name='Name')), + ('description', models.TextField(blank=True, help_text='Long description of the data source.', + verbose_name='Description')), + ('url', models.URLField(help_text='The base URL of the SDMX endpoint of the data source.', + verbose_name='API URL')), + ('documentation', models.URLField(help_text='Documentation URL of the data source.', null=True, + verbose_name='Documentation URL')), + ('data_content_type', models.CharField(choices=[('JSON', 'JSON'), ('XML', 'XML')], default='XML', + help_text='The format in which the API returns its data.', + max_length=16, verbose_name='API type')), + ('headers', + models.JSONField(default=dict, help_text='HTTP headers to attach to every request, as a JSON object.', + verbose_name='HTTP Headers')), + ('resources', models.JSONField(default=dict, help_text='Unknown and undocumented JSON object.', + verbose_name='Resources')), + ('supports_agencyscheme', models.BooleanField(default=True, + help_text='Whether the data source supports AgencyScheme or not.', + verbose_name='Supports AgencyScheme')), + ('supports_categoryscheme', models.BooleanField(default=True, + help_text='Whether the data source supports CategoryScheme or not.', + verbose_name='Supports CategoryScheme')), + ('supports_codelist', models.BooleanField(default=True, + help_text='Whether the data source supports CodeList or not.', + verbose_name='Supports CodeList')), + ('supports_conceptscheme', models.BooleanField(default=True, + help_text='Whether the data source supports ConceptScheme or not.', + verbose_name='Supports ConceptScheme')), + ('supports_data', models.BooleanField(default=True, + help_text='Whether the data source supports DataSet or not.', + verbose_name='Supports DataSet')), + ('supports_dataflow', models.BooleanField(default=True, + help_text='Whether the data source supports DataflowDefinition or not.', + verbose_name='Supports DataflowDefinition')), + ('supports_datastructure', models.BooleanField(default=True, + help_text='Whether the data source supports CategoryScheme or not.', + verbose_name='Supports DataStructureDefinition')), + ('supports_provisionagreement', models.BooleanField(default=True, + help_text='Whether the data source supports CategoryScheme or not.', + verbose_name='Supports ProvisionAgreement')), + ('supports_preview', models.BooleanField(default=False, + help_text='Whether the data source supports previews of data or not.', + verbose_name='Supports previews')), + ('supports_structurespecific_data', models.BooleanField(default=False, + help_text='Whether the data source returns structure-specific data messages or not.', + verbose_name='Supports structure-specific data messages')), + ('builtin', models.BooleanField(help_text='Whether the source is built-in in PandaSDMX or not.', + verbose_name='Builtin')), + ('last_sync', models.DateTimeField( + help_text='The datetime at which the data flows of this source were last syncronized.', null=True, + verbose_name='Last updated')), ], ), migrations.CreateModel( name='Project', fields=[ - ('slug', models.SlugField(help_text='Unique alphanumeric string which identifies the project.', max_length=64, primary_key=True, serialize=False, verbose_name='Slug')), - ('name', models.CharField(help_text='The display name of the project.', max_length=512, verbose_name='Name')), - ('description', models.TextField(blank=True, help_text='A brief description of the project, to be displayed inthe overview.', verbose_name='Description')), - ('flows', models.ManyToManyField(blank=True, help_text='The DataFlows used in this project.', related_name='used_in', to='core.DataFlow')), + ('slug', + models.SlugField(help_text='Unique alphanumeric string which identifies the project.', max_length=64, + primary_key=True, serialize=False, verbose_name='Slug')), + ('name', + models.CharField(help_text='The display name of the project.', max_length=512, verbose_name='Name')), + ('description', models.TextField(blank=True, + help_text='A brief description of the project, to be displayed inthe overview.', + verbose_name='Description')), + ('flows', models.ManyToManyField(blank=True, help_text='The DataFlows used in this project.', + related_name='used_in', to='core.DataFlow')), ], ), migrations.AddField( model_name='dataflow', name='datasource', - field=models.ForeignKey(help_text='The DataSource this object belongs to.', on_delete=django.db.models.deletion.RESTRICT, to='core.datasource'), + field=models.ForeignKey(help_text='The DataSource this object belongs to.', + on_delete=django.db.models.deletion.RESTRICT, to='core.datasource'), ), migrations.RunPython(create_builtin_sources) ] diff --git a/backend/sophon/core/migrations/0001_sources.json b/backend/sophon/core/migrations/0001_sources.json index da90d86..65d7f47 100644 --- a/backend/sophon/core/migrations/0001_sources.json +++ b/backend/sophon/core/migrations/0001_sources.json @@ -16,7 +16,9 @@ "url": "https://sdw-wsrest.ecb.europa.eu/service", "name": "European Central Bank", "documentation": "https://www.ecb.europa.eu/stats/ecb_statistics/co-operation_and_standards/sdmx/html/index.en.html", - "supports": {"preview": true} + "supports": { + "preview": true + } }, { "id": "ESTAT", @@ -72,7 +74,9 @@ "Accept": "application/vnd.sdmx.genericdata+xml;version=2.1" } }, - "supports": {"provisionagreement": false} + "supports": { + "provisionagreement": false + } }, { "id": "ISTAT", @@ -103,7 +107,10 @@ "name": "Norges Bank (NO)", "documentation": "https://www.norges-bank.no/en/topics/Statistics/open-data/", "url": "https://data.norges-bank.no/api", - "supports": {"categoryscheme": false, "structure-specific data": true}, + "supports": { + "categoryscheme": false, + "structure-specific data": true + }, "headers": { "data": { "accept": "application/vnd.sdmx.genericdata+xml;version=2.1" @@ -128,17 +135,22 @@ { "id": "SPC", "name": "Pacific Data Hub", - "documentation":"https://stats.pacificdata.org/?locale=en", + "documentation": "https://stats.pacificdata.org/?locale=en", "url": "https://stats-nsi-stable.pacificdata.org/rest", - "supports": {"preview": false, "provisionagreement": false} + "supports": { + "preview": false, + "provisionagreement": false + } }, - { "id": "UNSD", "name": "United Nations Statistics Division", "documentation": "https://unstats.un.org/home/", "url": "https://data.un.org/WS/rest", - "supports": {"preview": true, "provisionagreement": false} + "supports": { + "preview": true, + "provisionagreement": false + } }, { "id": "WB", diff --git a/backend/sophon/core/migrations/0002_auto_20210415_1453.py b/backend/sophon/core/migrations/0002_auto_20210415_1453.py index 1122a91..dd88d9f 100644 --- a/backend/sophon/core/migrations/0002_auto_20210415_1453.py +++ b/backend/sophon/core/migrations/0002_auto_20210415_1453.py @@ -1,12 +1,11 @@ # Generated by Django 3.2 on 2021-04-15 14:53 +import django.db.models.deletion from django.conf import settings from django.db import migrations, models -import django.db.models.deletion class Migration(migrations.Migration): - dependencies = [ migrations.swappable_dependency(settings.AUTH_USER_MODEL), ('core', '0001_initial'), @@ -16,18 +15,23 @@ class Migration(migrations.Migration): migrations.AddField( model_name='project', name='collaborators', - field=models.ManyToManyField(blank=True, help_text='The users who can edit the project.', related_name='collaborates_in', to=settings.AUTH_USER_MODEL), + field=models.ManyToManyField(blank=True, help_text='The users who can edit the project.', + related_name='collaborates_in', to=settings.AUTH_USER_MODEL), ), # No projects should be in the db before this migration is run, so this should be fine migrations.AddField( model_name='project', name='owner', - field=models.ForeignKey(default=0, help_text='The user who owns the project, and has full access to it.', on_delete=django.db.models.deletion.CASCADE, to='auth.user'), + field=models.ForeignKey(default=0, help_text='The user who owns the project, and has full access to it.', + on_delete=django.db.models.deletion.CASCADE, to='auth.user'), preserve_default=False, ), migrations.AddField( model_name='project', name='visibility', - field=models.CharField(choices=[('PUBLIC', '🌍 Public'), ('INTERNAL', '🏭 Internal'), ('PRIVATE', '🔒 Private')], default='INTERNAL', help_text='A setting specifying who can view the project.', max_length=16, verbose_name='Visibility'), + field=models.CharField( + choices=[('PUBLIC', '🌍 Public'), ('INTERNAL', '🏭 Internal'), ('PRIVATE', '🔒 Private')], + default='INTERNAL', help_text='A setting specifying who can view the project.', max_length=16, + verbose_name='Visibility'), ), ] diff --git a/backend/sophon/core/migrations/0003_rename_id_dataflow_sdmx_id.py b/backend/sophon/core/migrations/0003_rename_id_dataflow_sdmx_id.py index 78e0a82..40da5be 100644 --- a/backend/sophon/core/migrations/0003_rename_id_dataflow_sdmx_id.py +++ b/backend/sophon/core/migrations/0003_rename_id_dataflow_sdmx_id.py @@ -4,7 +4,6 @@ from django.db import migrations class Migration(migrations.Migration): - dependencies = [ ('core', '0002_auto_20210415_1453'), ] diff --git a/backend/sophon/core/migrations/0004_alter_dataflow_surrogate_id.py b/backend/sophon/core/migrations/0004_alter_dataflow_surrogate_id.py index 1d815fe..d23238a 100644 --- a/backend/sophon/core/migrations/0004_alter_dataflow_surrogate_id.py +++ b/backend/sophon/core/migrations/0004_alter_dataflow_surrogate_id.py @@ -4,7 +4,6 @@ from django.db import migrations, models class Migration(migrations.Migration): - dependencies = [ ('core', '0003_rename_id_dataflow_sdmx_id'), ] @@ -13,6 +12,7 @@ class Migration(migrations.Migration): migrations.AlterField( model_name='dataflow', name='surrogate_id', - field=models.BigAutoField(help_text='Internal id used by Django to identify this DataFlow.', primary_key=True, serialize=False, verbose_name='Surrogate id'), + field=models.BigAutoField(help_text='Internal id used by Django to identify this DataFlow.', + primary_key=True, serialize=False, verbose_name='Surrogate id'), ), ] diff --git a/backend/sophon/core/models.py b/backend/sophon/core/models.py index 33cb253..7668c47 100644 --- a/backend/sophon/core/models.py +++ b/backend/sophon/core/models.py @@ -1,14 +1,13 @@ -from django.db import models -from django.core import validators -from django.contrib.auth.models import User -from django.utils import timezone +import json +import logging +import typing as t + import pandas import pandasdmx import pandasdmx.message -import typing as t -import json -import abc -import logging +from django.contrib.auth.models import User +from django.db import models +from django.utils import timezone log = logging.getLogger(__name__) @@ -277,7 +276,6 @@ class DataSource(models.Model): log.info(f"Syncing DataFlows of {self!r}...") for description, sdmx_id in zip(flows, flows.index): - db_flow, _created = DataFlow.objects.update_or_create( **{ "datasource": self, diff --git a/backend/sophon/core/permissions.py b/backend/sophon/core/permissions.py index b4e7670..b8a4ef4 100644 --- a/backend/sophon/core/permissions.py +++ b/backend/sophon/core/permissions.py @@ -1,4 +1,5 @@ import logging + from rest_framework import permissions log = logging.getLogger(__name__) diff --git a/backend/sophon/core/serializers.py b/backend/sophon/core/serializers.py index 855da5f..fb479a7 100644 --- a/backend/sophon/core/serializers.py +++ b/backend/sophon/core/serializers.py @@ -1,4 +1,5 @@ from rest_framework import serializers + from . import models diff --git a/backend/sophon/core/urls.py b/backend/sophon/core/urls.py index 00d9fb1..3e5b42b 100644 --- a/backend/sophon/core/urls.py +++ b/backend/sophon/core/urls.py @@ -1,14 +1,13 @@ from django.urls import path, include from rest_framework.routers import DefaultRouter -from . import views +from . import views router = DefaultRouter() router.register("datasources", views.DataSourceViewSet) router.register("dataflows", views.DataFlowViewSet) router.register("projects", views.ProjectViewSet) - urlpatterns = [ path("", include(router.urls)), ] diff --git a/backend/sophon/core/views.py b/backend/sophon/core/views.py index 6b0eaa7..fa0b868 100644 --- a/backend/sophon/core/views.py +++ b/backend/sophon/core/views.py @@ -1,8 +1,9 @@ -from rest_framework import viewsets, decorators, response, permissions, mixins, generics -from . import models, serializers, permissions as custom_permissions -from datetime import datetime from logging import getLogger +from rest_framework import viewsets, decorators, response, permissions + +from . import models, serializers, permissions as custom_permissions + log = getLogger(__name__) diff --git a/backend/sophon/settings.py b/backend/sophon/settings.py index 2d47890..959c95f 100644 --- a/backend/sophon/settings.py +++ b/backend/sophon/settings.py @@ -10,8 +10,8 @@ For the full list of settings and their values, see https://docs.djangoproject.com/en/3.1/ref/settings/ """ -from pathlib import Path import os +from pathlib import Path # Build paths inside the project like this: BASE_DIR / 'subdir'. BASE_DIR = Path(__file__).resolve().parent.parent