1
Fork 0
mirror of https://github.com/Steffo99/sophon.git synced 2024-12-22 14:54:22 +00:00

🧹 Optimize imports

This commit is contained in:
Steffo 2021-04-18 20:05:07 +02:00
parent 06b2f7adff
commit 2b8fe4ee65
12 changed files with 123 additions and 61 deletions

View file

@ -1,4 +1,5 @@
from django.contrib import admin from django.contrib import admin
from . import models from . import models

View file

@ -1,8 +1,10 @@
# Generated by Django 3.2 on 2021-04-08 14:36 # Generated by Django 3.2 on 2021-04-08 14:36
from django.db import migrations, models
import django.db.models.deletion
import importlib.resources import importlib.resources
import django.db.models.deletion
from django.db import migrations, models
from .. import models as core_models from .. import models as core_models
@ -19,7 +21,6 @@ def create_builtin_sources(apps, schema_editor):
class Migration(migrations.Migration): class Migration(migrations.Migration):
initial = True initial = True
dependencies = [ dependencies = [
@ -29,49 +30,94 @@ class Migration(migrations.Migration):
migrations.CreateModel( migrations.CreateModel(
name='DataFlow', name='DataFlow',
fields=[ fields=[
('surrogate_id', models.IntegerField(help_text='Internal id used by Django to identify this DataFlow.', primary_key=True, serialize=False, verbose_name='Surrogate id')), ('surrogate_id', models.IntegerField(help_text='Internal id used by Django to identify this DataFlow.',
('id', models.CharField(help_text='Internal string used in SDMX communication to identify the DataFlow.', max_length=64, verbose_name='SDMX id')), primary_key=True, serialize=False, verbose_name='Surrogate id')),
('description', models.TextField(blank=True, help_text='Natural language description of the DataFlow.', verbose_name='Description')), ('id',
models.CharField(help_text='Internal string used in SDMX communication to identify the DataFlow.',
max_length=64, verbose_name='SDMX id')),
('description', models.TextField(blank=True, help_text='Natural language description of the DataFlow.',
verbose_name='Description')),
], ],
), ),
migrations.CreateModel( migrations.CreateModel(
name='DataSource', name='DataSource',
fields=[ fields=[
('id', models.CharField(help_text='Internal id used by PandaSDMX to reference the source.', max_length=16, primary_key=True, serialize=False, verbose_name='PandaSDMX id')), ('id',
('name', models.CharField(help_text='Full length name of the data source.', max_length=512, verbose_name='Name')), models.CharField(help_text='Internal id used by PandaSDMX to reference the source.', max_length=16,
('description', models.TextField(blank=True, help_text='Long description of the data source.', verbose_name='Description')), primary_key=True, serialize=False, verbose_name='PandaSDMX id')),
('url', models.URLField(help_text='The base URL of the SDMX endpoint of the data source.', verbose_name='API URL')), ('name', models.CharField(help_text='Full length name of the data source.', max_length=512,
('documentation', models.URLField(help_text='Documentation URL of the data source.', null=True, verbose_name='Documentation URL')), verbose_name='Name')),
('data_content_type', models.CharField(choices=[('JSON', 'JSON'), ('XML', 'XML')], default='XML', help_text='The format in which the API returns its data.', max_length=16, verbose_name='API type')), ('description', models.TextField(blank=True, help_text='Long description of the data source.',
('headers', models.JSONField(default=dict, help_text='HTTP headers to attach to every request, as a JSON object.', verbose_name='HTTP Headers')), verbose_name='Description')),
('resources', models.JSONField(default=dict, help_text='Unknown and undocumented JSON object.', verbose_name='Resources')), ('url', models.URLField(help_text='The base URL of the SDMX endpoint of the data source.',
('supports_agencyscheme', models.BooleanField(default=True, help_text='Whether the data source supports <a href="https://pandasdmx.readthedocs.io/en/latest/api.html#pandasdmx.model.AgencyScheme">AgencyScheme </a> or not.', verbose_name='Supports AgencyScheme')), verbose_name='API URL')),
('supports_categoryscheme', models.BooleanField(default=True, help_text='Whether the data source supports <a href="https://pandasdmx.readthedocs.io/en/latest/api.html#pandasdmx.model.CategoryScheme">CategoryScheme </a> or not.', verbose_name='Supports CategoryScheme')), ('documentation', models.URLField(help_text='Documentation URL of the data source.', null=True,
('supports_codelist', models.BooleanField(default=True, help_text='Whether the data source supports <a href="https://pandasdmx.readthedocs.io/en/latest/api.html#pandasdmx.model.CodeList">CodeList </a> or not.', verbose_name='Supports CodeList')), verbose_name='Documentation URL')),
('supports_conceptscheme', models.BooleanField(default=True, help_text='Whether the data source supports <a href="https://pandasdmx.readthedocs.io/en/latest/api.html#pandasdmx.model.ConceptScheme">ConceptScheme </a> or not.', verbose_name='Supports ConceptScheme')), ('data_content_type', models.CharField(choices=[('JSON', 'JSON'), ('XML', 'XML')], default='XML',
('supports_data', models.BooleanField(default=True, help_text='Whether the data source supports <a href="https://pandasdmx.readthedocs.io/en/latest/api.html#pandasdmx.model.DataSet">DataSet </a> or not.', verbose_name='Supports DataSet')), help_text='The format in which the API returns its data.',
('supports_dataflow', models.BooleanField(default=True, help_text='Whether the data source supports <a href="https://pandasdmx.readthedocs.io/en/latest/api.html#pandasdmx.model.DataflowDefinition">DataflowDefinition </a> or not.', verbose_name='Supports DataflowDefinition')), max_length=16, verbose_name='API type')),
('supports_datastructure', models.BooleanField(default=True, help_text='Whether the data source supports <a href="https://pandasdmx.readthedocs.io/en/latest/api.html#pandasdmx.model.DataStructureDefinition">CategoryScheme </a> or not.', verbose_name='Supports DataStructureDefinition')), ('headers',
('supports_provisionagreement', models.BooleanField(default=True, help_text='Whether the data source supports <a href="https://pandasdmx.readthedocs.io/en/latest/api.html#pandasdmx.model.ProvisionAgreement">CategoryScheme </a> or not.', verbose_name='Supports ProvisionAgreement')), models.JSONField(default=dict, help_text='HTTP headers to attach to every request, as a JSON object.',
('supports_preview', models.BooleanField(default=False, help_text='Whether the data source supports <a href="https://pandasdmx.readthedocs.io/en/latest/api.html#pandasdmx.Request.preview_data">previews of data </a> or not.', verbose_name='Supports previews')), verbose_name='HTTP Headers')),
('supports_structurespecific_data', models.BooleanField(default=False, help_text='Whether the data source returns <a href="https://pandasdmx.readthedocs.io/en/latest/api.html#pandasdmx.source.Source">structure-specific data messages </a> or not.', verbose_name='Supports structure-specific data messages')), ('resources', models.JSONField(default=dict, help_text='Unknown and undocumented JSON object.',
('builtin', models.BooleanField(help_text='Whether the source is built-in in PandaSDMX or not.', verbose_name='Builtin')), verbose_name='Resources')),
('last_sync', models.DateTimeField(help_text='The datetime at which the data flows of this source were last syncronized.', null=True, verbose_name='Last updated')), ('supports_agencyscheme', models.BooleanField(default=True,
help_text='Whether the data source supports <a href="https://pandasdmx.readthedocs.io/en/latest/api.html#pandasdmx.model.AgencyScheme">AgencyScheme </a> or not.',
verbose_name='Supports AgencyScheme')),
('supports_categoryscheme', models.BooleanField(default=True,
help_text='Whether the data source supports <a href="https://pandasdmx.readthedocs.io/en/latest/api.html#pandasdmx.model.CategoryScheme">CategoryScheme </a> or not.',
verbose_name='Supports CategoryScheme')),
('supports_codelist', models.BooleanField(default=True,
help_text='Whether the data source supports <a href="https://pandasdmx.readthedocs.io/en/latest/api.html#pandasdmx.model.CodeList">CodeList </a> or not.',
verbose_name='Supports CodeList')),
('supports_conceptscheme', models.BooleanField(default=True,
help_text='Whether the data source supports <a href="https://pandasdmx.readthedocs.io/en/latest/api.html#pandasdmx.model.ConceptScheme">ConceptScheme </a> or not.',
verbose_name='Supports ConceptScheme')),
('supports_data', models.BooleanField(default=True,
help_text='Whether the data source supports <a href="https://pandasdmx.readthedocs.io/en/latest/api.html#pandasdmx.model.DataSet">DataSet </a> or not.',
verbose_name='Supports DataSet')),
('supports_dataflow', models.BooleanField(default=True,
help_text='Whether the data source supports <a href="https://pandasdmx.readthedocs.io/en/latest/api.html#pandasdmx.model.DataflowDefinition">DataflowDefinition </a> or not.',
verbose_name='Supports DataflowDefinition')),
('supports_datastructure', models.BooleanField(default=True,
help_text='Whether the data source supports <a href="https://pandasdmx.readthedocs.io/en/latest/api.html#pandasdmx.model.DataStructureDefinition">CategoryScheme </a> or not.',
verbose_name='Supports DataStructureDefinition')),
('supports_provisionagreement', models.BooleanField(default=True,
help_text='Whether the data source supports <a href="https://pandasdmx.readthedocs.io/en/latest/api.html#pandasdmx.model.ProvisionAgreement">CategoryScheme </a> or not.',
verbose_name='Supports ProvisionAgreement')),
('supports_preview', models.BooleanField(default=False,
help_text='Whether the data source supports <a href="https://pandasdmx.readthedocs.io/en/latest/api.html#pandasdmx.Request.preview_data">previews of data </a> or not.',
verbose_name='Supports previews')),
('supports_structurespecific_data', models.BooleanField(default=False,
help_text='Whether the data source returns <a href="https://pandasdmx.readthedocs.io/en/latest/api.html#pandasdmx.source.Source">structure-specific data messages </a> or not.',
verbose_name='Supports structure-specific data messages')),
('builtin', models.BooleanField(help_text='Whether the source is built-in in PandaSDMX or not.',
verbose_name='Builtin')),
('last_sync', models.DateTimeField(
help_text='The datetime at which the data flows of this source were last syncronized.', null=True,
verbose_name='Last updated')),
], ],
), ),
migrations.CreateModel( migrations.CreateModel(
name='Project', name='Project',
fields=[ fields=[
('slug', models.SlugField(help_text='Unique alphanumeric string which identifies the project.', max_length=64, primary_key=True, serialize=False, verbose_name='Slug')), ('slug',
('name', models.CharField(help_text='The display name of the project.', max_length=512, verbose_name='Name')), models.SlugField(help_text='Unique alphanumeric string which identifies the project.', max_length=64,
('description', models.TextField(blank=True, help_text='A brief description of the project, to be displayed inthe overview.', verbose_name='Description')), primary_key=True, serialize=False, verbose_name='Slug')),
('flows', models.ManyToManyField(blank=True, help_text='The DataFlows used in this project.', related_name='used_in', to='core.DataFlow')), ('name',
models.CharField(help_text='The display name of the project.', max_length=512, verbose_name='Name')),
('description', models.TextField(blank=True,
help_text='A brief description of the project, to be displayed inthe overview.',
verbose_name='Description')),
('flows', models.ManyToManyField(blank=True, help_text='The DataFlows used in this project.',
related_name='used_in', to='core.DataFlow')),
], ],
), ),
migrations.AddField( migrations.AddField(
model_name='dataflow', model_name='dataflow',
name='datasource', name='datasource',
field=models.ForeignKey(help_text='The DataSource this object belongs to.', on_delete=django.db.models.deletion.RESTRICT, to='core.datasource'), field=models.ForeignKey(help_text='The DataSource this object belongs to.',
on_delete=django.db.models.deletion.RESTRICT, to='core.datasource'),
), ),
migrations.RunPython(create_builtin_sources) migrations.RunPython(create_builtin_sources)
] ]

View file

@ -16,7 +16,9 @@
"url": "https://sdw-wsrest.ecb.europa.eu/service", "url": "https://sdw-wsrest.ecb.europa.eu/service",
"name": "European Central Bank", "name": "European Central Bank",
"documentation": "https://www.ecb.europa.eu/stats/ecb_statistics/co-operation_and_standards/sdmx/html/index.en.html", "documentation": "https://www.ecb.europa.eu/stats/ecb_statistics/co-operation_and_standards/sdmx/html/index.en.html",
"supports": {"preview": true} "supports": {
"preview": true
}
}, },
{ {
"id": "ESTAT", "id": "ESTAT",
@ -72,7 +74,9 @@
"Accept": "application/vnd.sdmx.genericdata+xml;version=2.1" "Accept": "application/vnd.sdmx.genericdata+xml;version=2.1"
} }
}, },
"supports": {"provisionagreement": false} "supports": {
"provisionagreement": false
}
}, },
{ {
"id": "ISTAT", "id": "ISTAT",
@ -103,7 +107,10 @@
"name": "Norges Bank (NO)", "name": "Norges Bank (NO)",
"documentation": "https://www.norges-bank.no/en/topics/Statistics/open-data/", "documentation": "https://www.norges-bank.no/en/topics/Statistics/open-data/",
"url": "https://data.norges-bank.no/api", "url": "https://data.norges-bank.no/api",
"supports": {"categoryscheme": false, "structure-specific data": true}, "supports": {
"categoryscheme": false,
"structure-specific data": true
},
"headers": { "headers": {
"data": { "data": {
"accept": "application/vnd.sdmx.genericdata+xml;version=2.1" "accept": "application/vnd.sdmx.genericdata+xml;version=2.1"
@ -128,17 +135,22 @@
{ {
"id": "SPC", "id": "SPC",
"name": "Pacific Data Hub", "name": "Pacific Data Hub",
"documentation":"https://stats.pacificdata.org/?locale=en", "documentation": "https://stats.pacificdata.org/?locale=en",
"url": "https://stats-nsi-stable.pacificdata.org/rest", "url": "https://stats-nsi-stable.pacificdata.org/rest",
"supports": {"preview": false, "provisionagreement": false} "supports": {
"preview": false,
"provisionagreement": false
}
}, },
{ {
"id": "UNSD", "id": "UNSD",
"name": "United Nations Statistics Division", "name": "United Nations Statistics Division",
"documentation": "https://unstats.un.org/home/", "documentation": "https://unstats.un.org/home/",
"url": "https://data.un.org/WS/rest", "url": "https://data.un.org/WS/rest",
"supports": {"preview": true, "provisionagreement": false} "supports": {
"preview": true,
"provisionagreement": false
}
}, },
{ {
"id": "WB", "id": "WB",

View file

@ -1,12 +1,11 @@
# Generated by Django 3.2 on 2021-04-15 14:53 # Generated by Django 3.2 on 2021-04-15 14:53
import django.db.models.deletion
from django.conf import settings from django.conf import settings
from django.db import migrations, models from django.db import migrations, models
import django.db.models.deletion
class Migration(migrations.Migration): class Migration(migrations.Migration):
dependencies = [ dependencies = [
migrations.swappable_dependency(settings.AUTH_USER_MODEL), migrations.swappable_dependency(settings.AUTH_USER_MODEL),
('core', '0001_initial'), ('core', '0001_initial'),
@ -16,18 +15,23 @@ class Migration(migrations.Migration):
migrations.AddField( migrations.AddField(
model_name='project', model_name='project',
name='collaborators', name='collaborators',
field=models.ManyToManyField(blank=True, help_text='The users who can edit the project.', related_name='collaborates_in', to=settings.AUTH_USER_MODEL), field=models.ManyToManyField(blank=True, help_text='The users who can edit the project.',
related_name='collaborates_in', to=settings.AUTH_USER_MODEL),
), ),
# No projects should be in the db before this migration is run, so this should be fine # No projects should be in the db before this migration is run, so this should be fine
migrations.AddField( migrations.AddField(
model_name='project', model_name='project',
name='owner', name='owner',
field=models.ForeignKey(default=0, help_text='The user who owns the project, and has full access to it.', on_delete=django.db.models.deletion.CASCADE, to='auth.user'), field=models.ForeignKey(default=0, help_text='The user who owns the project, and has full access to it.',
on_delete=django.db.models.deletion.CASCADE, to='auth.user'),
preserve_default=False, preserve_default=False,
), ),
migrations.AddField( migrations.AddField(
model_name='project', model_name='project',
name='visibility', name='visibility',
field=models.CharField(choices=[('PUBLIC', '🌍 Public'), ('INTERNAL', '🏭 Internal'), ('PRIVATE', '🔒 Private')], default='INTERNAL', help_text='A setting specifying who can view the project.', max_length=16, verbose_name='Visibility'), field=models.CharField(
choices=[('PUBLIC', '🌍 Public'), ('INTERNAL', '🏭 Internal'), ('PRIVATE', '🔒 Private')],
default='INTERNAL', help_text='A setting specifying who can view the project.', max_length=16,
verbose_name='Visibility'),
), ),
] ]

View file

@ -4,7 +4,6 @@ from django.db import migrations
class Migration(migrations.Migration): class Migration(migrations.Migration):
dependencies = [ dependencies = [
('core', '0002_auto_20210415_1453'), ('core', '0002_auto_20210415_1453'),
] ]

View file

@ -4,7 +4,6 @@ from django.db import migrations, models
class Migration(migrations.Migration): class Migration(migrations.Migration):
dependencies = [ dependencies = [
('core', '0003_rename_id_dataflow_sdmx_id'), ('core', '0003_rename_id_dataflow_sdmx_id'),
] ]
@ -13,6 +12,7 @@ class Migration(migrations.Migration):
migrations.AlterField( migrations.AlterField(
model_name='dataflow', model_name='dataflow',
name='surrogate_id', name='surrogate_id',
field=models.BigAutoField(help_text='Internal id used by Django to identify this DataFlow.', primary_key=True, serialize=False, verbose_name='Surrogate id'), field=models.BigAutoField(help_text='Internal id used by Django to identify this DataFlow.',
primary_key=True, serialize=False, verbose_name='Surrogate id'),
), ),
] ]

View file

@ -1,14 +1,13 @@
from django.db import models import json
from django.core import validators import logging
from django.contrib.auth.models import User import typing as t
from django.utils import timezone
import pandas import pandas
import pandasdmx import pandasdmx
import pandasdmx.message import pandasdmx.message
import typing as t from django.contrib.auth.models import User
import json from django.db import models
import abc from django.utils import timezone
import logging
log = logging.getLogger(__name__) log = logging.getLogger(__name__)
@ -277,7 +276,6 @@ class DataSource(models.Model):
log.info(f"Syncing DataFlows of {self!r}...") log.info(f"Syncing DataFlows of {self!r}...")
for description, sdmx_id in zip(flows, flows.index): for description, sdmx_id in zip(flows, flows.index):
db_flow, _created = DataFlow.objects.update_or_create( db_flow, _created = DataFlow.objects.update_or_create(
**{ **{
"datasource": self, "datasource": self,

View file

@ -1,4 +1,5 @@
import logging import logging
from rest_framework import permissions from rest_framework import permissions
log = logging.getLogger(__name__) log = logging.getLogger(__name__)

View file

@ -1,4 +1,5 @@
from rest_framework import serializers from rest_framework import serializers
from . import models from . import models

View file

@ -1,14 +1,13 @@
from django.urls import path, include from django.urls import path, include
from rest_framework.routers import DefaultRouter from rest_framework.routers import DefaultRouter
from . import views
from . import views
router = DefaultRouter() router = DefaultRouter()
router.register("datasources", views.DataSourceViewSet) router.register("datasources", views.DataSourceViewSet)
router.register("dataflows", views.DataFlowViewSet) router.register("dataflows", views.DataFlowViewSet)
router.register("projects", views.ProjectViewSet) router.register("projects", views.ProjectViewSet)
urlpatterns = [ urlpatterns = [
path("", include(router.urls)), path("", include(router.urls)),
] ]

View file

@ -1,8 +1,9 @@
from rest_framework import viewsets, decorators, response, permissions, mixins, generics
from . import models, serializers, permissions as custom_permissions
from datetime import datetime
from logging import getLogger from logging import getLogger
from rest_framework import viewsets, decorators, response, permissions
from . import models, serializers, permissions as custom_permissions
log = getLogger(__name__) log = getLogger(__name__)

View file

@ -10,8 +10,8 @@ For the full list of settings and their values, see
https://docs.djangoproject.com/en/3.1/ref/settings/ https://docs.djangoproject.com/en/3.1/ref/settings/
""" """
from pathlib import Path
import os import os
from pathlib import Path
# Build paths inside the project like this: BASE_DIR / 'subdir'. # Build paths inside the project like this: BASE_DIR / 'subdir'.
BASE_DIR = Path(__file__).resolve().parent.parent BASE_DIR = Path(__file__).resolve().parent.parent