mirror of
https://github.com/Steffo99/sophon.git
synced 2024-12-22 14:54:22 +00:00
🧹 Optimize imports
This commit is contained in:
parent
06b2f7adff
commit
2b8fe4ee65
12 changed files with 123 additions and 61 deletions
|
|
@ -1,4 +1,5 @@
|
|||
from django.contrib import admin
|
||||
|
||||
from . import models
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -1,8 +1,10 @@
|
|||
# Generated by Django 3.2 on 2021-04-08 14:36
|
||||
|
||||
from django.db import migrations, models
|
||||
import django.db.models.deletion
|
||||
import importlib.resources
|
||||
|
||||
import django.db.models.deletion
|
||||
from django.db import migrations, models
|
||||
|
||||
from .. import models as core_models
|
||||
|
||||
|
||||
|
|
@ -19,7 +21,6 @@ def create_builtin_sources(apps, schema_editor):
|
|||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
initial = True
|
||||
|
||||
dependencies = [
|
||||
|
|
@ -29,49 +30,94 @@ class Migration(migrations.Migration):
|
|||
migrations.CreateModel(
|
||||
name='DataFlow',
|
||||
fields=[
|
||||
('surrogate_id', models.IntegerField(help_text='Internal id used by Django to identify this DataFlow.', primary_key=True, serialize=False, verbose_name='Surrogate id')),
|
||||
('id', models.CharField(help_text='Internal string used in SDMX communication to identify the DataFlow.', max_length=64, verbose_name='SDMX id')),
|
||||
('description', models.TextField(blank=True, help_text='Natural language description of the DataFlow.', verbose_name='Description')),
|
||||
('surrogate_id', models.IntegerField(help_text='Internal id used by Django to identify this DataFlow.',
|
||||
primary_key=True, serialize=False, verbose_name='Surrogate id')),
|
||||
('id',
|
||||
models.CharField(help_text='Internal string used in SDMX communication to identify the DataFlow.',
|
||||
max_length=64, verbose_name='SDMX id')),
|
||||
('description', models.TextField(blank=True, help_text='Natural language description of the DataFlow.',
|
||||
verbose_name='Description')),
|
||||
],
|
||||
),
|
||||
migrations.CreateModel(
|
||||
name='DataSource',
|
||||
fields=[
|
||||
('id', models.CharField(help_text='Internal id used by PandaSDMX to reference the source.', max_length=16, primary_key=True, serialize=False, verbose_name='PandaSDMX id')),
|
||||
('name', models.CharField(help_text='Full length name of the data source.', max_length=512, verbose_name='Name')),
|
||||
('description', models.TextField(blank=True, help_text='Long description of the data source.', verbose_name='Description')),
|
||||
('url', models.URLField(help_text='The base URL of the SDMX endpoint of the data source.', verbose_name='API URL')),
|
||||
('documentation', models.URLField(help_text='Documentation URL of the data source.', null=True, verbose_name='Documentation URL')),
|
||||
('data_content_type', models.CharField(choices=[('JSON', 'JSON'), ('XML', 'XML')], default='XML', help_text='The format in which the API returns its data.', max_length=16, verbose_name='API type')),
|
||||
('headers', models.JSONField(default=dict, help_text='HTTP headers to attach to every request, as a JSON object.', verbose_name='HTTP Headers')),
|
||||
('resources', models.JSONField(default=dict, help_text='Unknown and undocumented JSON object.', verbose_name='Resources')),
|
||||
('supports_agencyscheme', models.BooleanField(default=True, help_text='Whether the data source supports <a href="https://pandasdmx.readthedocs.io/en/latest/api.html#pandasdmx.model.AgencyScheme">AgencyScheme </a> or not.', verbose_name='Supports AgencyScheme')),
|
||||
('supports_categoryscheme', models.BooleanField(default=True, help_text='Whether the data source supports <a href="https://pandasdmx.readthedocs.io/en/latest/api.html#pandasdmx.model.CategoryScheme">CategoryScheme </a> or not.', verbose_name='Supports CategoryScheme')),
|
||||
('supports_codelist', models.BooleanField(default=True, help_text='Whether the data source supports <a href="https://pandasdmx.readthedocs.io/en/latest/api.html#pandasdmx.model.CodeList">CodeList </a> or not.', verbose_name='Supports CodeList')),
|
||||
('supports_conceptscheme', models.BooleanField(default=True, help_text='Whether the data source supports <a href="https://pandasdmx.readthedocs.io/en/latest/api.html#pandasdmx.model.ConceptScheme">ConceptScheme </a> or not.', verbose_name='Supports ConceptScheme')),
|
||||
('supports_data', models.BooleanField(default=True, help_text='Whether the data source supports <a href="https://pandasdmx.readthedocs.io/en/latest/api.html#pandasdmx.model.DataSet">DataSet </a> or not.', verbose_name='Supports DataSet')),
|
||||
('supports_dataflow', models.BooleanField(default=True, help_text='Whether the data source supports <a href="https://pandasdmx.readthedocs.io/en/latest/api.html#pandasdmx.model.DataflowDefinition">DataflowDefinition </a> or not.', verbose_name='Supports DataflowDefinition')),
|
||||
('supports_datastructure', models.BooleanField(default=True, help_text='Whether the data source supports <a href="https://pandasdmx.readthedocs.io/en/latest/api.html#pandasdmx.model.DataStructureDefinition">CategoryScheme </a> or not.', verbose_name='Supports DataStructureDefinition')),
|
||||
('supports_provisionagreement', models.BooleanField(default=True, help_text='Whether the data source supports <a href="https://pandasdmx.readthedocs.io/en/latest/api.html#pandasdmx.model.ProvisionAgreement">CategoryScheme </a> or not.', verbose_name='Supports ProvisionAgreement')),
|
||||
('supports_preview', models.BooleanField(default=False, help_text='Whether the data source supports <a href="https://pandasdmx.readthedocs.io/en/latest/api.html#pandasdmx.Request.preview_data">previews of data </a> or not.', verbose_name='Supports previews')),
|
||||
('supports_structurespecific_data', models.BooleanField(default=False, help_text='Whether the data source returns <a href="https://pandasdmx.readthedocs.io/en/latest/api.html#pandasdmx.source.Source">structure-specific data messages </a> or not.', verbose_name='Supports structure-specific data messages')),
|
||||
('builtin', models.BooleanField(help_text='Whether the source is built-in in PandaSDMX or not.', verbose_name='Builtin')),
|
||||
('last_sync', models.DateTimeField(help_text='The datetime at which the data flows of this source were last syncronized.', null=True, verbose_name='Last updated')),
|
||||
('id',
|
||||
models.CharField(help_text='Internal id used by PandaSDMX to reference the source.', max_length=16,
|
||||
primary_key=True, serialize=False, verbose_name='PandaSDMX id')),
|
||||
('name', models.CharField(help_text='Full length name of the data source.', max_length=512,
|
||||
verbose_name='Name')),
|
||||
('description', models.TextField(blank=True, help_text='Long description of the data source.',
|
||||
verbose_name='Description')),
|
||||
('url', models.URLField(help_text='The base URL of the SDMX endpoint of the data source.',
|
||||
verbose_name='API URL')),
|
||||
('documentation', models.URLField(help_text='Documentation URL of the data source.', null=True,
|
||||
verbose_name='Documentation URL')),
|
||||
('data_content_type', models.CharField(choices=[('JSON', 'JSON'), ('XML', 'XML')], default='XML',
|
||||
help_text='The format in which the API returns its data.',
|
||||
max_length=16, verbose_name='API type')),
|
||||
('headers',
|
||||
models.JSONField(default=dict, help_text='HTTP headers to attach to every request, as a JSON object.',
|
||||
verbose_name='HTTP Headers')),
|
||||
('resources', models.JSONField(default=dict, help_text='Unknown and undocumented JSON object.',
|
||||
verbose_name='Resources')),
|
||||
('supports_agencyscheme', models.BooleanField(default=True,
|
||||
help_text='Whether the data source supports <a href="https://pandasdmx.readthedocs.io/en/latest/api.html#pandasdmx.model.AgencyScheme">AgencyScheme </a> or not.',
|
||||
verbose_name='Supports AgencyScheme')),
|
||||
('supports_categoryscheme', models.BooleanField(default=True,
|
||||
help_text='Whether the data source supports <a href="https://pandasdmx.readthedocs.io/en/latest/api.html#pandasdmx.model.CategoryScheme">CategoryScheme </a> or not.',
|
||||
verbose_name='Supports CategoryScheme')),
|
||||
('supports_codelist', models.BooleanField(default=True,
|
||||
help_text='Whether the data source supports <a href="https://pandasdmx.readthedocs.io/en/latest/api.html#pandasdmx.model.CodeList">CodeList </a> or not.',
|
||||
verbose_name='Supports CodeList')),
|
||||
('supports_conceptscheme', models.BooleanField(default=True,
|
||||
help_text='Whether the data source supports <a href="https://pandasdmx.readthedocs.io/en/latest/api.html#pandasdmx.model.ConceptScheme">ConceptScheme </a> or not.',
|
||||
verbose_name='Supports ConceptScheme')),
|
||||
('supports_data', models.BooleanField(default=True,
|
||||
help_text='Whether the data source supports <a href="https://pandasdmx.readthedocs.io/en/latest/api.html#pandasdmx.model.DataSet">DataSet </a> or not.',
|
||||
verbose_name='Supports DataSet')),
|
||||
('supports_dataflow', models.BooleanField(default=True,
|
||||
help_text='Whether the data source supports <a href="https://pandasdmx.readthedocs.io/en/latest/api.html#pandasdmx.model.DataflowDefinition">DataflowDefinition </a> or not.',
|
||||
verbose_name='Supports DataflowDefinition')),
|
||||
('supports_datastructure', models.BooleanField(default=True,
|
||||
help_text='Whether the data source supports <a href="https://pandasdmx.readthedocs.io/en/latest/api.html#pandasdmx.model.DataStructureDefinition">CategoryScheme </a> or not.',
|
||||
verbose_name='Supports DataStructureDefinition')),
|
||||
('supports_provisionagreement', models.BooleanField(default=True,
|
||||
help_text='Whether the data source supports <a href="https://pandasdmx.readthedocs.io/en/latest/api.html#pandasdmx.model.ProvisionAgreement">CategoryScheme </a> or not.',
|
||||
verbose_name='Supports ProvisionAgreement')),
|
||||
('supports_preview', models.BooleanField(default=False,
|
||||
help_text='Whether the data source supports <a href="https://pandasdmx.readthedocs.io/en/latest/api.html#pandasdmx.Request.preview_data">previews of data </a> or not.',
|
||||
verbose_name='Supports previews')),
|
||||
('supports_structurespecific_data', models.BooleanField(default=False,
|
||||
help_text='Whether the data source returns <a href="https://pandasdmx.readthedocs.io/en/latest/api.html#pandasdmx.source.Source">structure-specific data messages </a> or not.',
|
||||
verbose_name='Supports structure-specific data messages')),
|
||||
('builtin', models.BooleanField(help_text='Whether the source is built-in in PandaSDMX or not.',
|
||||
verbose_name='Builtin')),
|
||||
('last_sync', models.DateTimeField(
|
||||
help_text='The datetime at which the data flows of this source were last syncronized.', null=True,
|
||||
verbose_name='Last updated')),
|
||||
],
|
||||
),
|
||||
migrations.CreateModel(
|
||||
name='Project',
|
||||
fields=[
|
||||
('slug', models.SlugField(help_text='Unique alphanumeric string which identifies the project.', max_length=64, primary_key=True, serialize=False, verbose_name='Slug')),
|
||||
('name', models.CharField(help_text='The display name of the project.', max_length=512, verbose_name='Name')),
|
||||
('description', models.TextField(blank=True, help_text='A brief description of the project, to be displayed inthe overview.', verbose_name='Description')),
|
||||
('flows', models.ManyToManyField(blank=True, help_text='The DataFlows used in this project.', related_name='used_in', to='core.DataFlow')),
|
||||
('slug',
|
||||
models.SlugField(help_text='Unique alphanumeric string which identifies the project.', max_length=64,
|
||||
primary_key=True, serialize=False, verbose_name='Slug')),
|
||||
('name',
|
||||
models.CharField(help_text='The display name of the project.', max_length=512, verbose_name='Name')),
|
||||
('description', models.TextField(blank=True,
|
||||
help_text='A brief description of the project, to be displayed inthe overview.',
|
||||
verbose_name='Description')),
|
||||
('flows', models.ManyToManyField(blank=True, help_text='The DataFlows used in this project.',
|
||||
related_name='used_in', to='core.DataFlow')),
|
||||
],
|
||||
),
|
||||
migrations.AddField(
|
||||
model_name='dataflow',
|
||||
name='datasource',
|
||||
field=models.ForeignKey(help_text='The DataSource this object belongs to.', on_delete=django.db.models.deletion.RESTRICT, to='core.datasource'),
|
||||
field=models.ForeignKey(help_text='The DataSource this object belongs to.',
|
||||
on_delete=django.db.models.deletion.RESTRICT, to='core.datasource'),
|
||||
),
|
||||
migrations.RunPython(create_builtin_sources)
|
||||
]
|
||||
|
|
|
|||
|
|
@ -16,7 +16,9 @@
|
|||
"url": "https://sdw-wsrest.ecb.europa.eu/service",
|
||||
"name": "European Central Bank",
|
||||
"documentation": "https://www.ecb.europa.eu/stats/ecb_statistics/co-operation_and_standards/sdmx/html/index.en.html",
|
||||
"supports": {"preview": true}
|
||||
"supports": {
|
||||
"preview": true
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "ESTAT",
|
||||
|
|
@ -72,7 +74,9 @@
|
|||
"Accept": "application/vnd.sdmx.genericdata+xml;version=2.1"
|
||||
}
|
||||
},
|
||||
"supports": {"provisionagreement": false}
|
||||
"supports": {
|
||||
"provisionagreement": false
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "ISTAT",
|
||||
|
|
@ -103,7 +107,10 @@
|
|||
"name": "Norges Bank (NO)",
|
||||
"documentation": "https://www.norges-bank.no/en/topics/Statistics/open-data/",
|
||||
"url": "https://data.norges-bank.no/api",
|
||||
"supports": {"categoryscheme": false, "structure-specific data": true},
|
||||
"supports": {
|
||||
"categoryscheme": false,
|
||||
"structure-specific data": true
|
||||
},
|
||||
"headers": {
|
||||
"data": {
|
||||
"accept": "application/vnd.sdmx.genericdata+xml;version=2.1"
|
||||
|
|
@ -128,17 +135,22 @@
|
|||
{
|
||||
"id": "SPC",
|
||||
"name": "Pacific Data Hub",
|
||||
"documentation":"https://stats.pacificdata.org/?locale=en",
|
||||
"documentation": "https://stats.pacificdata.org/?locale=en",
|
||||
"url": "https://stats-nsi-stable.pacificdata.org/rest",
|
||||
"supports": {"preview": false, "provisionagreement": false}
|
||||
"supports": {
|
||||
"preview": false,
|
||||
"provisionagreement": false
|
||||
}
|
||||
},
|
||||
|
||||
{
|
||||
"id": "UNSD",
|
||||
"name": "United Nations Statistics Division",
|
||||
"documentation": "https://unstats.un.org/home/",
|
||||
"url": "https://data.un.org/WS/rest",
|
||||
"supports": {"preview": true, "provisionagreement": false}
|
||||
"supports": {
|
||||
"preview": true,
|
||||
"provisionagreement": false
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "WB",
|
||||
|
|
|
|||
|
|
@ -1,12 +1,11 @@
|
|||
# Generated by Django 3.2 on 2021-04-15 14:53
|
||||
|
||||
import django.db.models.deletion
|
||||
from django.conf import settings
|
||||
from django.db import migrations, models
|
||||
import django.db.models.deletion
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
dependencies = [
|
||||
migrations.swappable_dependency(settings.AUTH_USER_MODEL),
|
||||
('core', '0001_initial'),
|
||||
|
|
@ -16,18 +15,23 @@ class Migration(migrations.Migration):
|
|||
migrations.AddField(
|
||||
model_name='project',
|
||||
name='collaborators',
|
||||
field=models.ManyToManyField(blank=True, help_text='The users who can edit the project.', related_name='collaborates_in', to=settings.AUTH_USER_MODEL),
|
||||
field=models.ManyToManyField(blank=True, help_text='The users who can edit the project.',
|
||||
related_name='collaborates_in', to=settings.AUTH_USER_MODEL),
|
||||
),
|
||||
# No projects should be in the db before this migration is run, so this should be fine
|
||||
migrations.AddField(
|
||||
model_name='project',
|
||||
name='owner',
|
||||
field=models.ForeignKey(default=0, help_text='The user who owns the project, and has full access to it.', on_delete=django.db.models.deletion.CASCADE, to='auth.user'),
|
||||
field=models.ForeignKey(default=0, help_text='The user who owns the project, and has full access to it.',
|
||||
on_delete=django.db.models.deletion.CASCADE, to='auth.user'),
|
||||
preserve_default=False,
|
||||
),
|
||||
migrations.AddField(
|
||||
model_name='project',
|
||||
name='visibility',
|
||||
field=models.CharField(choices=[('PUBLIC', '🌍 Public'), ('INTERNAL', '🏭 Internal'), ('PRIVATE', '🔒 Private')], default='INTERNAL', help_text='A setting specifying who can view the project.', max_length=16, verbose_name='Visibility'),
|
||||
field=models.CharField(
|
||||
choices=[('PUBLIC', '🌍 Public'), ('INTERNAL', '🏭 Internal'), ('PRIVATE', '🔒 Private')],
|
||||
default='INTERNAL', help_text='A setting specifying who can view the project.', max_length=16,
|
||||
verbose_name='Visibility'),
|
||||
),
|
||||
]
|
||||
|
|
|
|||
|
|
@ -4,7 +4,6 @@ from django.db import migrations
|
|||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
dependencies = [
|
||||
('core', '0002_auto_20210415_1453'),
|
||||
]
|
||||
|
|
|
|||
|
|
@ -4,7 +4,6 @@ from django.db import migrations, models
|
|||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
dependencies = [
|
||||
('core', '0003_rename_id_dataflow_sdmx_id'),
|
||||
]
|
||||
|
|
@ -13,6 +12,7 @@ class Migration(migrations.Migration):
|
|||
migrations.AlterField(
|
||||
model_name='dataflow',
|
||||
name='surrogate_id',
|
||||
field=models.BigAutoField(help_text='Internal id used by Django to identify this DataFlow.', primary_key=True, serialize=False, verbose_name='Surrogate id'),
|
||||
field=models.BigAutoField(help_text='Internal id used by Django to identify this DataFlow.',
|
||||
primary_key=True, serialize=False, verbose_name='Surrogate id'),
|
||||
),
|
||||
]
|
||||
|
|
|
|||
|
|
@ -1,14 +1,13 @@
|
|||
from django.db import models
|
||||
from django.core import validators
|
||||
from django.contrib.auth.models import User
|
||||
from django.utils import timezone
|
||||
import json
|
||||
import logging
|
||||
import typing as t
|
||||
|
||||
import pandas
|
||||
import pandasdmx
|
||||
import pandasdmx.message
|
||||
import typing as t
|
||||
import json
|
||||
import abc
|
||||
import logging
|
||||
from django.contrib.auth.models import User
|
||||
from django.db import models
|
||||
from django.utils import timezone
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
|
@ -277,7 +276,6 @@ class DataSource(models.Model):
|
|||
|
||||
log.info(f"Syncing DataFlows of {self!r}...")
|
||||
for description, sdmx_id in zip(flows, flows.index):
|
||||
|
||||
db_flow, _created = DataFlow.objects.update_or_create(
|
||||
**{
|
||||
"datasource": self,
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
import logging
|
||||
|
||||
from rest_framework import permissions
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
from rest_framework import serializers
|
||||
|
||||
from . import models
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -1,14 +1,13 @@
|
|||
from django.urls import path, include
|
||||
from rest_framework.routers import DefaultRouter
|
||||
from . import views
|
||||
|
||||
from . import views
|
||||
|
||||
router = DefaultRouter()
|
||||
router.register("datasources", views.DataSourceViewSet)
|
||||
router.register("dataflows", views.DataFlowViewSet)
|
||||
router.register("projects", views.ProjectViewSet)
|
||||
|
||||
|
||||
urlpatterns = [
|
||||
path("", include(router.urls)),
|
||||
]
|
||||
|
|
|
|||
|
|
@ -1,8 +1,9 @@
|
|||
from rest_framework import viewsets, decorators, response, permissions, mixins, generics
|
||||
from . import models, serializers, permissions as custom_permissions
|
||||
from datetime import datetime
|
||||
from logging import getLogger
|
||||
|
||||
from rest_framework import viewsets, decorators, response, permissions
|
||||
|
||||
from . import models, serializers, permissions as custom_permissions
|
||||
|
||||
log = getLogger(__name__)
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -10,8 +10,8 @@ For the full list of settings and their values, see
|
|||
https://docs.djangoproject.com/en/3.1/ref/settings/
|
||||
"""
|
||||
|
||||
from pathlib import Path
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
# Build paths inside the project like this: BASE_DIR / 'subdir'.
|
||||
BASE_DIR = Path(__file__).resolve().parent.parent
|
||||
|
|
|
|||
Loading…
Reference in a new issue