steffo/sophon
steffo/sophon
1
Fork 0
mirror of https://github.com/Steffo99/sophon.git synced 2024-12-22 23:04:21 +00:00
Code Activity

🧹 Optimize imports

Browse source
This commit is contained in:
Steffo 2021-04-18 20:05:07 +02:00
parent 31b1a6e8c2
commit f673bc0818
Signed by: steffo
GPG key ID: 6965406171929D01
12 changed files with 123 additions and 61 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
backend/sophon/core/admin.py
View file

@ -1,4 +1,5 @@
from django.contrib import admin
from . import models

108
backend/sophon/core/migrations/0001_initial.py
View file

@ -1,8 +1,10 @@
# Generated by Django 3.2 on 2021-04-08 14:36
from django.db import migrations, models
import django.db.models.deletion
import importlib.resources
import django.db.models.deletion
from django.db import migrations, models
from .. import models as core_models
@ -19,7 +21,6 @@ def create_builtin_sources(apps, schema_editor):
class Migration(migrations.Migration):
initial = True
dependencies = [
@ -29,49 +30,94 @@ class Migration(migrations.Migration):
migrations.CreateModel(
name='DataFlow',
fields=[
('surrogate_id', models.IntegerField(help_text='Internal id used by Django to identify this DataFlow.', primary_key=True, serialize=False, verbose_name='Surrogate id')),
('id', models.CharField(help_text='Internal string used in SDMX communication to identify the DataFlow.', max_length=64, verbose_name='SDMX id')),
('description', models.TextField(blank=True, help_text='Natural language description of the DataFlow.', verbose_name='Description')),
('surrogate_id', models.IntegerField(help_text='Internal id used by Django to identify this DataFlow.',
primary_key=True, serialize=False, verbose_name='Surrogate id')),
('id',
models.CharField(help_text='Internal string used in SDMX communication to identify the DataFlow.',
max_length=64, verbose_name='SDMX id')),
('description', models.TextField(blank=True, help_text='Natural language description of the DataFlow.',
verbose_name='Description')),
],
),
migrations.CreateModel(
name='DataSource',
fields=[
('id', models.CharField(help_text='Internal id used by PandaSDMX to reference the source.', max_length=16, primary_key=True, serialize=False, verbose_name='PandaSDMX id')),
('name', models.CharField(help_text='Full length name of the data source.', max_length=512, verbose_name='Name')),
('description', models.TextField(blank=True, help_text='Long description of the data source.', verbose_name='Description')),
('url', models.URLField(help_text='The base URL of the SDMX endpoint of the data source.', verbose_name='API URL')),
('documentation', models.URLField(help_text='Documentation URL of the data source.', null=True, verbose_name='Documentation URL')),
('data_content_type', models.CharField(choices=[('JSON', 'JSON'), ('XML', 'XML')], default='XML', help_text='The format in which the API returns its data.', max_length=16, verbose_name='API type')),
('headers', models.JSONField(default=dict, help_text='HTTP headers to attach to every request, as a JSON object.', verbose_name='HTTP Headers')),
('resources', models.JSONField(default=dict, help_text='Unknown and undocumented JSON object.', verbose_name='Resources')),
('supports_agencyscheme', models.BooleanField(default=True, help_text='Whether the data source supports <a href="https://pandasdmx.readthedocs.io/en/latest/api.html#pandasdmx.model.AgencyScheme">AgencyScheme </a> or not.', verbose_name='Supports AgencyScheme')),
('supports_categoryscheme', models.BooleanField(default=True, help_text='Whether the data source supports <a href="https://pandasdmx.readthedocs.io/en/latest/api.html#pandasdmx.model.CategoryScheme">CategoryScheme </a> or not.', verbose_name='Supports CategoryScheme')),
('supports_codelist', models.BooleanField(default=True, help_text='Whether the data source supports <a href="https://pandasdmx.readthedocs.io/en/latest/api.html#pandasdmx.model.CodeList">CodeList </a> or not.', verbose_name='Supports CodeList')),
('supports_conceptscheme', models.BooleanField(default=True, help_text='Whether the data source supports <a href="https://pandasdmx.readthedocs.io/en/latest/api.html#pandasdmx.model.ConceptScheme">ConceptScheme </a> or not.', verbose_name='Supports ConceptScheme')),
('supports_data', models.BooleanField(default=True, help_text='Whether the data source supports <a href="https://pandasdmx.readthedocs.io/en/latest/api.html#pandasdmx.model.DataSet">DataSet </a> or not.', verbose_name='Supports DataSet')),
('supports_dataflow', models.BooleanField(default=True, help_text='Whether the data source supports <a href="https://pandasdmx.readthedocs.io/en/latest/api.html#pandasdmx.model.DataflowDefinition">DataflowDefinition </a> or not.', verbose_name='Supports DataflowDefinition')),
('supports_datastructure', models.BooleanField(default=True, help_text='Whether the data source supports <a href="https://pandasdmx.readthedocs.io/en/latest/api.html#pandasdmx.model.DataStructureDefinition">CategoryScheme </a> or not.', verbose_name='Supports DataStructureDefinition')),
('supports_provisionagreement', models.BooleanField(default=True, help_text='Whether the data source supports <a href="https://pandasdmx.readthedocs.io/en/latest/api.html#pandasdmx.model.ProvisionAgreement">CategoryScheme </a> or not.', verbose_name='Supports ProvisionAgreement')),
('supports_preview', models.BooleanField(default=False, help_text='Whether the data source supports <a href="https://pandasdmx.readthedocs.io/en/latest/api.html#pandasdmx.Request.preview_data">previews of data </a> or not.', verbose_name='Supports previews')),
('supports_structurespecific_data', models.BooleanField(default=False, help_text='Whether the data source returns <a href="https://pandasdmx.readthedocs.io/en/latest/api.html#pandasdmx.source.Source">structure-specific data messages </a> or not.', verbose_name='Supports structure-specific data messages')),
('builtin', models.BooleanField(help_text='Whether the source is built-in in PandaSDMX or not.', verbose_name='Builtin')),
('last_sync', models.DateTimeField(help_text='The datetime at which the data flows of this source were last syncronized.', null=True, verbose_name='Last updated')),
('id',
models.CharField(help_text='Internal id used by PandaSDMX to reference the source.', max_length=16,
primary_key=True, serialize=False, verbose_name='PandaSDMX id')),
('name', models.CharField(help_text='Full length name of the data source.', max_length=512,
verbose_name='Name')),
('description', models.TextField(blank=True, help_text='Long description of the data source.',
verbose_name='Description')),
('url', models.URLField(help_text='The base URL of the SDMX endpoint of the data source.',
verbose_name='API URL')),
('documentation', models.URLField(help_text='Documentation URL of the data source.', null=True,
verbose_name='Documentation URL')),
('data_content_type', models.CharField(choices=[('JSON', 'JSON'), ('XML', 'XML')], default='XML',
help_text='The format in which the API returns its data.',
max_length=16, verbose_name='API type')),
('headers',
models.JSONField(default=dict, help_text='HTTP headers to attach to every request, as a JSON object.',
verbose_name='HTTP Headers')),
('resources', models.JSONField(default=dict, help_text='Unknown and undocumented JSON object.',
verbose_name='Resources')),
('supports_agencyscheme', models.BooleanField(default=True,
help_text='Whether the data source supports <a href="https://pandasdmx.readthedocs.io/en/latest/api.html#pandasdmx.model.AgencyScheme">AgencyScheme </a> or not.',
verbose_name='Supports AgencyScheme')),
('supports_categoryscheme', models.BooleanField(default=True,
help_text='Whether the data source supports <a href="https://pandasdmx.readthedocs.io/en/latest/api.html#pandasdmx.model.CategoryScheme">CategoryScheme </a> or not.',
verbose_name='Supports CategoryScheme')),
('supports_codelist', models.BooleanField(default=True,
help_text='Whether the data source supports <a href="https://pandasdmx.readthedocs.io/en/latest/api.html#pandasdmx.model.CodeList">CodeList </a> or not.',
verbose_name='Supports CodeList')),
('supports_conceptscheme', models.BooleanField(default=True,
help_text='Whether the data source supports <a href="https://pandasdmx.readthedocs.io/en/latest/api.html#pandasdmx.model.ConceptScheme">ConceptScheme </a> or not.',
verbose_name='Supports ConceptScheme')),
('supports_data', models.BooleanField(default=True,
help_text='Whether the data source supports <a href="https://pandasdmx.readthedocs.io/en/latest/api.html#pandasdmx.model.DataSet">DataSet </a> or not.',
verbose_name='Supports DataSet')),
('supports_dataflow', models.BooleanField(default=True,
help_text='Whether the data source supports <a href="https://pandasdmx.readthedocs.io/en/latest/api.html#pandasdmx.model.DataflowDefinition">DataflowDefinition </a> or not.',
verbose_name='Supports DataflowDefinition')),
('supports_datastructure', models.BooleanField(default=True,
help_text='Whether the data source supports <a href="https://pandasdmx.readthedocs.io/en/latest/api.html#pandasdmx.model.DataStructureDefinition">CategoryScheme </a> or not.',
verbose_name='Supports DataStructureDefinition')),
('supports_provisionagreement', models.BooleanField(default=True,
help_text='Whether the data source supports <a href="https://pandasdmx.readthedocs.io/en/latest/api.html#pandasdmx.model.ProvisionAgreement">CategoryScheme </a> or not.',
verbose_name='Supports ProvisionAgreement')),
('supports_preview', models.BooleanField(default=False,
help_text='Whether the data source supports <a href="https://pandasdmx.readthedocs.io/en/latest/api.html#pandasdmx.Request.preview_data">previews of data </a> or not.',
verbose_name='Supports previews')),
('supports_structurespecific_data', models.BooleanField(default=False,
help_text='Whether the data source returns <a href="https://pandasdmx.readthedocs.io/en/latest/api.html#pandasdmx.source.Source">structure-specific data messages </a> or not.',
verbose_name='Supports structure-specific data messages')),
('builtin', models.BooleanField(help_text='Whether the source is built-in in PandaSDMX or not.',
verbose_name='Builtin')),
('last_sync', models.DateTimeField(
help_text='The datetime at which the data flows of this source were last syncronized.', null=True,
verbose_name='Last updated')),
],
),
migrations.CreateModel(
name='Project',
fields=[
('slug', models.SlugField(help_text='Unique alphanumeric string which identifies the project.', max_length=64, primary_key=True, serialize=False, verbose_name='Slug')),
('name', models.CharField(help_text='The display name of the project.', max_length=512, verbose_name='Name')),
('description', models.TextField(blank=True, help_text='A brief description of the project, to be displayed inthe overview.', verbose_name='Description')),
('flows', models.ManyToManyField(blank=True, help_text='The DataFlows used in this project.', related_name='used_in', to='core.DataFlow')),
('slug',
models.SlugField(help_text='Unique alphanumeric string which identifies the project.', max_length=64,
primary_key=True, serialize=False, verbose_name='Slug')),
('name',
models.CharField(help_text='The display name of the project.', max_length=512, verbose_name='Name')),
('description', models.TextField(blank=True,
help_text='A brief description of the project, to be displayed inthe overview.',
verbose_name='Description')),
('flows', models.ManyToManyField(blank=True, help_text='The DataFlows used in this project.',
related_name='used_in', to='core.DataFlow')),
],
),
migrations.AddField(
model_name='dataflow',
name='datasource',
field=models.ForeignKey(help_text='The DataSource this object belongs to.', on_delete=django.db.models.deletion.RESTRICT, to='core.datasource'),
field=models.ForeignKey(help_text='The DataSource this object belongs to.',
on_delete=django.db.models.deletion.RESTRICT, to='core.datasource'),
),
migrations.RunPython(create_builtin_sources)
]

26
backend/sophon/core/migrations/0001_sources.json
View file

@ -16,7 +16,9 @@
"url": "https://sdw-wsrest.ecb.europa.eu/service",
"name": "European Central Bank",
"documentation": "https://www.ecb.europa.eu/stats/ecb_statistics/co-operation_and_standards/sdmx/html/index.en.html",
"supports": {"preview": true}
"supports": {
"preview": true
}
},
{
"id": "ESTAT",
@ -72,7 +74,9 @@
"Accept": "application/vnd.sdmx.genericdata+xml;version=2.1"
}
},
"supports": {"provisionagreement": false}
"supports": {
"provisionagreement": false
}
},
{
"id": "ISTAT",
@ -103,7 +107,10 @@
"name": "Norges Bank (NO)",
"documentation": "https://www.norges-bank.no/en/topics/Statistics/open-data/",
"url": "https://data.norges-bank.no/api",
"supports": {"categoryscheme": false, "structure-specific data": true},
"supports": {
"categoryscheme": false,
"structure-specific data": true
},
"headers": {
"data": {
"accept": "application/vnd.sdmx.genericdata+xml;version=2.1"
@ -128,17 +135,22 @@
{
"id": "SPC",
"name": "Pacific Data Hub",
"documentation":"https://stats.pacificdata.org/?locale=en",
"documentation": "https://stats.pacificdata.org/?locale=en",
"url": "https://stats-nsi-stable.pacificdata.org/rest",
"supports": {"preview": false, "provisionagreement": false}
"supports": {
"preview": false,
"provisionagreement": false
}
},
{
"id": "UNSD",
"name": "United Nations Statistics Division",
"documentation": "https://unstats.un.org/home/",
"url": "https://data.un.org/WS/rest",
"supports": {"preview": true, "provisionagreement": false}
"supports": {
"preview": true,
"provisionagreement": false
}
},
{
"id": "WB",

14
backend/sophon/core/migrations/0002_auto_20210415_1453.py
View file

@ -1,12 +1,11 @@
# Generated by Django 3.2 on 2021-04-15 14:53
import django.db.models.deletion
from django.conf import settings
from django.db import migrations, models
import django.db.models.deletion
class Migration(migrations.Migration):
dependencies = [
migrations.swappable_dependency(settings.AUTH_USER_MODEL),
('core', '0001_initial'),
@ -16,18 +15,23 @@ class Migration(migrations.Migration):
migrations.AddField(
model_name='project',
name='collaborators',
field=models.ManyToManyField(blank=True, help_text='The users who can edit the project.', related_name='collaborates_in', to=settings.AUTH_USER_MODEL),
field=models.ManyToManyField(blank=True, help_text='The users who can edit the project.',
related_name='collaborates_in', to=settings.AUTH_USER_MODEL),
),
# No projects should be in the db before this migration is run, so this should be fine
migrations.AddField(
model_name='project',
name='owner',
field=models.ForeignKey(default=0, help_text='The user who owns the project, and has full access to it.', on_delete=django.db.models.deletion.CASCADE, to='auth.user'),
field=models.ForeignKey(default=0, help_text='The user who owns the project, and has full access to it.',
on_delete=django.db.models.deletion.CASCADE, to='auth.user'),
preserve_default=False,
),
migrations.AddField(
model_name='project',
name='visibility',
field=models.CharField(choices=[('PUBLIC', '🌍 Public'), ('INTERNAL', '🏭 Internal'), ('PRIVATE', '🔒 Private')], default='INTERNAL', help_text='A setting specifying who can view the project.', max_length=16, verbose_name='Visibility'),
field=models.CharField(
choices=[('PUBLIC', '🌍 Public'), ('INTERNAL', '🏭 Internal'), ('PRIVATE', '🔒 Private')],
default='INTERNAL', help_text='A setting specifying who can view the project.', max_length=16,
verbose_name='Visibility'),
),
]

1
backend/sophon/core/migrations/0003_rename_id_dataflow_sdmx_id.py
View file

@ -4,7 +4,6 @@ from django.db import migrations
class Migration(migrations.Migration):
dependencies = [
('core', '0002_auto_20210415_1453'),
]

4
backend/sophon/core/migrations/0004_alter_dataflow_surrogate_id.py
View file

@ -4,7 +4,6 @@ from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('core', '0003_rename_id_dataflow_sdmx_id'),
]
@ -13,6 +12,7 @@ class Migration(migrations.Migration):
migrations.AlterField(
model_name='dataflow',
name='surrogate_id',
field=models.BigAutoField(help_text='Internal id used by Django to identify this DataFlow.', primary_key=True, serialize=False, verbose_name='Surrogate id'),
field=models.BigAutoField(help_text='Internal id used by Django to identify this DataFlow.',
primary_key=True, serialize=False, verbose_name='Surrogate id'),
),
]

16
backend/sophon/core/models.py
View file

@ -1,14 +1,13 @@
from django.db import models
from django.core import validators
from django.contrib.auth.models import User
from django.utils import timezone
import json
import logging
import typing as t
import pandas
import pandasdmx
import pandasdmx.message
import typing as t
import json
import abc
import logging
from django.contrib.auth.models import User
from django.db import models
from django.utils import timezone
log = logging.getLogger(__name__)
@ -277,7 +276,6 @@ class DataSource(models.Model):
log.info(f"Syncing DataFlows of {self!r}...")
for description, sdmx_id in zip(flows, flows.index):
db_flow, _created = DataFlow.objects.update_or_create(
**{
"datasource": self,

1
backend/sophon/core/permissions.py
View file

@ -1,4 +1,5 @@
import logging
from rest_framework import permissions
log = logging.getLogger(__name__)

1
backend/sophon/core/serializers.py
View file

@ -1,4 +1,5 @@
from rest_framework import serializers
from . import models

3
backend/sophon/core/urls.py
View file

@ -1,14 +1,13 @@
from django.urls import path, include
from rest_framework.routers import DefaultRouter
from . import views
from . import views
router = DefaultRouter()
router.register("datasources", views.DataSourceViewSet)
router.register("dataflows", views.DataFlowViewSet)
router.register("projects", views.ProjectViewSet)
urlpatterns = [
path("", include(router.urls)),
]

7
backend/sophon/core/views.py
View file

@ -1,8 +1,9 @@
from rest_framework import viewsets, decorators, response, permissions, mixins, generics
from . import models, serializers, permissions as custom_permissions
from datetime import datetime
from logging import getLogger
from rest_framework import viewsets, decorators, response, permissions
from . import models, serializers, permissions as custom_permissions
log = getLogger(__name__)

2
backend/sophon/settings.py
View file

@ -10,8 +10,8 @@ For the full list of settings and their values, see
https://docs.djangoproject.com/en/3.1/ref/settings/
"""
from pathlib import Path
import os
from pathlib import Path
# Build paths inside the project like this: BASE_DIR / 'subdir'.
BASE_DIR = Path(__file__).resolve().parent.parent