steffo/sophon
steffo/sophon
1
Fork 0
mirror of https://github.com/Steffo99/sophon.git synced 2024-12-22 14:54:22 +00:00
Code Activity

💥 Basically rewrite all database parts from scratch

Browse source
This commit is contained in:
Steffo 2021-04-08 16:41:43 +02:00
parent 0a215babeb
commit be1a2c0e03
9 changed files with 534 additions and 177 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

65
sophon/core/admin.py
View file

@ -15,8 +15,8 @@ class ProjectAdmin(CoreAdmin):
"""
list_display = (
"slug",
"name",
"description",
)
@ -27,8 +27,62 @@ class DataSourceAdmin(CoreAdmin):
"""
list_display = (
"pandasdmx_id",
"builtin",
"id",
"name",
"data_content_type",
"last_sync",
)
fieldsets = (
(
None, {
"fields": (
"id",
"name",
"description",
)
}
),
(
"URLs", {
"fields": (
"url",
"documentation",
)
}
),
(
"API configuration", {
"fields": (
"data_content_type",
"headers",
"resources",
)
}
),
(
"Features supported", {
"fields": (
"supports_agencyscheme",
"supports_categoryscheme",
"supports_codelist",
"supports_conceptscheme",
"supports_data",
"supports_dataflow",
"supports_datastructure",
"supports_provisionagreement",
"supports_preview",
"supports_structurespecific_data",
)
}
),
(
"Syncronization", {
"fields": (
"last_sync",
)
}
)
)
@ -39,7 +93,6 @@ class DataFlowAdmin(CoreAdmin):
"""
list_display = (
"sdmx_id",
"datasource_id",
"description",
"datasource",
"id",
)

76
sophon/core/migrations/0001_initial.py
View file

@ -1,39 +1,21 @@
# Generated by Django 3.1.7 on 2021-04-04 23:41
# Manually edited by @Steffo99
# Generated by Django 3.2 on 2021-04-08 14:36
from django.db import migrations, models
import django.db.models.deletion
import importlib.resources
from .. import models as core_models
def create_builtin_sources(apps, schema_editor):
"""
Create in the database the sources that are already built-in in :mod:`pandasdmx`.
This function is called when performing the migration: see
`this page <https://docs.djangoproject.com/en/3.1/topics/migrations/#data-migrations>`_ for details on how this
works!
"""
DataSource = apps.get_model("core", "DataSource")
DataSource.objects.bulk_create([
DataSource(pandasdmx_id="ABS", builtin=True),
DataSource(pandasdmx_id="ESTAT", builtin=True),
DataSource(pandasdmx_id="ECB", builtin=True),
DataSource(pandasdmx_id="ILO", builtin=True),
DataSource(pandasdmx_id="IMF", builtin=True),
DataSource(pandasdmx_id="INEGI", builtin=True),
DataSource(pandasdmx_id="INSEE", builtin=True),
DataSource(pandasdmx_id="ISTAT", builtin=True),
DataSource(pandasdmx_id="LSD", builtin=True),
DataSource(pandasdmx_id="NB", builtin=True),
DataSource(pandasdmx_id="NBB", builtin=True),
DataSource(pandasdmx_id="OECD", builtin=True),
DataSource(pandasdmx_id="SGR", builtin=True),
DataSource(pandasdmx_id="SPC", builtin=True),
DataSource(pandasdmx_id="STAT_EE", builtin=True),
DataSource(pandasdmx_id="UNSD", builtin=True),
DataSource(pandasdmx_id="UNICEF", builtin=True),
DataSource(pandasdmx_id="WB", builtin=True),
DataSource(pandasdmx_id="WB_WDI", builtin=True),
])
file = importlib.resources.open_text("sophon.core.migrations", "0001_sources.json")
core_models.DataSource.create_from_sources_json(file=file)
file.close()
class Migration(migrations.Migration):
@ -44,22 +26,52 @@ class Migration(migrations.Migration):
]
operations = [
migrations.CreateModel(
name='DataFlow',
fields=[
('surrogate_id', models.IntegerField(help_text='Internal id used by Django to identify this DataFlow.', primary_key=True, serialize=False, verbose_name='Surrogate id')),
('id', models.CharField(help_text='Internal string used in SDMX communication to identify the DataFlow.', max_length=64, verbose_name='SDMX id')),
('description', models.TextField(blank=True, help_text='Natural language description of the DataFlow.', verbose_name='Description')),
],
),
migrations.CreateModel(
name='DataSource',
fields=[
('pandasdmx_id', models.CharField(max_length=16, primary_key=True, serialize=False, verbose_name='Internal pandasdmx source id')),
('builtin', models.BooleanField(verbose_name='If the source is builtin in pandasdmx')),
('settings', models.JSONField(null=True, verbose_name='Source info to pass to pandasdmx if the source is not builtin')),
('id', models.CharField(help_text='Internal id used by PandaSDMX to reference the source.', max_length=16, primary_key=True, serialize=False, verbose_name='PandaSDMX id')),
('name', models.CharField(help_text='Full length name of the data source.', max_length=512, verbose_name='Name')),
('description', models.TextField(blank=True, help_text='Long description of the data source.', verbose_name='Description')),
('url', models.URLField(help_text='The base URL of the SDMX endpoint of the data source.', verbose_name='API URL')),
('documentation', models.URLField(help_text='Documentation URL of the data source.', null=True, verbose_name='Documentation URL')),
('data_content_type', models.CharField(choices=[('JSON', 'JSON'), ('XML', 'XML')], default='XML', help_text='The format in which the API returns its data.', max_length=16, verbose_name='API type')),
('headers', models.JSONField(default=dict, help_text='HTTP headers to attach to every request, as a JSON object.', verbose_name='HTTP Headers')),
('resources', models.JSONField(default=dict, help_text='Unknown and undocumented JSON object.', verbose_name='Resources')),
('supports_agencyscheme', models.BooleanField(default=True, help_text='Whether the data source supports <a href="https://pandasdmx.readthedocs.io/en/latest/api.html#pandasdmx.model.AgencyScheme">AgencyScheme </a> or not.', verbose_name='Supports AgencyScheme')),
('supports_categoryscheme', models.BooleanField(default=True, help_text='Whether the data source supports <a href="https://pandasdmx.readthedocs.io/en/latest/api.html#pandasdmx.model.CategoryScheme">CategoryScheme </a> or not.', verbose_name='Supports CategoryScheme')),
('supports_codelist', models.BooleanField(default=True, help_text='Whether the data source supports <a href="https://pandasdmx.readthedocs.io/en/latest/api.html#pandasdmx.model.CodeList">CodeList </a> or not.', verbose_name='Supports CodeList')),
('supports_conceptscheme', models.BooleanField(default=True, help_text='Whether the data source supports <a href="https://pandasdmx.readthedocs.io/en/latest/api.html#pandasdmx.model.ConceptScheme">ConceptScheme </a> or not.', verbose_name='Supports ConceptScheme')),
('supports_data', models.BooleanField(default=True, help_text='Whether the data source supports <a href="https://pandasdmx.readthedocs.io/en/latest/api.html#pandasdmx.model.DataSet">DataSet </a> or not.', verbose_name='Supports DataSet')),
('supports_dataflow', models.BooleanField(default=True, help_text='Whether the data source supports <a href="https://pandasdmx.readthedocs.io/en/latest/api.html#pandasdmx.model.DataflowDefinition">DataflowDefinition </a> or not.', verbose_name='Supports DataflowDefinition')),
('supports_datastructure', models.BooleanField(default=True, help_text='Whether the data source supports <a href="https://pandasdmx.readthedocs.io/en/latest/api.html#pandasdmx.model.DataStructureDefinition">CategoryScheme </a> or not.', verbose_name='Supports DataStructureDefinition')),
('supports_provisionagreement', models.BooleanField(default=True, help_text='Whether the data source supports <a href="https://pandasdmx.readthedocs.io/en/latest/api.html#pandasdmx.model.ProvisionAgreement">CategoryScheme </a> or not.', verbose_name='Supports ProvisionAgreement')),
('supports_preview', models.BooleanField(default=False, help_text='Whether the data source supports <a href="https://pandasdmx.readthedocs.io/en/latest/api.html#pandasdmx.Request.preview_data">previews of data </a> or not.', verbose_name='Supports previews')),
('supports_structurespecific_data', models.BooleanField(default=False, help_text='Whether the data source returns <a href="https://pandasdmx.readthedocs.io/en/latest/api.html#pandasdmx.source.Source">structure-specific data messages </a> or not.', verbose_name='Supports structure-specific data messages')),
('builtin', models.BooleanField(help_text='Whether the source is built-in in PandaSDMX or not.', verbose_name='Builtin')),
('last_sync', models.DateTimeField(help_text='The datetime at which the data flows of this source were last syncronized.', null=True, verbose_name='Last updated')),
],
),
migrations.CreateModel(
name='Project',
fields=[
('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('name', models.CharField(max_length=512, verbose_name='Project name')),
('description', models.CharField(max_length=8192, verbose_name='Project description')),
('sources', models.ManyToManyField(related_name='used_in', to='core.DataSource')),
('slug', models.SlugField(help_text='Unique alphanumeric string which identifies the project.', max_length=64, primary_key=True, serialize=False, verbose_name='Slug')),
('name', models.CharField(help_text='The display name of the project.', max_length=512, verbose_name='Name')),
('description', models.TextField(blank=True, help_text='A brief description of the project, to be displayed inthe overview.', verbose_name='Description')),
('flows', models.ManyToManyField(blank=True, help_text='The DataFlows used in this project.', related_name='used_in', to='core.DataFlow')),
],
),
migrations.AddField(
model_name='dataflow',
name='datasource',
field=models.ForeignKey(help_text='The DataSource this object belongs to.', on_delete=django.db.models.deletion.RESTRICT, to='core.datasource'),
),
migrations.RunPython(create_builtin_sources)
]

182
sophon/core/migrations/0001_sources.json Normal file
View file

@ -0,0 +1,182 @@
[
{
"id": "ABS",
"data_content_type": "JSON",
"url": "https://stat.data.abs.gov.au/sdmx-json",
"name": "Australian Bureau of Statistics",
"documentation": "https://www.abs.gov.au/"
},
{
"id": "ECB",
"resources": {
"data": {
"headers": {}
}
},
"url": "https://sdw-wsrest.ecb.europa.eu/service",
"name": "European Central Bank",
"documentation": "https://www.ecb.europa.eu/stats/ecb_statistics/co-operation_and_standards/sdmx/html/index.en.html",
"supports": {"preview": true}
},
{
"id": "ESTAT",
"documentation": "https://data.un.org/Host.aspx?Content=API",
"url": "https://ec.europa.eu/eurostat/SDMX/diss-web/rest",
"name": "Eurostat",
"supports": {
"agencyscheme": false,
"categoryscheme": false,
"codelist": false,
"conceptscheme": false,
"provisionagreement": false
}
},
{
"id": "ILO",
"name": "International Labor Organization",
"documentation": "https://www.ilo.org/ilostat/",
"url": "https://www.ilo.org/sdmx/rest",
"headers": {
"accept": "application/vnd.sdmx.structurespecificdata+xml;version=2.1"
},
"supports": {
"provisionagreement": false
}
},
{
"id": "IMF",
"url": "https://sdmxcentral.imf.org/ws/public/sdmxapi/rest",
"name": "International Monetary Fund",
"supports": {
"provisionagreement": false
}
},
{
"id": "INEGI",
"url": "https://sdmx.snieg.mx/service/rest",
"name": "Instituto Nacional de Estadística y Geografía (MX)",
"documentation": "https://sdmx.snieg.mx/infrastructure",
"supports": {
"agencyscheme": false,
"provisionagreement": false,
"structure-specific data": true
}
},
{
"id": "INSEE",
"name": "Institut national de la statistique et des études économiques (FR)",
"documentation": "https://www.bdm.insee.fr/bdm2/statique?page=sdmx",
"url": "https://www.bdm.insee.fr/series/sdmx",
"headers": {
"data": {
"Accept": "application/vnd.sdmx.genericdata+xml;version=2.1"
}
},
"supports": {"provisionagreement": false}
},
{
"id": "ISTAT",
"name": "Instituto Nationale di Statistica (IT)",
"documentation": "https://ec.europa.eu/eurostat/web/sdmx-web-services/rest-sdmx-2.1",
"url": "http://sdmx.istat.it/SDMXWS/rest",
"supports": {
"provisionagreement": false,
"structure-specific data": true
}
},
{
"id": "OECD",
"data_content_type": "JSON",
"url": "https://stats.oecd.org/SDMX-JSON",
"documentation": "https://stats.oecd.org/SDMX-JSON/",
"name": "Organisation for Economic Co-operation and Development"
},
{
"id": "NBB",
"data_content_type": "JSON",
"documentation": "https://www.nbb.be/doc/dq/migratie_belgostat/en/nbb_stat-technical-manual.pdf",
"url": "https://stat.nbb.be/sdmx-json",
"name": "National Bank of Belgium"
},
{
"id": "NB",
"name": "Norges Bank (NO)",
"documentation": "https://www.norges-bank.no/en/topics/Statistics/open-data/",
"url": "https://data.norges-bank.no/api",
"supports": {"categoryscheme": false, "structure-specific data": true},
"headers": {
"data": {
"accept": "application/vnd.sdmx.genericdata+xml;version=2.1"
}
}
},
{
"id": "SGR",
"url": "https://registry.sdmx.org/ws/rest",
"name": "SDMX Global Registry",
"documentation": "https://registry.sdmx.org/ws/rest"
},
{
"id": "UNICEF",
"name": "UN International Children's Emergency Fund",
"documentation": "https://data.unicef.org/",
"url": "https://sdmx.data.unicef.org/ws/public/sdmxapi/rest",
"headers": {
"accept": "application/vnd.sdmx.structure+xml;version=2.1"
}
},
{
"id": "SPC",
"name": "Pacific Data Hub",
"documentation":"https://stats.pacificdata.org/?locale=en",
"url": "https://stats-nsi-stable.pacificdata.org/rest",
"supports": {"preview": false, "provisionagreement": false}
},
{
"id": "UNSD",
"name": "United Nations Statistics Division",
"documentation": "https://unstats.un.org/home/",
"url": "https://data.un.org/WS/rest",
"supports": {"preview": true, "provisionagreement": false}
},
{
"id": "WB",
"name": "World Bank World Integrated Trade Solution",
"documentation": "http://wits.worldbank.org",
"url": "http://wits.worldbank.org/API/V1/SDMX/V21/rest",
"supports": {
"agencyscheme": false,
"provisionagreement": false
}
},
{
"id": "WB_WDI",
"name": "World Bank World Development Indicators",
"documentation": "https://datahelpdesk.worldbank.org/knowledgebase/articles/1886701-sdmx-api-queries",
"url": "http://api.worldbank.org/v2/sdmx/rest",
"supports": {
"provisionagreement": false,
"structure-specific data": true
}
},
{
"id": "LSD",
"documentation": "https://osp.stat.gov.lt/rdb-rest",
"url": "https://osp-rs.stat.gov.lt/rest_xml",
"name": "Statistics Lithuania",
"supports": {
"categoryscheme": false,
"codelist": false,
"conceptscheme": false,
"provisionagreement": false
}
},
{
"id": "STAT_EE",
"data_content_type": "JSON",
"documentation": "https://www.stat.ee/sites/default/files/2020-09/API-instructions.pdf",
"url": "http://andmebaas.stat.ee/sdmx-json",
"name": "Statistics Estonia"
}
]

43
sophon/core/migrations/0002_auto_20210405_1641.py
View file

@ -1,43 +0,0 @@
# Generated by Django 3.1.7 on 2021-04-05 16:41
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('core', '0001_initial'),
]
operations = [
migrations.AlterField(
model_name='datasource',
name='builtin',
field=models.BooleanField(help_text='Whether the source is builtin in PandaSDMX or not.', verbose_name='Builtin'),
),
migrations.AlterField(
model_name='datasource',
name='pandasdmx_id',
field=models.CharField(help_text='Internal id used by PandaSDMX to reference the source.', max_length=16, primary_key=True, serialize=False, verbose_name='PandaSDMX id'),
),
migrations.AlterField(
model_name='datasource',
name='settings',
field=models.JSONField(help_text='Info parameter to pass to pandasdmx.add_source if the source is not builtin (see https://pandasdmx.readthedocs.io/en/latest/api.html#pandasdmx.add_source).', null=True, verbose_name='Settings'),
),
migrations.AlterField(
model_name='project',
name='description',
field=models.CharField(help_text='A brief description of the project, to be displayed inthe overview.', max_length=8192, verbose_name='Project description'),
),
migrations.AlterField(
model_name='project',
name='name',
field=models.CharField(help_text='The display name of the project.', max_length=512, verbose_name='Project name'),
),
migrations.AlterField(
model_name='project',
name='sources',
field=models.ManyToManyField(help_text='The sources used by this project.', null=True, related_name='used_in', to='core.DataSource'),
),
]

18
sophon/core/migrations/0003_auto_20210406_0044.py
View file

@ -1,18 +0,0 @@
# Generated by Django 3.1.7 on 2021-04-06 00:44
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('core', '0002_auto_20210405_1641'),
]
operations = [
migrations.AlterField(
model_name='project',
name='sources',
field=models.ManyToManyField(blank=True, help_text='The sources used by this project.', related_name='used_in', to='core.DataSource'),
),
]

24
sophon/core/migrations/0004_dataflow.py
View file

@ -1,24 +0,0 @@
# Generated by Django 3.1.7 on 2021-04-06 20:55
from django.db import migrations, models
import django.db.models.deletion
class Migration(migrations.Migration):
dependencies = [
('core', '0003_auto_20210406_0044'),
]
operations = [
migrations.CreateModel(
name='DataFlow',
fields=[
('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('sdmx_id', models.CharField(help_text='Internal string used in SDMX communication to identify the DataFlow.', max_length=64, verbose_name='SDMX id')),
('last_update', models.DateTimeField(help_text='The datetime at which the properties of this DataFlow were last updated.', verbose_name='Last updated')),
('description', models.CharField(help_text='Natural language description of the DataFlow.', max_length=8192, verbose_name='DataFlow description')),
('datasource_id', models.ForeignKey(help_text='The DataSource this object belongs to.', on_delete=django.db.models.deletion.RESTRICT, to='core.datasource')),
],
),
]

18
sophon/core/migrations/0005_alter_dataflow_last_update.py
View file

@ -1,18 +0,0 @@
# Generated by Django 3.2 on 2021-04-07 17:34
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('core', '0004_dataflow'),
]
operations = [
migrations.AlterField(
model_name='dataflow',
name='last_update',
field=models.DateTimeField(auto_now=True, help_text='The datetime at which the properties of this DataFlow were last updated.', verbose_name='Last updated'),
),
]

256
sophon/core/models.py
View file

@ -1,7 +1,10 @@
from django.db import models
from django.core import validators
import pandas
import pandasdmx
import pandasdmx.message
import typing as t
import json
class DataSource(models.Model):
@ -16,24 +19,203 @@ class DataSource(models.Model):
method.
"""
pandasdmx_id = models.CharField(
id = models.CharField(
"PandaSDMX id",
help_text="Internal id used by PandaSDMX to reference the source.",
max_length=16,
primary_key=True,
)
name = models.CharField(
"Name",
help_text="Full length name of the data source.",
max_length=512,
)
description = models.TextField(
"Description",
help_text="Long description of the data source.",
blank=True,
)
url = models.URLField(
"API URL",
help_text="The base URL of the SDMX endpoint of the data source."
)
documentation = models.URLField(
"Documentation URL",
help_text="Documentation URL of the data source.",
null=True,
)
data_content_type = models.CharField(
"API type",
help_text="The format in which the API returns its data.",
choices=[
("JSON", "JSON"),
("XML", "XML"),
],
default="XML",
max_length=16,
)
headers = models.JSONField(
"HTTP Headers",
help_text="HTTP headers to attach to every request, as a JSON object.",
default=dict,
)
resources = models.JSONField(
"Resources",
help_text="Unknown and undocumented JSON object.",
default=dict,
)
supports_agencyscheme = models.BooleanField(
"Supports AgencyScheme",
help_text='Whether the data source supports '
'<a href="https://pandasdmx.readthedocs.io/en/latest/api.html#pandasdmx.model.AgencyScheme">'
'AgencyScheme '
'</a> or not.',
default=True,
)
supports_categoryscheme = models.BooleanField(
"Supports CategoryScheme",
help_text='Whether the data source supports '
'<a href="https://pandasdmx.readthedocs.io/en/latest/api.html#pandasdmx.model.CategoryScheme">'
'CategoryScheme '
'</a> or not.',
default=True,
)
supports_codelist = models.BooleanField(
"Supports CodeList",
help_text='Whether the data source supports '
'<a href="https://pandasdmx.readthedocs.io/en/latest/api.html#pandasdmx.model.CodeList">'
'CodeList '
'</a> or not.',
default=True,
)
supports_conceptscheme = models.BooleanField(
"Supports ConceptScheme",
help_text='Whether the data source supports '
'<a href="https://pandasdmx.readthedocs.io/en/latest/api.html#pandasdmx.model.ConceptScheme">'
'ConceptScheme '
'</a> or not.',
default=True,
)
supports_data = models.BooleanField(
"Supports DataSet",
help_text='Whether the data source supports '
'<a href="https://pandasdmx.readthedocs.io/en/latest/api.html#pandasdmx.model.DataSet">'
'DataSet '
'</a> or not.',
default=True,
)
supports_dataflow = models.BooleanField(
"Supports DataflowDefinition",
help_text='Whether the data source supports '
'<a href="https://pandasdmx.readthedocs.io/en/latest/api.html#pandasdmx.model.DataflowDefinition">'
'DataflowDefinition '
'</a> or not.',
default=True,
)
supports_datastructure = models.BooleanField(
"Supports DataStructureDefinition",
help_text='Whether the data source supports '
'<a href="https://pandasdmx.readthedocs.io/en/latest/api.html#pandasdmx.model.DataStructureDefinition">'
'CategoryScheme '
'</a> or not.',
default=True,
)
supports_provisionagreement = models.BooleanField(
"Supports ProvisionAgreement",
help_text='Whether the data source supports '
'<a href="https://pandasdmx.readthedocs.io/en/latest/api.html#pandasdmx.model.ProvisionAgreement">'
'CategoryScheme '
'</a> or not.',
default=True,
)
supports_preview = models.BooleanField(
"Supports previews",
help_text='Whether the data source supports '
'<a href="https://pandasdmx.readthedocs.io/en/latest/api.html#pandasdmx.Request.preview_data">'
'previews of data '
'</a> or not.',
default=False,
)
supports_structurespecific_data = models.BooleanField(
"Supports structure-specific data messages",
help_text='Whether the data source returns '
'<a href="https://pandasdmx.readthedocs.io/en/latest/api.html#pandasdmx.source.Source">'
'structure-specific data messages '
'</a> or not.',
default=False,
)
def supports_dict(self) -> dict:
return {
"agencyscheme": self.supports_agencyscheme,
"categoryscheme": self.supports_categoryscheme,
"codelist": self.supports_codelist,
"conceptscheme": self.supports_conceptscheme,
"data": self.supports_data,
"dataflow": self.supports_dataflow,
"datastructure": self.supports_datastructure,
"provisionagreement": self.supports_provisionagreement,
"preview": self.supports_preview,
"structure-specific data": self.supports_structurespecific_data,
}
def info_dict(self) -> dict:
return {
"id": self.id,
"name": self.name,
"data_content_type": self.data_content_type,
"url": self.url,
"documentation": self.documentation,
"supports": self.supports_dict(),
"headers": self.headers,
"resources": self.resources,
}
builtin = models.BooleanField(
"Builtin",
help_text="Whether the source is builtin in PandaSDMX or not.",
)
settings = models.JSONField(
"Settings",
help_text="Info parameter to pass to pandasdmx.add_source if the source is not builtin "
"(see https://pandasdmx.readthedocs.io/en/latest/api.html#pandasdmx.add_source).",
null=True
help_text="Whether the source is built-in in PandaSDMX or not.",
)
@classmethod
def create_from_sources_json(cls, file: t.TextIO):
j_sources: list = json.load(file)
for j_source in j_sources:
# Flatten supports
if supports := j_source.get("supports"):
del j_source["supports"]
for key, value in supports.items():
if key == "structure-specific data":
j_source["supports_structurespecific_data"] = value
else:
j_source[f"supports_{key}"] = value
cls.objects.update_or_create(
id=j_source["id"],
defaults={
**j_source,
"builtin": True,
}
)
def to_pandasdmx_source(self) -> pandasdmx.source.Source:
"""
Convert the :class:`.DataSource` to a :class:`pandasdmx.source.Source`\\ .
@ -42,7 +224,7 @@ class DataSource(models.Model):
.. todo:: :func:`.to_pandasdmx` does not currently support non :attr:`.builtin` sources.
"""
return pandasdmx.source.sources[self.pandasdmx_id]
return pandasdmx.source.sources[self.id]
def to_pandasdmx_request(self) -> pandasdmx.Request:
"""
@ -52,6 +234,12 @@ class DataSource(models.Model):
"""
return pandasdmx.Request(source=self.to_pandasdmx_source().id)
last_sync = models.DateTimeField(
"Last updated",
help_text="The datetime at which the data flows of this source were last syncronized.",
null=True,
)
def request_flows(self) -> tuple[pandas.Series, pandas.Series]:
"""
Retrieve all available dataflows and datastructures as two :class:`pandas.Series`\\ .
@ -71,7 +259,7 @@ class DataSource(models.Model):
return flows, structs
def __str__(self):
return self.pandasdmx_id
return self.id
class DataFlow(models.Model):
@ -81,30 +269,32 @@ class DataFlow(models.Model):
See `this page <https://ec.europa.eu/eurostat/online-help/redisstat-admin/en/TECH_A_main/>`_ for more details.
"""
datasource_id = models.ForeignKey(
surrogate_id = models.IntegerField(
"Surrogate id",
help_text="Internal id used by Django to identify this DataFlow.",
primary_key=True,
)
datasource = models.ForeignKey(
DataSource,
help_text="The DataSource this object belongs to.",
on_delete=models.RESTRICT,
)
sdmx_id = models.CharField(
id = models.CharField(
"SDMX id",
help_text="Internal string used in SDMX communication to identify the DataFlow.",
max_length=64,
)
last_update = models.DateTimeField(
"Last updated",
help_text="The datetime at which the properties of this DataFlow were last updated.",
auto_now=True,
)
description = models.CharField(
"DataFlow description",
description = models.TextField(
"Description",
help_text="Natural language description of the DataFlow.",
max_length=8192,
blank=True,
)
def __str__(self):
return f"{self.datasource_id} | {self.sdmx_id} | {self.description}"
return f"[{self.datasource}] {self.id}"
class Project(models.Model):
@ -113,23 +303,31 @@ class Project(models.Model):
hypothesis.
"""
slug = models.SlugField(
"Slug",
help_text="Unique alphanumeric string which identifies the project.",
max_length=64,
primary_key=True,
)
name = models.CharField(
"Project name",
"Name",
help_text="The display name of the project.",
max_length=512,
)
description = models.CharField(
"Project description",
description = models.TextField(
"Description",
help_text="A brief description of the project, to be displayed inthe overview.",
max_length=8192,
blank=True,
)
sources = models.ManyToManyField(
DataSource,
help_text="The sources used by this project.",
flows = models.ManyToManyField(
DataFlow,
help_text="The DataFlows used in this project.",
related_name="used_in",
blank=True,
)
def __str__(self):
return self.name
return self.slug

29
sophon/core/serializers.py
View file

@ -10,9 +10,25 @@ class DataSourceSerializer(serializers.ModelSerializer):
class Meta:
model = models.DataSource
fields = [
"pandasdmx_id",
"id",
"name",
"description",
"url",
"documentation",
"data_content_type",
"headers",
"supports_agencyscheme",
"supports_categoryscheme",
"supports_codelist",
"supports_conceptscheme",
"supports_data",
"supports_dataflow",
"supports_datastructure",
"supports_provisionagreement",
"supports_preview",
"supports_structurespecific_data",
"builtin",
"settings",
"last_sync",
]
@ -24,10 +40,9 @@ class DataFlowSerializer(serializers.ModelSerializer):
class Meta:
model = models.DataFlow
fields = [
"surrogate_id",
"datasource",
"id",
"datasource_id",
"sdmx_id",
"last_update",
"description",
]
@ -40,8 +55,8 @@ class ProjectSerializer(serializers.ModelSerializer):
class Meta:
model = models.Project
fields = [
"id",
"slug",
"name",
"description",
"sources",
"flows",
]