From be1a2c0e038f742acc52c859a5875ca3d6e75597 Mon Sep 17 00:00:00 2001 From: Stefano Pigozzi Date: Thu, 8 Apr 2021 16:41:43 +0200 Subject: [PATCH] =?UTF-8?q?=F0=9F=92=A5=20Basically=20rewrite=20all=20data?= =?UTF-8?q?base=20parts=20from=20scratch?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sophon/core/admin.py | 65 ++++- sophon/core/migrations/0001_initial.py | 76 +++--- sophon/core/migrations/0001_sources.json | 182 +++++++++++++ .../migrations/0002_auto_20210405_1641.py | 43 --- .../migrations/0003_auto_20210406_0044.py | 18 -- sophon/core/migrations/0004_dataflow.py | 24 -- .../0005_alter_dataflow_last_update.py | 18 -- sophon/core/models.py | 256 ++++++++++++++++-- sophon/core/serializers.py | 29 +- 9 files changed, 534 insertions(+), 177 deletions(-) create mode 100644 sophon/core/migrations/0001_sources.json delete mode 100644 sophon/core/migrations/0002_auto_20210405_1641.py delete mode 100644 sophon/core/migrations/0003_auto_20210406_0044.py delete mode 100644 sophon/core/migrations/0004_dataflow.py delete mode 100644 sophon/core/migrations/0005_alter_dataflow_last_update.py diff --git a/sophon/core/admin.py b/sophon/core/admin.py index ca4d77e..645dcf5 100644 --- a/sophon/core/admin.py +++ b/sophon/core/admin.py @@ -15,8 +15,8 @@ class ProjectAdmin(CoreAdmin): """ list_display = ( + "slug", "name", - "description", ) @@ -27,8 +27,62 @@ class DataSourceAdmin(CoreAdmin): """ list_display = ( - "pandasdmx_id", - "builtin", + "id", + "name", + "data_content_type", + "last_sync", + ) + + fieldsets = ( + ( + None, { + "fields": ( + "id", + "name", + "description", + ) + } + ), + ( + "URLs", { + "fields": ( + "url", + "documentation", + ) + } + ), + ( + "API configuration", { + "fields": ( + "data_content_type", + "headers", + "resources", + ) + } + ), + ( + "Features supported", { + "fields": ( + "supports_agencyscheme", + "supports_categoryscheme", + "supports_codelist", + "supports_conceptscheme", + "supports_data", + "supports_dataflow", + "supports_datastructure", + "supports_provisionagreement", + "supports_preview", + "supports_structurespecific_data", + ) + } + ), + ( + "Syncronization", { + "fields": ( + "last_sync", + ) + } + ) ) @@ -39,7 +93,6 @@ class DataFlowAdmin(CoreAdmin): """ list_display = ( - "sdmx_id", - "datasource_id", - "description", + "datasource", + "id", ) diff --git a/sophon/core/migrations/0001_initial.py b/sophon/core/migrations/0001_initial.py index 1b2bc99..d36e104 100644 --- a/sophon/core/migrations/0001_initial.py +++ b/sophon/core/migrations/0001_initial.py @@ -1,39 +1,21 @@ -# Generated by Django 3.1.7 on 2021-04-04 23:41 -# Manually edited by @Steffo99 +# Generated by Django 3.2 on 2021-04-08 14:36 from django.db import migrations, models +import django.db.models.deletion +import importlib.resources +from .. import models as core_models def create_builtin_sources(apps, schema_editor): """ Create in the database the sources that are already built-in in :mod:`pandasdmx`. - This function is called when performing the migration: see `this page `_ for details on how this works! """ - DataSource = apps.get_model("core", "DataSource") - DataSource.objects.bulk_create([ - DataSource(pandasdmx_id="ABS", builtin=True), - DataSource(pandasdmx_id="ESTAT", builtin=True), - DataSource(pandasdmx_id="ECB", builtin=True), - DataSource(pandasdmx_id="ILO", builtin=True), - DataSource(pandasdmx_id="IMF", builtin=True), - DataSource(pandasdmx_id="INEGI", builtin=True), - DataSource(pandasdmx_id="INSEE", builtin=True), - DataSource(pandasdmx_id="ISTAT", builtin=True), - DataSource(pandasdmx_id="LSD", builtin=True), - DataSource(pandasdmx_id="NB", builtin=True), - DataSource(pandasdmx_id="NBB", builtin=True), - DataSource(pandasdmx_id="OECD", builtin=True), - DataSource(pandasdmx_id="SGR", builtin=True), - DataSource(pandasdmx_id="SPC", builtin=True), - DataSource(pandasdmx_id="STAT_EE", builtin=True), - DataSource(pandasdmx_id="UNSD", builtin=True), - DataSource(pandasdmx_id="UNICEF", builtin=True), - DataSource(pandasdmx_id="WB", builtin=True), - DataSource(pandasdmx_id="WB_WDI", builtin=True), - ]) + file = importlib.resources.open_text("sophon.core.migrations", "0001_sources.json") + core_models.DataSource.create_from_sources_json(file=file) + file.close() class Migration(migrations.Migration): @@ -44,22 +26,52 @@ class Migration(migrations.Migration): ] operations = [ + migrations.CreateModel( + name='DataFlow', + fields=[ + ('surrogate_id', models.IntegerField(help_text='Internal id used by Django to identify this DataFlow.', primary_key=True, serialize=False, verbose_name='Surrogate id')), + ('id', models.CharField(help_text='Internal string used in SDMX communication to identify the DataFlow.', max_length=64, verbose_name='SDMX id')), + ('description', models.TextField(blank=True, help_text='Natural language description of the DataFlow.', verbose_name='Description')), + ], + ), migrations.CreateModel( name='DataSource', fields=[ - ('pandasdmx_id', models.CharField(max_length=16, primary_key=True, serialize=False, verbose_name='Internal pandasdmx source id')), - ('builtin', models.BooleanField(verbose_name='If the source is builtin in pandasdmx')), - ('settings', models.JSONField(null=True, verbose_name='Source info to pass to pandasdmx if the source is not builtin')), + ('id', models.CharField(help_text='Internal id used by PandaSDMX to reference the source.', max_length=16, primary_key=True, serialize=False, verbose_name='PandaSDMX id')), + ('name', models.CharField(help_text='Full length name of the data source.', max_length=512, verbose_name='Name')), + ('description', models.TextField(blank=True, help_text='Long description of the data source.', verbose_name='Description')), + ('url', models.URLField(help_text='The base URL of the SDMX endpoint of the data source.', verbose_name='API URL')), + ('documentation', models.URLField(help_text='Documentation URL of the data source.', null=True, verbose_name='Documentation URL')), + ('data_content_type', models.CharField(choices=[('JSON', 'JSON'), ('XML', 'XML')], default='XML', help_text='The format in which the API returns its data.', max_length=16, verbose_name='API type')), + ('headers', models.JSONField(default=dict, help_text='HTTP headers to attach to every request, as a JSON object.', verbose_name='HTTP Headers')), + ('resources', models.JSONField(default=dict, help_text='Unknown and undocumented JSON object.', verbose_name='Resources')), + ('supports_agencyscheme', models.BooleanField(default=True, help_text='Whether the data source supports AgencyScheme or not.', verbose_name='Supports AgencyScheme')), + ('supports_categoryscheme', models.BooleanField(default=True, help_text='Whether the data source supports CategoryScheme or not.', verbose_name='Supports CategoryScheme')), + ('supports_codelist', models.BooleanField(default=True, help_text='Whether the data source supports CodeList or not.', verbose_name='Supports CodeList')), + ('supports_conceptscheme', models.BooleanField(default=True, help_text='Whether the data source supports ConceptScheme or not.', verbose_name='Supports ConceptScheme')), + ('supports_data', models.BooleanField(default=True, help_text='Whether the data source supports DataSet or not.', verbose_name='Supports DataSet')), + ('supports_dataflow', models.BooleanField(default=True, help_text='Whether the data source supports DataflowDefinition or not.', verbose_name='Supports DataflowDefinition')), + ('supports_datastructure', models.BooleanField(default=True, help_text='Whether the data source supports CategoryScheme or not.', verbose_name='Supports DataStructureDefinition')), + ('supports_provisionagreement', models.BooleanField(default=True, help_text='Whether the data source supports CategoryScheme or not.', verbose_name='Supports ProvisionAgreement')), + ('supports_preview', models.BooleanField(default=False, help_text='Whether the data source supports previews of data or not.', verbose_name='Supports previews')), + ('supports_structurespecific_data', models.BooleanField(default=False, help_text='Whether the data source returns structure-specific data messages or not.', verbose_name='Supports structure-specific data messages')), + ('builtin', models.BooleanField(help_text='Whether the source is built-in in PandaSDMX or not.', verbose_name='Builtin')), + ('last_sync', models.DateTimeField(help_text='The datetime at which the data flows of this source were last syncronized.', null=True, verbose_name='Last updated')), ], ), migrations.CreateModel( name='Project', fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('name', models.CharField(max_length=512, verbose_name='Project name')), - ('description', models.CharField(max_length=8192, verbose_name='Project description')), - ('sources', models.ManyToManyField(related_name='used_in', to='core.DataSource')), + ('slug', models.SlugField(help_text='Unique alphanumeric string which identifies the project.', max_length=64, primary_key=True, serialize=False, verbose_name='Slug')), + ('name', models.CharField(help_text='The display name of the project.', max_length=512, verbose_name='Name')), + ('description', models.TextField(blank=True, help_text='A brief description of the project, to be displayed inthe overview.', verbose_name='Description')), + ('flows', models.ManyToManyField(blank=True, help_text='The DataFlows used in this project.', related_name='used_in', to='core.DataFlow')), ], ), + migrations.AddField( + model_name='dataflow', + name='datasource', + field=models.ForeignKey(help_text='The DataSource this object belongs to.', on_delete=django.db.models.deletion.RESTRICT, to='core.datasource'), + ), migrations.RunPython(create_builtin_sources) ] diff --git a/sophon/core/migrations/0001_sources.json b/sophon/core/migrations/0001_sources.json new file mode 100644 index 0000000..da90d86 --- /dev/null +++ b/sophon/core/migrations/0001_sources.json @@ -0,0 +1,182 @@ +[ + { + "id": "ABS", + "data_content_type": "JSON", + "url": "https://stat.data.abs.gov.au/sdmx-json", + "name": "Australian Bureau of Statistics", + "documentation": "https://www.abs.gov.au/" + }, + { + "id": "ECB", + "resources": { + "data": { + "headers": {} + } + }, + "url": "https://sdw-wsrest.ecb.europa.eu/service", + "name": "European Central Bank", + "documentation": "https://www.ecb.europa.eu/stats/ecb_statistics/co-operation_and_standards/sdmx/html/index.en.html", + "supports": {"preview": true} + }, + { + "id": "ESTAT", + "documentation": "https://data.un.org/Host.aspx?Content=API", + "url": "https://ec.europa.eu/eurostat/SDMX/diss-web/rest", + "name": "Eurostat", + "supports": { + "agencyscheme": false, + "categoryscheme": false, + "codelist": false, + "conceptscheme": false, + "provisionagreement": false + } + }, + { + "id": "ILO", + "name": "International Labor Organization", + "documentation": "https://www.ilo.org/ilostat/", + "url": "https://www.ilo.org/sdmx/rest", + "headers": { + "accept": "application/vnd.sdmx.structurespecificdata+xml;version=2.1" + }, + "supports": { + "provisionagreement": false + } + }, + { + "id": "IMF", + "url": "https://sdmxcentral.imf.org/ws/public/sdmxapi/rest", + "name": "International Monetary Fund", + "supports": { + "provisionagreement": false + } + }, + { + "id": "INEGI", + "url": "https://sdmx.snieg.mx/service/rest", + "name": "Instituto Nacional de Estadística y Geografía (MX)", + "documentation": "https://sdmx.snieg.mx/infrastructure", + "supports": { + "agencyscheme": false, + "provisionagreement": false, + "structure-specific data": true + } + }, + { + "id": "INSEE", + "name": "Institut national de la statistique et des études économiques (FR)", + "documentation": "https://www.bdm.insee.fr/bdm2/statique?page=sdmx", + "url": "https://www.bdm.insee.fr/series/sdmx", + "headers": { + "data": { + "Accept": "application/vnd.sdmx.genericdata+xml;version=2.1" + } + }, + "supports": {"provisionagreement": false} + }, + { + "id": "ISTAT", + "name": "Instituto Nationale di Statistica (IT)", + "documentation": "https://ec.europa.eu/eurostat/web/sdmx-web-services/rest-sdmx-2.1", + "url": "http://sdmx.istat.it/SDMXWS/rest", + "supports": { + "provisionagreement": false, + "structure-specific data": true + } + }, + { + "id": "OECD", + "data_content_type": "JSON", + "url": "https://stats.oecd.org/SDMX-JSON", + "documentation": "https://stats.oecd.org/SDMX-JSON/", + "name": "Organisation for Economic Co-operation and Development" + }, + { + "id": "NBB", + "data_content_type": "JSON", + "documentation": "https://www.nbb.be/doc/dq/migratie_belgostat/en/nbb_stat-technical-manual.pdf", + "url": "https://stat.nbb.be/sdmx-json", + "name": "National Bank of Belgium" + }, + { + "id": "NB", + "name": "Norges Bank (NO)", + "documentation": "https://www.norges-bank.no/en/topics/Statistics/open-data/", + "url": "https://data.norges-bank.no/api", + "supports": {"categoryscheme": false, "structure-specific data": true}, + "headers": { + "data": { + "accept": "application/vnd.sdmx.genericdata+xml;version=2.1" + } + } + }, + { + "id": "SGR", + "url": "https://registry.sdmx.org/ws/rest", + "name": "SDMX Global Registry", + "documentation": "https://registry.sdmx.org/ws/rest" + }, + { + "id": "UNICEF", + "name": "UN International Children's Emergency Fund", + "documentation": "https://data.unicef.org/", + "url": "https://sdmx.data.unicef.org/ws/public/sdmxapi/rest", + "headers": { + "accept": "application/vnd.sdmx.structure+xml;version=2.1" + } + }, + { + "id": "SPC", + "name": "Pacific Data Hub", + "documentation":"https://stats.pacificdata.org/?locale=en", + "url": "https://stats-nsi-stable.pacificdata.org/rest", + "supports": {"preview": false, "provisionagreement": false} + }, + + { + "id": "UNSD", + "name": "United Nations Statistics Division", + "documentation": "https://unstats.un.org/home/", + "url": "https://data.un.org/WS/rest", + "supports": {"preview": true, "provisionagreement": false} + }, + { + "id": "WB", + "name": "World Bank World Integrated Trade Solution", + "documentation": "http://wits.worldbank.org", + "url": "http://wits.worldbank.org/API/V1/SDMX/V21/rest", + "supports": { + "agencyscheme": false, + "provisionagreement": false + } + }, + { + "id": "WB_WDI", + "name": "World Bank World Development Indicators", + "documentation": "https://datahelpdesk.worldbank.org/knowledgebase/articles/1886701-sdmx-api-queries", + "url": "http://api.worldbank.org/v2/sdmx/rest", + "supports": { + "provisionagreement": false, + "structure-specific data": true + } + }, + { + "id": "LSD", + "documentation": "https://osp.stat.gov.lt/rdb-rest", + "url": "https://osp-rs.stat.gov.lt/rest_xml", + "name": "Statistics Lithuania", + "supports": { + "categoryscheme": false, + "codelist": false, + "conceptscheme": false, + "provisionagreement": false + } + }, + { + "id": "STAT_EE", + "data_content_type": "JSON", + "documentation": "https://www.stat.ee/sites/default/files/2020-09/API-instructions.pdf", + "url": "http://andmebaas.stat.ee/sdmx-json", + "name": "Statistics Estonia" + } +] diff --git a/sophon/core/migrations/0002_auto_20210405_1641.py b/sophon/core/migrations/0002_auto_20210405_1641.py deleted file mode 100644 index 6095ddf..0000000 --- a/sophon/core/migrations/0002_auto_20210405_1641.py +++ /dev/null @@ -1,43 +0,0 @@ -# Generated by Django 3.1.7 on 2021-04-05 16:41 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('core', '0001_initial'), - ] - - operations = [ - migrations.AlterField( - model_name='datasource', - name='builtin', - field=models.BooleanField(help_text='Whether the source is builtin in PandaSDMX or not.', verbose_name='Builtin'), - ), - migrations.AlterField( - model_name='datasource', - name='pandasdmx_id', - field=models.CharField(help_text='Internal id used by PandaSDMX to reference the source.', max_length=16, primary_key=True, serialize=False, verbose_name='PandaSDMX id'), - ), - migrations.AlterField( - model_name='datasource', - name='settings', - field=models.JSONField(help_text='Info parameter to pass to pandasdmx.add_source if the source is not builtin (see https://pandasdmx.readthedocs.io/en/latest/api.html#pandasdmx.add_source).', null=True, verbose_name='Settings'), - ), - migrations.AlterField( - model_name='project', - name='description', - field=models.CharField(help_text='A brief description of the project, to be displayed inthe overview.', max_length=8192, verbose_name='Project description'), - ), - migrations.AlterField( - model_name='project', - name='name', - field=models.CharField(help_text='The display name of the project.', max_length=512, verbose_name='Project name'), - ), - migrations.AlterField( - model_name='project', - name='sources', - field=models.ManyToManyField(help_text='The sources used by this project.', null=True, related_name='used_in', to='core.DataSource'), - ), - ] diff --git a/sophon/core/migrations/0003_auto_20210406_0044.py b/sophon/core/migrations/0003_auto_20210406_0044.py deleted file mode 100644 index ca4b218..0000000 --- a/sophon/core/migrations/0003_auto_20210406_0044.py +++ /dev/null @@ -1,18 +0,0 @@ -# Generated by Django 3.1.7 on 2021-04-06 00:44 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('core', '0002_auto_20210405_1641'), - ] - - operations = [ - migrations.AlterField( - model_name='project', - name='sources', - field=models.ManyToManyField(blank=True, help_text='The sources used by this project.', related_name='used_in', to='core.DataSource'), - ), - ] diff --git a/sophon/core/migrations/0004_dataflow.py b/sophon/core/migrations/0004_dataflow.py deleted file mode 100644 index 988c49c..0000000 --- a/sophon/core/migrations/0004_dataflow.py +++ /dev/null @@ -1,24 +0,0 @@ -# Generated by Django 3.1.7 on 2021-04-06 20:55 - -from django.db import migrations, models -import django.db.models.deletion - - -class Migration(migrations.Migration): - - dependencies = [ - ('core', '0003_auto_20210406_0044'), - ] - - operations = [ - migrations.CreateModel( - name='DataFlow', - fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('sdmx_id', models.CharField(help_text='Internal string used in SDMX communication to identify the DataFlow.', max_length=64, verbose_name='SDMX id')), - ('last_update', models.DateTimeField(help_text='The datetime at which the properties of this DataFlow were last updated.', verbose_name='Last updated')), - ('description', models.CharField(help_text='Natural language description of the DataFlow.', max_length=8192, verbose_name='DataFlow description')), - ('datasource_id', models.ForeignKey(help_text='The DataSource this object belongs to.', on_delete=django.db.models.deletion.RESTRICT, to='core.datasource')), - ], - ), - ] diff --git a/sophon/core/migrations/0005_alter_dataflow_last_update.py b/sophon/core/migrations/0005_alter_dataflow_last_update.py deleted file mode 100644 index aaeccb0..0000000 --- a/sophon/core/migrations/0005_alter_dataflow_last_update.py +++ /dev/null @@ -1,18 +0,0 @@ -# Generated by Django 3.2 on 2021-04-07 17:34 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('core', '0004_dataflow'), - ] - - operations = [ - migrations.AlterField( - model_name='dataflow', - name='last_update', - field=models.DateTimeField(auto_now=True, help_text='The datetime at which the properties of this DataFlow were last updated.', verbose_name='Last updated'), - ), - ] diff --git a/sophon/core/models.py b/sophon/core/models.py index eac84da..45b5e01 100644 --- a/sophon/core/models.py +++ b/sophon/core/models.py @@ -1,7 +1,10 @@ from django.db import models +from django.core import validators import pandas import pandasdmx import pandasdmx.message +import typing as t +import json class DataSource(models.Model): @@ -16,24 +19,203 @@ class DataSource(models.Model): method. """ - pandasdmx_id = models.CharField( + id = models.CharField( "PandaSDMX id", help_text="Internal id used by PandaSDMX to reference the source.", max_length=16, primary_key=True, ) + name = models.CharField( + "Name", + help_text="Full length name of the data source.", + max_length=512, + ) + + description = models.TextField( + "Description", + help_text="Long description of the data source.", + blank=True, + ) + + url = models.URLField( + "API URL", + help_text="The base URL of the SDMX endpoint of the data source." + ) + + documentation = models.URLField( + "Documentation URL", + help_text="Documentation URL of the data source.", + null=True, + ) + + data_content_type = models.CharField( + "API type", + help_text="The format in which the API returns its data.", + choices=[ + ("JSON", "JSON"), + ("XML", "XML"), + ], + default="XML", + max_length=16, + ) + + headers = models.JSONField( + "HTTP Headers", + help_text="HTTP headers to attach to every request, as a JSON object.", + default=dict, + ) + + resources = models.JSONField( + "Resources", + help_text="Unknown and undocumented JSON object.", + default=dict, + ) + + supports_agencyscheme = models.BooleanField( + "Supports AgencyScheme", + help_text='Whether the data source supports ' + '' + 'AgencyScheme ' + ' or not.', + default=True, + ) + + supports_categoryscheme = models.BooleanField( + "Supports CategoryScheme", + help_text='Whether the data source supports ' + '' + 'CategoryScheme ' + ' or not.', + default=True, + ) + + supports_codelist = models.BooleanField( + "Supports CodeList", + help_text='Whether the data source supports ' + '' + 'CodeList ' + ' or not.', + default=True, + ) + + supports_conceptscheme = models.BooleanField( + "Supports ConceptScheme", + help_text='Whether the data source supports ' + '' + 'ConceptScheme ' + ' or not.', + default=True, + ) + + supports_data = models.BooleanField( + "Supports DataSet", + help_text='Whether the data source supports ' + '' + 'DataSet ' + ' or not.', + default=True, + ) + + supports_dataflow = models.BooleanField( + "Supports DataflowDefinition", + help_text='Whether the data source supports ' + '' + 'DataflowDefinition ' + ' or not.', + default=True, + ) + + supports_datastructure = models.BooleanField( + "Supports DataStructureDefinition", + help_text='Whether the data source supports ' + '' + 'CategoryScheme ' + ' or not.', + default=True, + ) + + supports_provisionagreement = models.BooleanField( + "Supports ProvisionAgreement", + help_text='Whether the data source supports ' + '' + 'CategoryScheme ' + ' or not.', + default=True, + ) + + supports_preview = models.BooleanField( + "Supports previews", + help_text='Whether the data source supports ' + '' + 'previews of data ' + ' or not.', + default=False, + ) + + supports_structurespecific_data = models.BooleanField( + "Supports structure-specific data messages", + help_text='Whether the data source returns ' + '' + 'structure-specific data messages ' + ' or not.', + default=False, + ) + + def supports_dict(self) -> dict: + return { + "agencyscheme": self.supports_agencyscheme, + "categoryscheme": self.supports_categoryscheme, + "codelist": self.supports_codelist, + "conceptscheme": self.supports_conceptscheme, + "data": self.supports_data, + "dataflow": self.supports_dataflow, + "datastructure": self.supports_datastructure, + "provisionagreement": self.supports_provisionagreement, + "preview": self.supports_preview, + "structure-specific data": self.supports_structurespecific_data, + } + + def info_dict(self) -> dict: + return { + "id": self.id, + "name": self.name, + "data_content_type": self.data_content_type, + "url": self.url, + "documentation": self.documentation, + "supports": self.supports_dict(), + "headers": self.headers, + "resources": self.resources, + } + builtin = models.BooleanField( "Builtin", - help_text="Whether the source is builtin in PandaSDMX or not.", - ) - settings = models.JSONField( - "Settings", - help_text="Info parameter to pass to pandasdmx.add_source if the source is not builtin " - "(see https://pandasdmx.readthedocs.io/en/latest/api.html#pandasdmx.add_source).", - null=True + help_text="Whether the source is built-in in PandaSDMX or not.", ) + @classmethod + def create_from_sources_json(cls, file: t.TextIO): + j_sources: list = json.load(file) + + for j_source in j_sources: + + # Flatten supports + if supports := j_source.get("supports"): + del j_source["supports"] + for key, value in supports.items(): + if key == "structure-specific data": + j_source["supports_structurespecific_data"] = value + else: + j_source[f"supports_{key}"] = value + + cls.objects.update_or_create( + id=j_source["id"], + defaults={ + **j_source, + "builtin": True, + } + ) + def to_pandasdmx_source(self) -> pandasdmx.source.Source: """ Convert the :class:`.DataSource` to a :class:`pandasdmx.source.Source`\\ . @@ -42,7 +224,7 @@ class DataSource(models.Model): .. todo:: :func:`.to_pandasdmx` does not currently support non :attr:`.builtin` sources. """ - return pandasdmx.source.sources[self.pandasdmx_id] + return pandasdmx.source.sources[self.id] def to_pandasdmx_request(self) -> pandasdmx.Request: """ @@ -52,6 +234,12 @@ class DataSource(models.Model): """ return pandasdmx.Request(source=self.to_pandasdmx_source().id) + last_sync = models.DateTimeField( + "Last updated", + help_text="The datetime at which the data flows of this source were last syncronized.", + null=True, + ) + def request_flows(self) -> tuple[pandas.Series, pandas.Series]: """ Retrieve all available dataflows and datastructures as two :class:`pandas.Series`\\ . @@ -71,7 +259,7 @@ class DataSource(models.Model): return flows, structs def __str__(self): - return self.pandasdmx_id + return self.id class DataFlow(models.Model): @@ -81,30 +269,32 @@ class DataFlow(models.Model): See `this page `_ for more details. """ - datasource_id = models.ForeignKey( + surrogate_id = models.IntegerField( + "Surrogate id", + help_text="Internal id used by Django to identify this DataFlow.", + primary_key=True, + ) + + datasource = models.ForeignKey( DataSource, help_text="The DataSource this object belongs to.", on_delete=models.RESTRICT, ) - sdmx_id = models.CharField( + + id = models.CharField( "SDMX id", help_text="Internal string used in SDMX communication to identify the DataFlow.", max_length=64, ) - last_update = models.DateTimeField( - "Last updated", - help_text="The datetime at which the properties of this DataFlow were last updated.", - auto_now=True, - ) - description = models.CharField( - "DataFlow description", + description = models.TextField( + "Description", help_text="Natural language description of the DataFlow.", - max_length=8192, + blank=True, ) def __str__(self): - return f"{self.datasource_id} | {self.sdmx_id} | {self.description}" + return f"[{self.datasource}] {self.id}" class Project(models.Model): @@ -113,23 +303,31 @@ class Project(models.Model): hypothesis. """ + slug = models.SlugField( + "Slug", + help_text="Unique alphanumeric string which identifies the project.", + max_length=64, + primary_key=True, + ) + name = models.CharField( - "Project name", + "Name", help_text="The display name of the project.", max_length=512, ) - description = models.CharField( - "Project description", + + description = models.TextField( + "Description", help_text="A brief description of the project, to be displayed inthe overview.", - max_length=8192, + blank=True, ) - sources = models.ManyToManyField( - DataSource, - help_text="The sources used by this project.", + flows = models.ManyToManyField( + DataFlow, + help_text="The DataFlows used in this project.", related_name="used_in", blank=True, ) def __str__(self): - return self.name + return self.slug diff --git a/sophon/core/serializers.py b/sophon/core/serializers.py index 7e551b6..4279ec1 100644 --- a/sophon/core/serializers.py +++ b/sophon/core/serializers.py @@ -10,9 +10,25 @@ class DataSourceSerializer(serializers.ModelSerializer): class Meta: model = models.DataSource fields = [ - "pandasdmx_id", + "id", + "name", + "description", + "url", + "documentation", + "data_content_type", + "headers", + "supports_agencyscheme", + "supports_categoryscheme", + "supports_codelist", + "supports_conceptscheme", + "supports_data", + "supports_dataflow", + "supports_datastructure", + "supports_provisionagreement", + "supports_preview", + "supports_structurespecific_data", "builtin", - "settings", + "last_sync", ] @@ -24,10 +40,9 @@ class DataFlowSerializer(serializers.ModelSerializer): class Meta: model = models.DataFlow fields = [ + "surrogate_id", + "datasource", "id", - "datasource_id", - "sdmx_id", - "last_update", "description", ] @@ -40,8 +55,8 @@ class ProjectSerializer(serializers.ModelSerializer): class Meta: model = models.Project fields = [ - "id", + "slug", "name", "description", - "sources", + "flows", ]