💥 Basically rewrite all database parts from scratch

sophon/core/admin.py

@@ -15,8 +15,8 @@ class ProjectAdmin(CoreAdmin):
     """
 
     list_display = (
+        "slug",
         "name",
-        "description",
     )
 
 
@@ -27,8 +27,62 @@ class DataSourceAdmin(CoreAdmin):
     """
 
     list_display = (
-        "pandasdmx_id",
+        "id",
-        "builtin",
+        "name",
+        "data_content_type",
+        "last_sync",
+    )
+
+    fieldsets = (
+        (
+            None, {
+                "fields": (
+                    "id",
+                    "name",
+                    "description",
+                )
+            }
+        ),
+        (
+            "URLs", {
+                "fields": (
+                    "url",
+                    "documentation",
+                )
+            }
+        ),
+        (
+            "API configuration", {
+                "fields": (
+                    "data_content_type",
+                    "headers",
+                    "resources",
+                )
+            }
+        ),
+        (
+            "Features supported", {
+                "fields": (
+                    "supports_agencyscheme",
+                    "supports_categoryscheme",
+                    "supports_codelist",
+                    "supports_conceptscheme",
+                    "supports_data",
+                    "supports_dataflow",
+                    "supports_datastructure",
+                    "supports_provisionagreement",
+                    "supports_preview",
+                    "supports_structurespecific_data",
+                )
+            }
+        ),
+        (
+            "Syncronization", {
+                "fields": (
+                    "last_sync",
+                )
+            }
+        )
     )
 
 
@@ -39,7 +93,6 @@ class DataFlowAdmin(CoreAdmin):
     """
 
     list_display = (
-        "sdmx_id",
+        "datasource",
-        "datasource_id",
+        "id",
-        "description",
     )

sophon/core/migrations/0001_initial.py

@@ -1,39 +1,21 @@
-# Generated by Django 3.1.7 on 2021-04-04 23:41
+# Generated by Django 3.2 on 2021-04-08 14:36
-# Manually edited by @Steffo99
 
 from django.db import migrations, models
+import django.db.models.deletion
+import importlib.resources
+from .. import models as core_models
 
 
 def create_builtin_sources(apps, schema_editor):
     """
     Create in the database the sources that are already built-in in :mod:`pandasdmx`.
-
     This function is called when performing the migration: see
     `this page <https://docs.djangoproject.com/en/3.1/topics/migrations/#data-migrations>`_ for details on how this
     works!
     """
-    DataSource = apps.get_model("core", "DataSource")
+    file = importlib.resources.open_text("sophon.core.migrations", "0001_sources.json")
-    DataSource.objects.bulk_create([
+    core_models.DataSource.create_from_sources_json(file=file)
-        DataSource(pandasdmx_id="ABS", builtin=True),
+    file.close()
-        DataSource(pandasdmx_id="ESTAT", builtin=True),
-        DataSource(pandasdmx_id="ECB", builtin=True),
-        DataSource(pandasdmx_id="ILO", builtin=True),
-        DataSource(pandasdmx_id="IMF", builtin=True),
-        DataSource(pandasdmx_id="INEGI", builtin=True),
-        DataSource(pandasdmx_id="INSEE", builtin=True),
-        DataSource(pandasdmx_id="ISTAT", builtin=True),
-        DataSource(pandasdmx_id="LSD", builtin=True),
-        DataSource(pandasdmx_id="NB", builtin=True),
-        DataSource(pandasdmx_id="NBB", builtin=True),
-        DataSource(pandasdmx_id="OECD", builtin=True),
-        DataSource(pandasdmx_id="SGR", builtin=True),
-        DataSource(pandasdmx_id="SPC", builtin=True),
-        DataSource(pandasdmx_id="STAT_EE", builtin=True),
-        DataSource(pandasdmx_id="UNSD", builtin=True),
-        DataSource(pandasdmx_id="UNICEF", builtin=True),
-        DataSource(pandasdmx_id="WB", builtin=True),
-        DataSource(pandasdmx_id="WB_WDI", builtin=True),
-    ])
 
 
 class Migration(migrations.Migration):
@@ -44,22 +26,52 @@ class Migration(migrations.Migration):
     ]
 
     operations = [
+        migrations.CreateModel(
+            name='DataFlow',
+            fields=[
+                ('surrogate_id', models.IntegerField(help_text='Internal id used by Django to identify this DataFlow.', primary_key=True, serialize=False, verbose_name='Surrogate id')),
+                ('id', models.CharField(help_text='Internal string used in SDMX communication to identify the DataFlow.', max_length=64, verbose_name='SDMX id')),
+                ('description', models.TextField(blank=True, help_text='Natural language description of the DataFlow.', verbose_name='Description')),
+            ],
+        ),
         migrations.CreateModel(
             name='DataSource',
             fields=[
-                ('pandasdmx_id', models.CharField(max_length=16, primary_key=True, serialize=False, verbose_name='Internal pandasdmx source id')),
+                ('id', models.CharField(help_text='Internal id used by PandaSDMX to reference the source.', max_length=16, primary_key=True, serialize=False, verbose_name='PandaSDMX id')),
-                ('builtin', models.BooleanField(verbose_name='If the source is builtin in pandasdmx')),
+                ('name', models.CharField(help_text='Full length name of the data source.', max_length=512, verbose_name='Name')),
-                ('settings', models.JSONField(null=True, verbose_name='Source info to pass to pandasdmx if the source is not builtin')),
+                ('description', models.TextField(blank=True, help_text='Long description of the data source.', verbose_name='Description')),
+                ('url', models.URLField(help_text='The base URL of the SDMX endpoint of the data source.', verbose_name='API URL')),
+                ('documentation', models.URLField(help_text='Documentation URL of the data source.', null=True, verbose_name='Documentation URL')),
+                ('data_content_type', models.CharField(choices=[('JSON', 'JSON'), ('XML', 'XML')], default='XML', help_text='The format in which the API returns its data.', max_length=16, verbose_name='API type')),
+                ('headers', models.JSONField(default=dict, help_text='HTTP headers to attach to every request, as a JSON object.', verbose_name='HTTP Headers')),
+                ('resources', models.JSONField(default=dict, help_text='Unknown and undocumented JSON object.', verbose_name='Resources')),
+                ('supports_agencyscheme', models.BooleanField(default=True, help_text='Whether the data source supports <a href="https://pandasdmx.readthedocs.io/en/latest/api.html#pandasdmx.model.AgencyScheme">AgencyScheme </a> or not.', verbose_name='Supports AgencyScheme')),
+                ('supports_categoryscheme', models.BooleanField(default=True, help_text='Whether the data source supports <a href="https://pandasdmx.readthedocs.io/en/latest/api.html#pandasdmx.model.CategoryScheme">CategoryScheme </a> or not.', verbose_name='Supports CategoryScheme')),
+                ('supports_codelist', models.BooleanField(default=True, help_text='Whether the data source supports <a href="https://pandasdmx.readthedocs.io/en/latest/api.html#pandasdmx.model.CodeList">CodeList </a> or not.', verbose_name='Supports CodeList')),
+                ('supports_conceptscheme', models.BooleanField(default=True, help_text='Whether the data source supports <a href="https://pandasdmx.readthedocs.io/en/latest/api.html#pandasdmx.model.ConceptScheme">ConceptScheme </a> or not.', verbose_name='Supports ConceptScheme')),
+                ('supports_data', models.BooleanField(default=True, help_text='Whether the data source supports <a href="https://pandasdmx.readthedocs.io/en/latest/api.html#pandasdmx.model.DataSet">DataSet </a> or not.', verbose_name='Supports DataSet')),
+                ('supports_dataflow', models.BooleanField(default=True, help_text='Whether the data source supports <a href="https://pandasdmx.readthedocs.io/en/latest/api.html#pandasdmx.model.DataflowDefinition">DataflowDefinition </a> or not.', verbose_name='Supports DataflowDefinition')),
+                ('supports_datastructure', models.BooleanField(default=True, help_text='Whether the data source supports <a href="https://pandasdmx.readthedocs.io/en/latest/api.html#pandasdmx.model.DataStructureDefinition">CategoryScheme </a> or not.', verbose_name='Supports DataStructureDefinition')),
+                ('supports_provisionagreement', models.BooleanField(default=True, help_text='Whether the data source supports <a href="https://pandasdmx.readthedocs.io/en/latest/api.html#pandasdmx.model.ProvisionAgreement">CategoryScheme </a> or not.', verbose_name='Supports ProvisionAgreement')),
+                ('supports_preview', models.BooleanField(default=False, help_text='Whether the data source supports <a href="https://pandasdmx.readthedocs.io/en/latest/api.html#pandasdmx.Request.preview_data">previews of data </a> or not.', verbose_name='Supports previews')),
+                ('supports_structurespecific_data', models.BooleanField(default=False, help_text='Whether the data source returns <a href="https://pandasdmx.readthedocs.io/en/latest/api.html#pandasdmx.source.Source">structure-specific data messages </a> or not.', verbose_name='Supports structure-specific data messages')),
+                ('builtin', models.BooleanField(help_text='Whether the source is built-in in PandaSDMX or not.', verbose_name='Builtin')),
+                ('last_sync', models.DateTimeField(help_text='The datetime at which the data flows of this source were last syncronized.', null=True, verbose_name='Last updated')),
             ],
         ),
         migrations.CreateModel(
             name='Project',
             fields=[
-                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('slug', models.SlugField(help_text='Unique alphanumeric string which identifies the project.', max_length=64, primary_key=True, serialize=False, verbose_name='Slug')),
-                ('name', models.CharField(max_length=512, verbose_name='Project name')),
+                ('name', models.CharField(help_text='The display name of the project.', max_length=512, verbose_name='Name')),
-                ('description', models.CharField(max_length=8192, verbose_name='Project description')),
+                ('description', models.TextField(blank=True, help_text='A brief description of the project, to be displayed inthe overview.', verbose_name='Description')),
-                ('sources', models.ManyToManyField(related_name='used_in', to='core.DataSource')),
+                ('flows', models.ManyToManyField(blank=True, help_text='The DataFlows used in this project.', related_name='used_in', to='core.DataFlow')),
             ],
         ),
+        migrations.AddField(
+            model_name='dataflow',
+            name='datasource',
+            field=models.ForeignKey(help_text='The DataSource this object belongs to.', on_delete=django.db.models.deletion.RESTRICT, to='core.datasource'),
+        ),
         migrations.RunPython(create_builtin_sources)
     ]

sophon/core/migrations/0001_sources.json

@@ -0,0 +1,182 @@
+[
+    {
+        "id": "ABS",
+        "data_content_type": "JSON",
+        "url": "https://stat.data.abs.gov.au/sdmx-json",
+        "name": "Australian Bureau of Statistics",
+        "documentation": "https://www.abs.gov.au/"
+    },
+    {
+        "id": "ECB",
+        "resources": {
+            "data": {
+                "headers": {}
+            }
+        },
+        "url": "https://sdw-wsrest.ecb.europa.eu/service",
+        "name": "European Central Bank",
+        "documentation": "https://www.ecb.europa.eu/stats/ecb_statistics/co-operation_and_standards/sdmx/html/index.en.html",
+        "supports": {"preview": true}
+    },
+    {
+        "id": "ESTAT",
+        "documentation": "https://data.un.org/Host.aspx?Content=API",
+        "url": "https://ec.europa.eu/eurostat/SDMX/diss-web/rest",
+        "name": "Eurostat",
+        "supports": {
+            "agencyscheme": false,
+            "categoryscheme": false,
+            "codelist": false,
+            "conceptscheme": false,
+            "provisionagreement": false
+        }
+    },
+    {
+        "id": "ILO",
+        "name": "International Labor Organization",
+        "documentation": "https://www.ilo.org/ilostat/",
+        "url": "https://www.ilo.org/sdmx/rest",
+        "headers": {
+            "accept": "application/vnd.sdmx.structurespecificdata+xml;version=2.1"
+        },
+        "supports": {
+            "provisionagreement": false
+        }
+    },
+    {
+        "id": "IMF",
+        "url": "https://sdmxcentral.imf.org/ws/public/sdmxapi/rest",
+        "name": "International Monetary Fund",
+        "supports": {
+            "provisionagreement": false
+        }
+    },
+    {
+        "id": "INEGI",
+        "url": "https://sdmx.snieg.mx/service/rest",
+        "name": "Instituto Nacional de Estadística y Geografía (MX)",
+        "documentation": "https://sdmx.snieg.mx/infrastructure",
+        "supports": {
+            "agencyscheme": false,
+            "provisionagreement": false,
+            "structure-specific data": true
+        }
+    },
+    {
+        "id": "INSEE",
+        "name": "Institut national de la statistique et des études économiques (FR)",
+        "documentation": "https://www.bdm.insee.fr/bdm2/statique?page=sdmx",
+        "url": "https://www.bdm.insee.fr/series/sdmx",
+        "headers": {
+            "data": {
+                "Accept": "application/vnd.sdmx.genericdata+xml;version=2.1"
+            }
+        },
+        "supports": {"provisionagreement": false}
+    },
+    {
+        "id": "ISTAT",
+        "name": "Instituto Nationale di Statistica (IT)",
+        "documentation": "https://ec.europa.eu/eurostat/web/sdmx-web-services/rest-sdmx-2.1",
+        "url": "http://sdmx.istat.it/SDMXWS/rest",
+        "supports": {
+            "provisionagreement": false,
+            "structure-specific data": true
+        }
+    },
+    {
+        "id": "OECD",
+        "data_content_type": "JSON",
+        "url": "https://stats.oecd.org/SDMX-JSON",
+        "documentation": "https://stats.oecd.org/SDMX-JSON/",
+        "name": "Organisation for Economic Co-operation and Development"
+    },
+    {
+        "id": "NBB",
+        "data_content_type": "JSON",
+        "documentation": "https://www.nbb.be/doc/dq/migratie_belgostat/en/nbb_stat-technical-manual.pdf",
+        "url": "https://stat.nbb.be/sdmx-json",
+        "name": "National Bank of Belgium"
+    },
+    {
+        "id": "NB",
+        "name": "Norges Bank (NO)",
+        "documentation": "https://www.norges-bank.no/en/topics/Statistics/open-data/",
+        "url": "https://data.norges-bank.no/api",
+        "supports": {"categoryscheme": false, "structure-specific data": true},
+        "headers": {
+            "data": {
+                "accept": "application/vnd.sdmx.genericdata+xml;version=2.1"
+            }
+        }
+    },
+    {
+        "id": "SGR",
+        "url": "https://registry.sdmx.org/ws/rest",
+        "name": "SDMX Global Registry",
+        "documentation": "https://registry.sdmx.org/ws/rest"
+    },
+    {
+        "id": "UNICEF",
+        "name": "UN International Children's Emergency Fund",
+        "documentation": "https://data.unicef.org/",
+        "url": "https://sdmx.data.unicef.org/ws/public/sdmxapi/rest",
+        "headers": {
+            "accept": "application/vnd.sdmx.structure+xml;version=2.1"
+        }
+    },
+    {
+        "id": "SPC",
+        "name": "Pacific Data Hub",
+        "documentation":"https://stats.pacificdata.org/?locale=en",
+        "url": "https://stats-nsi-stable.pacificdata.org/rest",
+        "supports": {"preview": false, "provisionagreement": false}
+    },
+
+    {
+        "id": "UNSD",
+        "name": "United Nations Statistics Division",
+        "documentation": "https://unstats.un.org/home/",
+        "url": "https://data.un.org/WS/rest",
+        "supports": {"preview": true, "provisionagreement": false}
+    },
+    {
+        "id": "WB",
+        "name": "World Bank World Integrated Trade Solution",
+        "documentation": "http://wits.worldbank.org",
+        "url": "http://wits.worldbank.org/API/V1/SDMX/V21/rest",
+        "supports": {
+            "agencyscheme": false,
+            "provisionagreement": false
+        }
+    },
+    {
+        "id": "WB_WDI",
+        "name": "World Bank World Development Indicators",
+        "documentation": "https://datahelpdesk.worldbank.org/knowledgebase/articles/1886701-sdmx-api-queries",
+        "url": "http://api.worldbank.org/v2/sdmx/rest",
+        "supports": {
+            "provisionagreement": false,
+            "structure-specific data": true
+        }
+    },
+    {
+        "id": "LSD",
+        "documentation": "https://osp.stat.gov.lt/rdb-rest",
+        "url": "https://osp-rs.stat.gov.lt/rest_xml",
+        "name": "Statistics Lithuania",
+        "supports": {
+            "categoryscheme": false,
+            "codelist": false,
+            "conceptscheme": false,
+            "provisionagreement": false
+        }
+    },
+    {
+        "id": "STAT_EE",
+        "data_content_type": "JSON",
+        "documentation": "https://www.stat.ee/sites/default/files/2020-09/API-instructions.pdf",
+        "url": "http://andmebaas.stat.ee/sdmx-json",
+        "name": "Statistics Estonia"
+    }
+]

sophon/core/migrations/0002_auto_20210405_1641.py

@@ -1,43 +0,0 @@
-# Generated by Django 3.1.7 on 2021-04-05 16:41
-
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('core', '0001_initial'),
-    ]
-
-    operations = [
-        migrations.AlterField(
-            model_name='datasource',
-            name='builtin',
-            field=models.BooleanField(help_text='Whether the source is builtin in PandaSDMX or not.', verbose_name='Builtin'),
-        ),
-        migrations.AlterField(
-            model_name='datasource',
-            name='pandasdmx_id',
-            field=models.CharField(help_text='Internal id used by PandaSDMX to reference the source.', max_length=16, primary_key=True, serialize=False, verbose_name='PandaSDMX id'),
-        ),
-        migrations.AlterField(
-            model_name='datasource',
-            name='settings',
-            field=models.JSONField(help_text='Info parameter to pass to pandasdmx.add_source if the source is not builtin (see https://pandasdmx.readthedocs.io/en/latest/api.html#pandasdmx.add_source).', null=True, verbose_name='Settings'),
-        ),
-        migrations.AlterField(
-            model_name='project',
-            name='description',
-            field=models.CharField(help_text='A brief description of the project, to be displayed inthe overview.', max_length=8192, verbose_name='Project description'),
-        ),
-        migrations.AlterField(
-            model_name='project',
-            name='name',
-            field=models.CharField(help_text='The display name of the project.', max_length=512, verbose_name='Project name'),
-        ),
-        migrations.AlterField(
-            model_name='project',
-            name='sources',
-            field=models.ManyToManyField(help_text='The sources used by this project.', null=True, related_name='used_in', to='core.DataSource'),
-        ),
-    ]

sophon/core/migrations/0003_auto_20210406_0044.py

@@ -1,18 +0,0 @@
-# Generated by Django 3.1.7 on 2021-04-06 00:44
-
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('core', '0002_auto_20210405_1641'),
-    ]
-
-    operations = [
-        migrations.AlterField(
-            model_name='project',
-            name='sources',
-            field=models.ManyToManyField(blank=True, help_text='The sources used by this project.', related_name='used_in', to='core.DataSource'),
-        ),
-    ]

sophon/core/migrations/0004_dataflow.py

@@ -1,24 +0,0 @@
-# Generated by Django 3.1.7 on 2021-04-06 20:55
-
-from django.db import migrations, models
-import django.db.models.deletion
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('core', '0003_auto_20210406_0044'),
-    ]
-
-    operations = [
-        migrations.CreateModel(
-            name='DataFlow',
-            fields=[
-                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
-                ('sdmx_id', models.CharField(help_text='Internal string used in SDMX communication to identify the DataFlow.', max_length=64, verbose_name='SDMX id')),
-                ('last_update', models.DateTimeField(help_text='The datetime at which the properties of this DataFlow were last updated.', verbose_name='Last updated')),
-                ('description', models.CharField(help_text='Natural language description of the DataFlow.', max_length=8192, verbose_name='DataFlow description')),
-                ('datasource_id', models.ForeignKey(help_text='The DataSource this object belongs to.', on_delete=django.db.models.deletion.RESTRICT, to='core.datasource')),
-            ],
-        ),
-    ]

sophon/core/migrations/0005_alter_dataflow_last_update.py

@@ -1,18 +0,0 @@
-# Generated by Django 3.2 on 2021-04-07 17:34
-
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('core', '0004_dataflow'),
-    ]
-
-    operations = [
-        migrations.AlterField(
-            model_name='dataflow',
-            name='last_update',
-            field=models.DateTimeField(auto_now=True, help_text='The datetime at which the properties of this DataFlow were last updated.', verbose_name='Last updated'),
-        ),
-    ]

sophon/core/models.py

@@ -1,7 +1,10 @@
 from django.db import models
+from django.core import validators
 import pandas
 import pandasdmx
 import pandasdmx.message
+import typing as t
+import json
 
 
 class DataSource(models.Model):
@@ -16,22 +19,201 @@ class DataSource(models.Model):
     method.
     """
 
-    pandasdmx_id = models.CharField(
+    id = models.CharField(
         "PandaSDMX id",
         help_text="Internal id used by PandaSDMX to reference the source.",
         max_length=16,
         primary_key=True,
     )
 
+    name = models.CharField(
+        "Name",
+        help_text="Full length name of the data source.",
+        max_length=512,
+    )
+
+    description = models.TextField(
+        "Description",
+        help_text="Long description of the data source.",
+        blank=True,
+    )
+
+    url = models.URLField(
+        "API URL",
+        help_text="The base URL of the SDMX endpoint of the data source."
+    )
+
+    documentation = models.URLField(
+        "Documentation URL",
+        help_text="Documentation URL of the data source.",
+        null=True,
+    )
+
+    data_content_type = models.CharField(
+        "API type",
+        help_text="The format in which the API returns its data.",
+        choices=[
+            ("JSON", "JSON"),
+            ("XML", "XML"),
+        ],
+        default="XML",
+        max_length=16,
+    )
+
+    headers = models.JSONField(
+        "HTTP Headers",
+        help_text="HTTP headers to attach to every request, as a JSON object.",
+        default=dict,
+    )
+
+    resources = models.JSONField(
+        "Resources",
+        help_text="Unknown and undocumented JSON object.",
+        default=dict,
+    )
+
+    supports_agencyscheme = models.BooleanField(
+        "Supports AgencyScheme",
+        help_text='Whether the data source supports '
+                  '<a href="https://pandasdmx.readthedocs.io/en/latest/api.html#pandasdmx.model.AgencyScheme">'
+                  'AgencyScheme '
+                  '</a> or not.',
+        default=True,
+    )
+
+    supports_categoryscheme = models.BooleanField(
+        "Supports CategoryScheme",
+        help_text='Whether the data source supports '
+                  '<a href="https://pandasdmx.readthedocs.io/en/latest/api.html#pandasdmx.model.CategoryScheme">'
+                  'CategoryScheme '
+                  '</a> or not.',
+        default=True,
+    )
+
+    supports_codelist = models.BooleanField(
+        "Supports CodeList",
+        help_text='Whether the data source supports '
+                  '<a href="https://pandasdmx.readthedocs.io/en/latest/api.html#pandasdmx.model.CodeList">'
+                  'CodeList '
+                  '</a> or not.',
+        default=True,
+    )
+
+    supports_conceptscheme = models.BooleanField(
+        "Supports ConceptScheme",
+        help_text='Whether the data source supports '
+                  '<a href="https://pandasdmx.readthedocs.io/en/latest/api.html#pandasdmx.model.ConceptScheme">'
+                  'ConceptScheme '
+                  '</a> or not.',
+        default=True,
+    )
+
+    supports_data = models.BooleanField(
+        "Supports DataSet",
+        help_text='Whether the data source supports '
+                  '<a href="https://pandasdmx.readthedocs.io/en/latest/api.html#pandasdmx.model.DataSet">'
+                  'DataSet '
+                  '</a> or not.',
+        default=True,
+    )
+
+    supports_dataflow = models.BooleanField(
+        "Supports DataflowDefinition",
+        help_text='Whether the data source supports '
+                  '<a href="https://pandasdmx.readthedocs.io/en/latest/api.html#pandasdmx.model.DataflowDefinition">'
+                  'DataflowDefinition '
+                  '</a> or not.',
+        default=True,
+    )
+
+    supports_datastructure = models.BooleanField(
+        "Supports DataStructureDefinition",
+        help_text='Whether the data source supports '
+                  '<a href="https://pandasdmx.readthedocs.io/en/latest/api.html#pandasdmx.model.DataStructureDefinition">'
+                  'CategoryScheme '
+                  '</a> or not.',
+        default=True,
+    )
+
+    supports_provisionagreement = models.BooleanField(
+        "Supports ProvisionAgreement",
+        help_text='Whether the data source supports '
+                  '<a href="https://pandasdmx.readthedocs.io/en/latest/api.html#pandasdmx.model.ProvisionAgreement">'
+                  'CategoryScheme '
+                  '</a> or not.',
+        default=True,
+    )
+
+    supports_preview = models.BooleanField(
+        "Supports previews",
+        help_text='Whether the data source supports '
+                  '<a href="https://pandasdmx.readthedocs.io/en/latest/api.html#pandasdmx.Request.preview_data">'
+                  'previews of data '
+                  '</a> or not.',
+        default=False,
+    )
+
+    supports_structurespecific_data = models.BooleanField(
+        "Supports structure-specific data messages",
+        help_text='Whether the data source returns '
+                  '<a href="https://pandasdmx.readthedocs.io/en/latest/api.html#pandasdmx.source.Source">'
+                  'structure-specific data messages '
+                  '</a> or not.',
+        default=False,
+    )
+
+    def supports_dict(self) -> dict:
+        return {
+            "agencyscheme": self.supports_agencyscheme,
+            "categoryscheme": self.supports_categoryscheme,
+            "codelist": self.supports_codelist,
+            "conceptscheme": self.supports_conceptscheme,
+            "data": self.supports_data,
+            "dataflow": self.supports_dataflow,
+            "datastructure": self.supports_datastructure,
+            "provisionagreement": self.supports_provisionagreement,
+            "preview": self.supports_preview,
+            "structure-specific data": self.supports_structurespecific_data,
+        }
+
+    def info_dict(self) -> dict:
+        return {
+            "id": self.id,
+            "name": self.name,
+            "data_content_type": self.data_content_type,
+            "url": self.url,
+            "documentation": self.documentation,
+            "supports": self.supports_dict(),
+            "headers": self.headers,
+            "resources": self.resources,
+        }
+
     builtin = models.BooleanField(
         "Builtin",
-        help_text="Whether the source is builtin in PandaSDMX or not.",
+        help_text="Whether the source is built-in in PandaSDMX or not.",
     )
-    settings = models.JSONField(
+
-        "Settings",
+    @classmethod
-        help_text="Info parameter to pass to pandasdmx.add_source if the source is not builtin "
+    def create_from_sources_json(cls, file: t.TextIO):
-                  "(see https://pandasdmx.readthedocs.io/en/latest/api.html#pandasdmx.add_source).",
+        j_sources: list = json.load(file)
-        null=True
+
+        for j_source in j_sources:
+
+            # Flatten supports
+            if supports := j_source.get("supports"):
+                del j_source["supports"]
+                for key, value in supports.items():
+                    if key == "structure-specific data":
+                        j_source["supports_structurespecific_data"] = value
+                    else:
+                        j_source[f"supports_{key}"] = value
+
+            cls.objects.update_or_create(
+                id=j_source["id"],
+                defaults={
+                    **j_source,
+                    "builtin": True,
+                }
             )
 
     def to_pandasdmx_source(self) -> pandasdmx.source.Source:
@@ -42,7 +224,7 @@ class DataSource(models.Model):
 
         .. todo:: :func:`.to_pandasdmx` does not currently support non :attr:`.builtin` sources.
         """
-        return pandasdmx.source.sources[self.pandasdmx_id]
+        return pandasdmx.source.sources[self.id]
 
     def to_pandasdmx_request(self) -> pandasdmx.Request:
         """
@@ -52,6 +234,12 @@ class DataSource(models.Model):
         """
         return pandasdmx.Request(source=self.to_pandasdmx_source().id)
 
+    last_sync = models.DateTimeField(
+        "Last updated",
+        help_text="The datetime at which the data flows of this source were last syncronized.",
+        null=True,
+    )
+
     def request_flows(self) -> tuple[pandas.Series, pandas.Series]:
         """
         Retrieve all available dataflows and datastructures as two :class:`pandas.Series`\\ .
@@ -71,7 +259,7 @@ class DataSource(models.Model):
         return flows, structs
 
     def __str__(self):
-        return self.pandasdmx_id
+        return self.id
 
 
 class DataFlow(models.Model):
@@ -81,30 +269,32 @@ class DataFlow(models.Model):
     See `this page <https://ec.europa.eu/eurostat/online-help/redisstat-admin/en/TECH_A_main/>`_ for more details.
     """
 
-    datasource_id = models.ForeignKey(
+    surrogate_id = models.IntegerField(
+        "Surrogate id",
+        help_text="Internal id used by Django to identify this DataFlow.",
+        primary_key=True,
+    )
+
+    datasource = models.ForeignKey(
         DataSource,
         help_text="The DataSource this object belongs to.",
         on_delete=models.RESTRICT,
     )
-    sdmx_id = models.CharField(
+
+    id = models.CharField(
         "SDMX id",
         help_text="Internal string used in SDMX communication to identify the DataFlow.",
         max_length=64,
     )
-    last_update = models.DateTimeField(
-        "Last updated",
-        help_text="The datetime at which the properties of this DataFlow were last updated.",
-        auto_now=True,
-    )
 
-    description = models.CharField(
+    description = models.TextField(
-        "DataFlow description",
+        "Description",
         help_text="Natural language description of the DataFlow.",
-        max_length=8192,
+        blank=True,
     )
 
     def __str__(self):
-        return f"{self.datasource_id} | {self.sdmx_id} | {self.description}"
+        return f"[{self.datasource}] {self.id}"
 
 
 class Project(models.Model):
@@ -113,23 +303,31 @@ class Project(models.Model):
     hypothesis.
     """
 
+    slug = models.SlugField(
+        "Slug",
+        help_text="Unique alphanumeric string which identifies the project.",
+        max_length=64,
+        primary_key=True,
+    )
+
     name = models.CharField(
-        "Project name",
+        "Name",
         help_text="The display name of the project.",
         max_length=512,
     )
-    description = models.CharField(
+
-        "Project description",
+    description = models.TextField(
+        "Description",
         help_text="A brief description of the project, to be displayed inthe overview.",
-        max_length=8192,
+        blank=True,
     )
 
-    sources = models.ManyToManyField(
+    flows = models.ManyToManyField(
-        DataSource,
+        DataFlow,
-        help_text="The sources used by this project.",
+        help_text="The DataFlows used in this project.",
         related_name="used_in",
         blank=True,
     )
 
     def __str__(self):
-        return self.name
+        return self.slug

sophon/core/serializers.py

@@ -10,9 +10,25 @@ class DataSourceSerializer(serializers.ModelSerializer):
     class Meta:
         model = models.DataSource
         fields = [
-            "pandasdmx_id",
+            "id",
+            "name",
+            "description",
+            "url",
+            "documentation",
+            "data_content_type",
+            "headers",
+            "supports_agencyscheme",
+            "supports_categoryscheme",
+            "supports_codelist",
+            "supports_conceptscheme",
+            "supports_data",
+            "supports_dataflow",
+            "supports_datastructure",
+            "supports_provisionagreement",
+            "supports_preview",
+            "supports_structurespecific_data",
             "builtin",
-            "settings",
+            "last_sync",
         ]
 
 
@@ -24,10 +40,9 @@ class DataFlowSerializer(serializers.ModelSerializer):
     class Meta:
         model = models.DataFlow
         fields = [
+            "surrogate_id",
+            "datasource",
             "id",
-            "datasource_id",
-            "sdmx_id",
-            "last_update",
             "description",
         ]
 
@@ -40,8 +55,8 @@ class ProjectSerializer(serializers.ModelSerializer):
     class Meta:
         model = models.Project
         fields = [
-            "id",
+            "slug",
             "name",
             "description",
-            "sources",
+            "flows",
         ]