From 93f28b9dad6d3c0d06f1fea5d934513b5a168f24 Mon Sep 17 00:00:00 2001
From: Stefano Pigozzi <me@steffo.eu>
Date: Sun, 18 Apr 2021 20:01:08 +0200
Subject: [PATCH] =?UTF-8?q?=E2=9C=A8=20Add=20sync=5Fflows=20to=20the=20adm?=
 =?UTF-8?q?in=20page?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 backend/sophon/core/admin.py                  | 27 ++++++++++++++++++-
 .../0003_rename_id_dataflow_sdmx_id.py        | 18 +++++++++++++
 .../0004_alter_dataflow_surrogate_id.py       | 18 +++++++++++++
 backend/sophon/core/models.py                 | 19 ++++++++-----
 backend/sophon/core/serializers.py            |  2 +-
 5 files changed, 75 insertions(+), 9 deletions(-)
 create mode 100644 backend/sophon/core/migrations/0003_rename_id_dataflow_sdmx_id.py
 create mode 100644 backend/sophon/core/migrations/0004_alter_dataflow_surrogate_id.py

diff --git a/backend/sophon/core/admin.py b/backend/sophon/core/admin.py
index 645dcf5..d48f6f1 100644
--- a/backend/sophon/core/admin.py
+++ b/backend/sophon/core/admin.py
@@ -19,6 +19,17 @@ class ProjectAdmin(CoreAdmin):
         "name",
     )
 
+    ordering = (
+        "slug",
+    )
+
+
+@admin.action(description="Syncronize DataFlows (slow, be patient!)")
+def sync_flows_admin(modeladmin, request, queryset):
+    for datasource in queryset:
+        datasource: models.DataSource
+        datasource.sync_flows()
+
 
 @admin.register(models.DataSource)
 class DataSourceAdmin(CoreAdmin):
@@ -33,6 +44,14 @@ class DataSourceAdmin(CoreAdmin):
         "last_sync",
     )
 
+    ordering = (
+        "last_sync",
+    )
+
+    actions = (
+        sync_flows_admin,
+    )
+
     fieldsets = (
         (
             None, {
@@ -94,5 +113,11 @@ class DataFlowAdmin(CoreAdmin):
 
     list_display = (
         "datasource",
-        "id",
+        "sdmx_id",
+        "description",
+    )
+
+    ordering = (
+        "datasource",
+        "sdmx_id",
     )
diff --git a/backend/sophon/core/migrations/0003_rename_id_dataflow_sdmx_id.py b/backend/sophon/core/migrations/0003_rename_id_dataflow_sdmx_id.py
new file mode 100644
index 0000000..78e0a82
--- /dev/null
+++ b/backend/sophon/core/migrations/0003_rename_id_dataflow_sdmx_id.py
@@ -0,0 +1,18 @@
+# Generated by Django 3.2 on 2021-04-18 17:51
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0002_auto_20210415_1453'),
+    ]
+
+    operations = [
+        migrations.RenameField(
+            model_name='dataflow',
+            old_name='id',
+            new_name='sdmx_id',
+        ),
+    ]
diff --git a/backend/sophon/core/migrations/0004_alter_dataflow_surrogate_id.py b/backend/sophon/core/migrations/0004_alter_dataflow_surrogate_id.py
new file mode 100644
index 0000000..1d815fe
--- /dev/null
+++ b/backend/sophon/core/migrations/0004_alter_dataflow_surrogate_id.py
@@ -0,0 +1,18 @@
+# Generated by Django 3.2 on 2021-04-18 17:52
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0003_rename_id_dataflow_sdmx_id'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='dataflow',
+            name='surrogate_id',
+            field=models.BigAutoField(help_text='Internal id used by Django to identify this DataFlow.', primary_key=True, serialize=False, verbose_name='Surrogate id'),
+        ),
+    ]
diff --git a/backend/sophon/core/models.py b/backend/sophon/core/models.py
index 82febd9..a5bdd11 100644
--- a/backend/sophon/core/models.py
+++ b/backend/sophon/core/models.py
@@ -9,6 +9,7 @@ import json
 import abc
 import datetime
 import logging
+import pytz
 
 log = logging.getLogger(__name__)
 
@@ -278,18 +279,22 @@ class DataSource(models.Model):
         log.info(f"Syncing DataFlows of {self!r}...")
         for description, sdmx_id in zip(flows, flows.index):
 
-            db_flow = DataFlow.objects.update_or_create(
+            db_flow, _created = DataFlow.objects.update_or_create(
                 **{
                     "datasource": self,
-                    "id": sdmx_id,
+                    "sdmx_id": sdmx_id,
                 },
                 defaults={
-                    "last_update": datetime.datetime.now(),
                     "description": description,
                 }
             )
             db_flow.save()
-            log.info(f"Synced {db_flow}!")
+            log.debug(f"Synced {db_flow}!")
+
+        log.debug(f"Updating last_sync value of {self!r}")
+        self.last_sync = datetime.datetime.now()
+        self.save()
+        log.info(f"Finished syncing DataFlows of {self!r}")
 
     def __str__(self):
         return self.id
@@ -302,7 +307,7 @@ class DataFlow(models.Model):
     See `this page <https://ec.europa.eu/eurostat/online-help/redisstat-admin/en/TECH_A_main/>`_ for more details.
     """
 
-    surrogate_id = models.IntegerField(
+    surrogate_id = models.BigAutoField(
         "Surrogate id",
         help_text="Internal id used by Django to identify this DataFlow.",
         primary_key=True,
@@ -314,7 +319,7 @@ class DataFlow(models.Model):
         on_delete=models.RESTRICT,
     )
 
-    id = models.CharField(
+    sdmx_id = models.CharField(
         "SDMX id",
         help_text="Internal string used in SDMX communication to identify the DataFlow.",
         max_length=64,
@@ -327,7 +332,7 @@ class DataFlow(models.Model):
     )
 
     def __str__(self):
-        return f"[{self.datasource}] {self.id}"
+        return f"[{self.datasource}] {self.sdmx_id}"
 
 
 class Project(models.Model):
diff --git a/backend/sophon/core/serializers.py b/backend/sophon/core/serializers.py
index e9181c8..855da5f 100644
--- a/backend/sophon/core/serializers.py
+++ b/backend/sophon/core/serializers.py
@@ -46,7 +46,7 @@ class DataFlowSerializer(serializers.ModelSerializer):
         fields = [
             "surrogate_id",
             "datasource",
-            "id",
+            "sdmx_id",
             "description",
         ]
         read_only_fields = [