diff --git a/backend/sophon/core/models.py b/backend/sophon/core/models.py index 04c961e..82febd9 100644 --- a/backend/sophon/core/models.py +++ b/backend/sophon/core/models.py @@ -7,6 +7,10 @@ import pandasdmx.message import typing as t import json import abc +import datetime +import logging + +log = logging.getLogger(__name__) class DataSource(models.Model): @@ -260,6 +264,33 @@ class DataSource(models.Model): structs = data["structure"] return flows, structs + def sync_flows(self) -> None: + """ + Create :class:`.DataFlow` objects for every dataflow returned by :meth:`.request_flows`, and update the ones + that already exist. + + .. warning:: This function does not delete any :class:`.DataFlow`, even if it doesn't exist anymore! + """ + + log.debug(f"Requesting dataflows of {self!r}...") + flows, structs = self.request_flows() + + log.info(f"Syncing DataFlows of {self!r}...") + for description, sdmx_id in zip(flows, flows.index): + + db_flow = DataFlow.objects.update_or_create( + **{ + "datasource": self, + "id": sdmx_id, + }, + defaults={ + "last_update": datetime.datetime.now(), + "description": description, + } + ) + db_flow.save() + log.info(f"Synced {db_flow}!") + def __str__(self): return self.id diff --git a/backend/sophon/core/views.py b/backend/sophon/core/views.py index ece35b4..6b0eaa7 100644 --- a/backend/sophon/core/views.py +++ b/backend/sophon/core/views.py @@ -67,31 +67,8 @@ class DataSourceViewSet(viewsets.ModelViewSet): log.debug(f"Getting DataSource from the database...") db_datasource: models.DataSource = self.get_object() - - log.debug(f"Requesting dataflows of {db_datasource!r}...") - flows, structs = db_datasource.request_flows() - - log.info(f"Syncing DataFlows of {db_datasource!r}...") - for description, sdmx_id in zip(flows, flows.index): - - log.debug(f"Searching in the database for: {db_datasource!r} | {sdmx_id!r}") - try: - db_flow = models.DataFlow.objects.get(datasource_id=db_datasource, sdmx_id=sdmx_id) - except models.DataFlow.DoesNotExist: - db_flow = models.DataFlow( - datasource_id=db_datasource, - sdmx_id=sdmx_id, - last_update=datetime.now(), - description=description, - ) - log.info(f"Created new DataFlow: {db_flow!r}") - else: - db_flow.last_update = datetime.now() - db_flow.description = description - log.debug(f"Updated DataFlow: {db_flow!r}") - - db_flow.save() + db_datasource.sync_flows() return response.Response({ - "updated": len(flows) + "success": True, })