From 650d4311ff443d52c1eb78e7ff3738557799c9df Mon Sep 17 00:00:00 2001 From: Pahaz White Date: Thu, 18 Apr 2019 18:32:33 +0300 Subject: [PATCH 1/6] lib.integra: support flash() on model loading finished --- lib/integra/tasks.py | 28 ++++++++++++++++------------ lib/integra/tests/test_loader.py | 13 +++++++------ lib/integra/utils.py | 16 +++++++--------- 3 files changed, 30 insertions(+), 27 deletions(-) diff --git a/lib/integra/tasks.py b/lib/integra/tasks.py index 66017936..ef993e12 100644 --- a/lib/integra/tasks.py +++ b/lib/integra/tasks.py @@ -9,23 +9,27 @@ class Integra: def __init__(self, config): + self.models = config['models'] self.loader = Loader(config) self.updater = Updater() def run(self): count = 0 - has_exception = False - for obj in self.loader.download(): - try: - status = self.updater.update(obj) - count += 1 if status else 0 - except Exception as exc: # noqa - app, model, data = obj['app'], obj['model'], obj['data'] - LOGGER.exception("integra error: %r; app=%s model=%s data=%r", - exc, app, model, data) - has_exception = True - if not has_exception: - self.updater.flush_updates() + for model in self.models: + has_exception = False + for obj in self.loader.download(model): + try: + status = self.updater.update(obj) + count += 1 if status else 0 + except Exception as exc: # noqa + app, model, data = obj['app'], obj['model'], obj['data'] + LOGGER.exception( + "integra error: %r; app=%s model=%s data=%r", + exc, app, model, data) + has_exception = True + if not has_exception: + self.updater.flush_updates() + self.updater.clear_updates() return count diff --git a/lib/integra/tests/test_loader.py b/lib/integra/tests/test_loader.py index 5293bcce..fad9786a 100644 --- a/lib/integra/tests/test_loader.py +++ b/lib/integra/tests/test_loader.py @@ -2,18 +2,19 @@ def test_loader_protocol(mocker): + model = { + 'url': '/api/v1/contact-list/', + 'app': 'housing', + 'model': 'contact'} config = { 'base_url': 'https://housing.pik-software.ru/', 'request': {}, - 'models': [{ - 'url': '/api/v1/contact-list/', - 'app': 'housing', - 'model': 'contact'}]} + 'models': [model]} result = [{'app': 'housing', 'model': 'contact', 'data': {'_uid': '0'}}] loader = Loader(config) with mocker.patch.object(loader, '_request', return_value=result): - downloaded = list(loader.download()) + downloaded = list(loader.download(model)) assert downloaded == result - loader._request.assert_called_once_with(config['models'][0], None) # noqa: pylint=protected-access + loader._request.assert_called_once_with(model, None) # noqa: pylint=protected-access diff --git a/lib/integra/utils.py b/lib/integra/utils.py index 974a504c..46c940b8 100644 --- a/lib/integra/utils.py +++ b/lib/integra/utils.py @@ -15,7 +15,7 @@ class Loader: l = Loader({ 'base_url': 'https://housing.pik-software.ru/', 'request': { - + 'auth': 'login:password', }, 'models': [ {'url': '/api/v1/contact-list/', @@ -27,14 +27,12 @@ class Loader: def __init__(self, config): self.url = config['base_url'] self.request = config['request'] - self.models = config['models'] - - def download(self): - for model in self.models: - key = f'{model["app"]}:{model["model"]}' - updated = UpdateState.objects.get_last_updated(key) - for data in self._request(model, updated): - yield data + + def download(self, model): + key = f'{model["app"]}:{model["model"]}' + updated = UpdateState.objects.get_last_updated(key) + for data in self._request(model, updated): + yield data def _request(self, model, updated=None): app_name, model_name = model["app"], model["model"] From ef4889b1c885c3b02524bd3e8f4909557c5949bc Mon Sep 17 00:00:00 2001 From: Pahaz White Date: Thu, 18 Apr 2019 18:38:46 +0300 Subject: [PATCH 2/6] lib.integra: add start loading info message --- lib/integra/tasks.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/lib/integra/tasks.py b/lib/integra/tasks.py index ef993e12..4b7b889f 100644 --- a/lib/integra/tasks.py +++ b/lib/integra/tasks.py @@ -17,15 +17,17 @@ def run(self): count = 0 for model in self.models: has_exception = False + LOGGER.info( + "integra: loading app=%s model=%s", + model['app'], model['model']) for obj in self.loader.download(model): try: status = self.updater.update(obj) count += 1 if status else 0 except Exception as exc: # noqa - app, model, data = obj['app'], obj['model'], obj['data'] LOGGER.exception( "integra error: %r; app=%s model=%s data=%r", - exc, app, model, data) + exc, obj['app'], obj['model'], obj['data']) has_exception = True if not has_exception: self.updater.flush_updates() From 5e6188b6915c2df0f71ee348d4c76f4bd71121ce Mon Sep 17 00:00:00 2001 From: Pahaz White Date: Thu, 18 Apr 2019 18:48:58 +0300 Subject: [PATCH 3/6] lib.integra.tasks: add statistic info --- lib/integra/tasks.py | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/lib/integra/tasks.py b/lib/integra/tasks.py index 4b7b889f..3822d310 100644 --- a/lib/integra/tasks.py +++ b/lib/integra/tasks.py @@ -14,7 +14,9 @@ def __init__(self, config): self.updater = Updater() def run(self): - count = 0 + processed = 0 + updated = 0 + errors = 0 for model in self.models: has_exception = False LOGGER.info( @@ -22,26 +24,31 @@ def run(self): model['app'], model['model']) for obj in self.loader.download(model): try: + processed += 1 status = self.updater.update(obj) - count += 1 if status else 0 + updated += 1 if status else 0 except Exception as exc: # noqa + has_exception = True + errors += 1 LOGGER.exception( "integra error: %r; app=%s model=%s data=%r", exc, obj['app'], obj['model'], obj['data']) - has_exception = True if not has_exception: self.updater.flush_updates() self.updater.clear_updates() - return count + return processed, updated, errors @shared_task def download_updates(): - count = 0 + processed, updated, errors = 0, 0, 0 configs = getattr(settings, 'INTEGRA_CONFIGS', None) if not configs: return 'no-configs' for config in configs: integrator = Integra(config) - count += integrator.run() - return f'ok:{count}' + c_processed, c_updated, c_errors = integrator.run() + processed += c_processed + updated += c_updated + errors += c_errors + return f'ok:{processed}/{updated}:errors:{errors}' From 5a493bbcc28253e46f18784bca36e0f8e2749829 Mon Sep 17 00:00:00 2001 From: Pahaz White Date: Thu, 18 Apr 2019 18:56:23 +0300 Subject: [PATCH 4/6] lib.integra.tests: add test_last_updated_counters --- lib/integra/tests/test_updater.py | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/lib/integra/tests/test_updater.py b/lib/integra/tests/test_updater.py index 31f03c72..d909722b 100644 --- a/lib/integra/tests/test_updater.py +++ b/lib/integra/tests/test_updater.py @@ -37,3 +37,30 @@ def test_updater_protocol_update(): assert UpdateState.objects.count() == count + 1 assert UpdateState.objects.last().key == 'updater1' assert UpdateState.objects.last().updated.isoformat() == updated_value + assert updater.last_updated == {} + + +def test_last_updated_counters(): + updater = Updater() + count = UpdateState.objects.count() + updated_value = '2018-01-12T22:33:45.011349' + + updater.update({ + 'app': 'integra', + 'model': 'updatestate', + 'data': {'_uid': 'updater1', '_type': 'updatestate', + 'updated': '2012-04-12T22:33:45.028342'}, + 'last_updated': '2012-04-12T22:33:45.028342', + }) + updater.update({ + 'app': 'integra', + 'model': 'updatestate', + 'data': {'_uid': 'updater1', '_type': 'updatestate', + 'updated': updated_value}, + 'last_updated': updated_value, + }) + + assert UpdateState.objects.count() == count + 1 + assert UpdateState.objects.last().key == 'updater1' + assert UpdateState.objects.last().updated.isoformat() == updated_value + assert updater.last_updated == {'integra:updatestate': updated_value} From a6bc5607c9bdcc4929af0db111cb8c48cc83656d Mon Sep 17 00:00:00 2001 From: Pahaz White Date: Thu, 18 Apr 2019 23:03:57 +0300 Subject: [PATCH 5/6] lib.integra.tests: more Integra tests! --- contacts/tests/test_lib_integra.py | 289 +++++++++++++++++++++++++++++ lib/integra/utils.py | 9 +- requirements.txt | 1 + 3 files changed, 296 insertions(+), 3 deletions(-) create mode 100644 contacts/tests/test_lib_integra.py diff --git a/contacts/tests/test_lib_integra.py b/contacts/tests/test_lib_integra.py new file mode 100644 index 00000000..a694750d --- /dev/null +++ b/contacts/tests/test_lib_integra.py @@ -0,0 +1,289 @@ +from copy import deepcopy +from unittest import mock + +from contacts.models import Contact +from lib.integra.tasks import Integra + +RESULT = [ + { + "_uid": "07326036-9baf-41cb-91fc-1d2f702fdec9", + "_type": "contact", + "_version": 5, + "created": "2018-02-17T22:02:57.666548", + "updated": "2019-04-07T10:49:17.500849", + "name": "Steven Carrol9l", + "phones": [ + "5203" + ], + "emails": [ + "steven carroll@example.com" + ], + "order_index": 10089 + }, + { + "_uid": "07326036-9baf-41cb-91fc-1d2f702fdec9", + "_type": "contact", + "_version": 6, + "created": "2018-02-17T22:02:57.666548", + "updated": "2019-04-07T10:49:27.494178", + "name": "Carla 2", + "phones": [ + "2386" + ], + "emails": [ + "carla maddox@example.com" + ], + "order_index": 1009 + }, + { + "_uid": "07326036-9baf-41cb-91fc-1d2f702fdec9", + "_type": "contact", + "_version": 13, + "created": "2018-02-17T22:02:57.666548", + "updated": "2019-04-07T10:50:58.897061", + "name": "d", + "phones": [ + "8500" + ], + "emails": [ + "wayne brown@example.com" + ], + "order_index": 100 + }, + { + "_uid": "0168245f-0298-4048-5f26-a108fed49d92", + "_type": "contact", + "_version": 15, + "created": "2019-01-06T18:13:52.922085", + "updated": "2019-04-07T10:51:23.401410", + "name": "lklklk", + "phones": [ + "9989" + ], + "emails": [ + "it-services@pik-comfort.ru", + "qwe@qwe.q9j2" + ], + "order_index": 100 + }, + { + "_uid": "12134bdd-d71b-4731-b72a-c16c9cf9b587", + "_type": "contact", + "_version": 6, + "created": "2018-02-17T22:02:57.609496", + "updated": "2019-04-07T10:53:16.349032", + "name": "Angela 3f", + "phones": [ + "2583" + ], + "emails": [ + "angela rogers@example.com" + ], + "order_index": 10021 + }, + { + "_uid": "12134bdd-d71b-4731-b72a-c16c9cf9b587", + "_type": "contact", + "_version": 32, + "created": "2018-02-17T22:02:57.609496", + "updated": "2019-04-07T10:57:37.778117", + "name": "Jonathan Sheppard", + "phones": [ + "2021" + ], + "emails": [ + "jonathan sheppard@example.com" + ], + "order_index": 1001 + }, + { + "_uid": "0f273e43-a8cc-471e-bc6f-a66479c9f340", + "_type": "contact", + "_version": 4, + "created": "2018-02-17T22:02:52.142320", + "updated": "2019-04-07T12:06:56.221892", + "name": "Suzanne 2", + "phones": [ + "7221" + ], + "emails": [ + "suzanne nichols@example.com" + ], + "order_index": 120 + }, + { + "_uid": "710c2484-baba-4d10-9972-61dfeca58237", + "_type": "contact", + "_version": 5, + "created": "2018-02-17T22:02:46.549939", + "updated": "2019-04-07T12:07:23.345818", + "name": "David 3", + "phones": [ + "3930" + ], + "emails": [ + "david huber@example.com" + ], + "order_index": 10 + }, + { + "_uid": "12134bdd-d71b-4731-b72a-c16c9cf9b587", + "_type": "contact", + "_version": 44, + "created": "2018-02-17T22:02:57.609496", + "updated": "2019-04-07T12:09:37.690856", + "name": "eee 2", + "phones": [ + "4382" + ], + "emails": [ + "richard nielsen@example.com" + ], + "order_index": 100 + }, + { + "_uid": "bd915ab7-fe7e-49db-bd06-890a292ba609", + "_type": "contact", + "_version": 4, + "created": "2018-02-17T22:02:57.602702", + "updated": "2019-04-07T12:09:49.146435", + "name": "q", + "phones": [ + "9663" + ], + "emails": [ + "craig bennett@example.com" + ], + "order_index": 1002 + }, + { + "_uid": "6cb6eece-fd33-4753-8aad-10fcec366a61", + "_type": "contact", + "_version": 6, + "created": "2018-02-17T22:02:57.642757", + "updated": "2019-04-07T13:05:15.727494", + "name": "e", + "phones": [ + "5827" + ], + "emails": [ + "linda johnson@example.com" + ], + "order_index": 1002 + }, + { + "_uid": "8df69d66-803e-48f3-b16f-545ee63c9dba", + "_type": "contact", + "_version": 12, + "created": "2018-02-17T22:02:57.632498", + "updated": "2019-04-07T13:13:41.236663", + "name": "awdawd", + "phones": [ + "9685" + ], + "emails": [ + "amyedwards@example.com" + ], + "order_index": 100 + }, + { + "_uid": "0168244e-78dd-ed03-f5d9-acc5b69fdd60", + "_type": "contact", + "_version": 21, + "created": "2019-01-06T17:55:49.086308", + "updated": "2019-04-07T13:24:46.201704", + "name": "2eee", + "phones": [], + "emails": [ + "it-services@pik-comfort.ru49" + ], + "order_index": 102222223 + }, + { + "_uid": "0168243a-7048-de6f-20da-7f222a7f1087", + "_type": "contact", + "_version": 9, + "created": "2019-01-06T17:33:56.387480", + "updated": "2019-04-07T13:26:38.702762", + "name": "022awdawd", + "phones": [], + "emails": [ + "it-services@pik-comfort.rx" + ], + "order_index": 1001 + }, + { + "_uid": "0168243a-7048-de6f-20da-7f222a7f1087", + "_type": "contact", + "_version": 2, + "created": "2019-04-14T23:09:47.202970", + "updated": "2019-04-14T23:09:57.585576", + "name": "new-one!", + "phones": [], + "emails": [], + "order_index": 100 + }, +] + + +def _make_integra(): + return Integra({ + 'base_url': 'http://127.0.0.1:8000', + 'request': {'auth': 'api-reader:MyPass39dza2es'}, + 'models': [ + {'url': '/api/v1/contact-list/', + 'app': 'contacts', + 'model': 'contact'}, + ]}) + + +@mock.patch('lib.integra.utils._fetch_data_from_api') +def test_integra_run(fetch): + fetch.return_value = deepcopy(RESULT) + integra = _make_integra() + + processed, updated, errors = integra.run() + + fetch.assert_called_once_with( + 'http://127.0.0.1:8000/api/v1/contact-list/', + ('api-reader', 'MyPass39dza2es'), + {'ordering': 'updated'}) + assert Contact.objects.count() == 10 + assert (processed, updated, errors) == (15, 14, 0) + + obj = Contact.objects.get(uid='0168243a-7048-de6f-20da-7f222a7f1087') + assert obj.version == 9 + assert obj.name == '022awdawd' + + +@mock.patch('lib.integra.utils._fetch_data_from_api') +def test_integra_run_on_the_same_result(fetch): + integra = _make_integra() + fetch.return_value = deepcopy(RESULT) + integra.run() + fetch.return_value = deepcopy(RESULT) + processed, updated, errors = integra.run() + assert (processed, updated, errors) == (15, 0, 0) + + +@mock.patch('lib.integra.utils._fetch_data_from_api') +def test_integra_update(fetch): + fetch.return_value = deepcopy(RESULT) + integra = _make_integra() + integra.run() + fetch.return_value = [ + { + "_uid": "0168243a-7048-de6f-20da-7f222a7f1087", + "_type": "contact", + "_version": 10, + "created": "2019-04-14T23:09:47.202970", + "updated": "2020-06-14T23:09:57.585576", + "name": "new-one! (up)", + "phones": [], + "emails": [], + "order_index": 100 + }, + ] + + processed, updated, errors = integra.run() + assert (processed, updated, errors) == (1, 1, 0) diff --git a/lib/integra/utils.py b/lib/integra/utils.py index 46c940b8..73963702 100644 --- a/lib/integra/utils.py +++ b/lib/integra/utils.py @@ -2,6 +2,7 @@ from urllib.parse import urljoin import requests +import dateutil.parser from core.utils.models import get_fields, has_field, get_model, \ get_base_manager, get_pk_name @@ -106,6 +107,8 @@ def update(self, obj): def _set_last_updated(self, obj): last_updated = obj.get('last_updated') if last_updated: + if isinstance(last_updated, str): + last_updated = dateutil.parser.parse(last_updated) key = f'{obj["app"]}:{obj["model"]}' current_value = self.last_updated.get(key) if current_value and current_value > last_updated: @@ -114,7 +117,9 @@ def _set_last_updated(self, obj): def flush_updates(self): for key, value in self.last_updated.items(): - UpdateState.objects.set_last_updated(key, value) + current_value = UpdateState.objects.get_last_updated(key) + if not current_value or current_value < value: + UpdateState.objects.set_last_updated(key, value) self.clear_updates() def clear_updates(self): @@ -150,7 +155,6 @@ def _prepare_model_attrs(model, data, is_strict=True) -> dict: def _fetch_data_from_api(url, auth, url_params=None): - results = [] next_page = 1 url_params = copy(url_params) if url_params else {} while next_page: @@ -161,4 +165,3 @@ def _fetch_data_from_api(url, auth, url_params=None): next_page = data['page_next'] for obj in data['results']: yield obj - return results diff --git a/requirements.txt b/requirements.txt index 6cd2ded1..ac41ecfb 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,6 +12,7 @@ google_cloud==0.34.0 google-cloud-resource-manager==0.28.3 google-cloud-runtimeconfig==0.28.3 google-cloud-translate==1.4.0 +python-dateutil==2.8.0 Django==2.1.7 From 67ea6d0b0ce65118198311d22e2accfed6cbdcbc Mon Sep 17 00:00:00 2001 From: Pahaz White Date: Thu, 18 Apr 2019 23:14:58 +0300 Subject: [PATCH 6/6] lib.integra.tests: fix internal representation of last_updated --- lib/integra/tests/test_updater.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/lib/integra/tests/test_updater.py b/lib/integra/tests/test_updater.py index d909722b..ab78c963 100644 --- a/lib/integra/tests/test_updater.py +++ b/lib/integra/tests/test_updater.py @@ -1,3 +1,5 @@ +import dateutil.parser + from lib.integra.models import UpdateState from lib.integra.utils import Updater @@ -63,4 +65,5 @@ def test_last_updated_counters(): assert UpdateState.objects.count() == count + 1 assert UpdateState.objects.last().key == 'updater1' assert UpdateState.objects.last().updated.isoformat() == updated_value - assert updater.last_updated == {'integra:updatestate': updated_value} + assert updater.last_updated == { + 'integra:updatestate': dateutil.parser.parse(updated_value)}