Skip to content

Commit

Permalink
Merge pull request #297 from pik-software/refactor-integra
Browse files Browse the repository at this point in the history 
lib.integra: support flash() on model loading finished
  • Loading branch information
@pahaz
pahaz authored Apr 18, 2019
2 parents f4391b8 + 72b611d commit 8f7f1f7
Show file tree
Hide file tree
Showing 6 changed files with 369 additions and 34 deletions.
289 changes: 289 additions & 0 deletions contacts/tests/test_lib_integra.py
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,289 @@
from copy import deepcopy
from unittest import mock

from contacts.models import Contact
from lib.integra.tasks import Integra

RESULT = [
{
"_uid": "07326036-9baf-41cb-91fc-1d2f702fdec9",
"_type": "contact",
"_version": 5,
"created": "2018-02-17T22:02:57.666548",
"updated": "2019-04-07T10:49:17.500849",
"name": "Steven Carrol9l",
"phones": [
"5203"
],
"emails": [
"steven carroll@example.com"
],
"order_index": 10089
},
{
"_uid": "07326036-9baf-41cb-91fc-1d2f702fdec9",
"_type": "contact",
"_version": 6,
"created": "2018-02-17T22:02:57.666548",
"updated": "2019-04-07T10:49:27.494178",
"name": "Carla 2",
"phones": [
"2386"
],
"emails": [
"carla maddox@example.com"
],
"order_index": 1009
},
{
"_uid": "07326036-9baf-41cb-91fc-1d2f702fdec9",
"_type": "contact",
"_version": 13,
"created": "2018-02-17T22:02:57.666548",
"updated": "2019-04-07T10:50:58.897061",
"name": "d",
"phones": [
"8500"
],
"emails": [
"wayne brown@example.com"
],
"order_index": 100
},
{
"_uid": "0168245f-0298-4048-5f26-a108fed49d92",
"_type": "contact",
"_version": 15,
"created": "2019-01-06T18:13:52.922085",
"updated": "2019-04-07T10:51:23.401410",
"name": "lklklk",
"phones": [
"9989"
],
"emails": [
"it-services@pik-comfort.ru",
"qwe@qwe.q9j2"
],
"order_index": 100
},
{
"_uid": "12134bdd-d71b-4731-b72a-c16c9cf9b587",
"_type": "contact",
"_version": 6,
"created": "2018-02-17T22:02:57.609496",
"updated": "2019-04-07T10:53:16.349032",
"name": "Angela 3f",
"phones": [
"2583"
],
"emails": [
"angela rogers@example.com"
],
"order_index": 10021
},
{
"_uid": "12134bdd-d71b-4731-b72a-c16c9cf9b587",
"_type": "contact",
"_version": 32,
"created": "2018-02-17T22:02:57.609496",
"updated": "2019-04-07T10:57:37.778117",
"name": "Jonathan Sheppard",
"phones": [
"2021"
],
"emails": [
"jonathan sheppard@example.com"
],
"order_index": 1001
},
{
"_uid": "0f273e43-a8cc-471e-bc6f-a66479c9f340",
"_type": "contact",
"_version": 4,
"created": "2018-02-17T22:02:52.142320",
"updated": "2019-04-07T12:06:56.221892",
"name": "Suzanne 2",
"phones": [
"7221"
],
"emails": [
"suzanne nichols@example.com"
],
"order_index": 120
},
{
"_uid": "710c2484-baba-4d10-9972-61dfeca58237",
"_type": "contact",
"_version": 5,
"created": "2018-02-17T22:02:46.549939",
"updated": "2019-04-07T12:07:23.345818",
"name": "David 3",
"phones": [
"3930"
],
"emails": [
"david huber@example.com"
],
"order_index": 10
},
{
"_uid": "12134bdd-d71b-4731-b72a-c16c9cf9b587",
"_type": "contact",
"_version": 44,
"created": "2018-02-17T22:02:57.609496",
"updated": "2019-04-07T12:09:37.690856",
"name": "eee 2",
"phones": [
"4382"
],
"emails": [
"richard nielsen@example.com"
],
"order_index": 100
},
{
"_uid": "bd915ab7-fe7e-49db-bd06-890a292ba609",
"_type": "contact",
"_version": 4,
"created": "2018-02-17T22:02:57.602702",
"updated": "2019-04-07T12:09:49.146435",
"name": "q",
"phones": [
"9663"
],
"emails": [
"craig bennett@example.com"
],
"order_index": 1002
},
{
"_uid": "6cb6eece-fd33-4753-8aad-10fcec366a61",
"_type": "contact",
"_version": 6,
"created": "2018-02-17T22:02:57.642757",
"updated": "2019-04-07T13:05:15.727494",
"name": "e",
"phones": [
"5827"
],
"emails": [
"linda johnson@example.com"
],
"order_index": 1002
},
{
"_uid": "8df69d66-803e-48f3-b16f-545ee63c9dba",
"_type": "contact",
"_version": 12,
"created": "2018-02-17T22:02:57.632498",
"updated": "2019-04-07T13:13:41.236663",
"name": "awdawd",
"phones": [
"9685"
],
"emails": [
"amyedwards@example.com"
],
"order_index": 100
},
{
"_uid": "0168244e-78dd-ed03-f5d9-acc5b69fdd60",
"_type": "contact",
"_version": 21,
"created": "2019-01-06T17:55:49.086308",
"updated": "2019-04-07T13:24:46.201704",
"name": "2eee",
"phones": [],
"emails": [
"it-services@pik-comfort.ru49"
],
"order_index": 102222223
},
{
"_uid": "0168243a-7048-de6f-20da-7f222a7f1087",
"_type": "contact",
"_version": 9,
"created": "2019-01-06T17:33:56.387480",
"updated": "2019-04-07T13:26:38.702762",
"name": "022awdawd",
"phones": [],
"emails": [
"it-services@pik-comfort.rx"
],
"order_index": 1001
},
{
"_uid": "0168243a-7048-de6f-20da-7f222a7f1087",
"_type": "contact",
"_version": 2,
"created": "2019-04-14T23:09:47.202970",
"updated": "2019-04-14T23:09:57.585576",
"name": "new-one!",
"phones": [],
"emails": [],
"order_index": 100
},
]


def _make_integra():
return Integra({
'base_url': 'http://127.0.0.1:8000',
'request': {'auth': 'api-reader:MyPass39dza2es'},
'models': [
{'url': '/api/v1/contact-list/',
'app': 'contacts',
'model': 'contact'},
]})


@mock.patch('lib.integra.utils._fetch_data_from_api')
def test_integra_run(fetch):
fetch.return_value = deepcopy(RESULT)
integra = _make_integra()

processed, updated, errors = integra.run()

fetch.assert_called_once_with(
'http://127.0.0.1:8000/api/v1/contact-list/',
('api-reader', 'MyPass39dza2es'),
{'ordering': 'updated'})
assert Contact.objects.count() == 10
assert (processed, updated, errors) == (15, 14, 0)

obj = Contact.objects.get(uid='0168243a-7048-de6f-20da-7f222a7f1087')
assert obj.version == 9
assert obj.name == '022awdawd'


@mock.patch('lib.integra.utils._fetch_data_from_api')
def test_integra_run_on_the_same_result(fetch):
integra = _make_integra()
fetch.return_value = deepcopy(RESULT)
integra.run()
fetch.return_value = deepcopy(RESULT)
processed, updated, errors = integra.run()
assert (processed, updated, errors) == (15, 0, 0)


@mock.patch('lib.integra.utils._fetch_data_from_api')
def test_integra_update(fetch):
fetch.return_value = deepcopy(RESULT)
integra = _make_integra()
integra.run()
fetch.return_value = [
{
"_uid": "0168243a-7048-de6f-20da-7f222a7f1087",
"_type": "contact",
"_version": 10,
"created": "2019-04-14T23:09:47.202970",
"updated": "2020-06-14T23:09:57.585576",
"name": "new-one! (up)",
"phones": [],
"emails": [],
"order_index": 100
},
]

processed, updated, errors = integra.run()
assert (processed, updated, errors) == (1, 1, 0)
47 changes: 30 additions & 17 deletions lib/integra/tasks.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,33 +9,46 @@

class Integra:
def __init__(self, config):
self.models = config['models']
self.loader = Loader(config)
self.updater = Updater()

def run(self):
count = 0
has_exception = False
for obj in self.loader.download():
try:
status = self.updater.update(obj)
count += 1 if status else 0
except Exception as exc: # noqa
app, model, data = obj['app'], obj['model'], obj['data']
LOGGER.exception("integra error: %r; app=%s model=%s data=%r",
exc, app, model, data)
has_exception = True
if not has_exception:
self.updater.flush_updates()
return count
processed = 0
updated = 0
errors = 0
for model in self.models:
has_exception = False
LOGGER.info(
"integra: loading app=%s model=%s",
model['app'], model['model'])
for obj in self.loader.download(model):
try:
processed += 1
status = self.updater.update(obj)
updated += 1 if status else 0
except Exception as exc: # noqa
has_exception = True
errors += 1
LOGGER.exception(
"integra error: %r; app=%s model=%s data=%r",
exc, obj['app'], obj['model'], obj['data'])
if not has_exception:
self.updater.flush_updates()
self.updater.clear_updates()
return processed, updated, errors


@shared_task
def download_updates():
count = 0
processed, updated, errors = 0, 0, 0
configs = getattr(settings, 'INTEGRA_CONFIGS', None)
if not configs:
return 'no-configs'
for config in configs:
integrator = Integra(config)
count += integrator.run()
return f'ok:{count}'
c_processed, c_updated, c_errors = integrator.run()
processed += c_processed
updated += c_updated
errors += c_errors
return f'ok:{processed}/{updated}:errors:{errors}'
13 changes: 7 additions & 6 deletions lib/integra/tests/test_loader.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,18 +2,19 @@


def test_loader_protocol(mocker):
model = {
'url': '/api/v1/contact-list/',
'app': 'housing',
'model': 'contact'}
config = {
'base_url': 'https://housing.pik-software.ru/',
'request': {},
'models': [{
'url': '/api/v1/contact-list/',
'app': 'housing',
'model': 'contact'}]}
'models': [model]}
result = [{'app': 'housing', 'model': 'contact', 'data': {'_uid': '0'}}]

loader = Loader(config)
with mocker.patch.object(loader, '_request', return_value=result):
downloaded = list(loader.download())
downloaded = list(loader.download(model))

assert downloaded == result
loader._request.assert_called_once_with(config['models'][0], None) # noqa: pylint=protected-access
loader._request.assert_called_once_with(model, None) # noqa: pylint=protected-access
Loading

0 comments on commit 8f7f1f7

Please sign in to comment.