diff --git a/README.md b/README.md index 80d445ea..9df3a5e6 100644 --- a/README.md +++ b/README.md @@ -68,6 +68,7 @@ You can download its zip file from the [Github Releases](https://github.com/clou * [Reports](docs/reports_usage.md) * [Translations](docs/translations_usage.md) * [Projects](docs/project_usage.md) +* [Streams](docs/stream_usage.md) ## Development diff --git a/connect/cli/plugins/commerce/commands.py b/connect/cli/plugins/commerce/commands.py index 0ba3592e..53647c67 100644 --- a/connect/cli/plugins/commerce/commands.py +++ b/connect/cli/plugins/commerce/commands.py @@ -14,7 +14,9 @@ display_streams_table, export_stream, get_destination_account, + print_errors, print_results, + sync_stream, ) @@ -165,5 +167,33 @@ def cmd_clone_stream( print_results(results) +@grp_commerce_streams.command( + name='sync', + short_help='Synchronize a stream from an excel file.', +) +@click.argument('input_file', metavar='input_file', nargs=1, required=True) # noqa: E304 +@pass_config +def cmd_sync_stream(config, input_file): + stream_id = None + if '.xlsx' not in input_file: + stream_id = input_file + input_file = f'{input_file}/{input_file}.xlsx' + else: + stream_id = input_file.split('/')[-1].split('.')[0] + results, errors = sync_stream( + account=config.active, + stream_id=stream_id, + input_file=input_file, + ) + + console.echo('') + + print_results(results) + + console.echo('') + + print_errors(errors) + + def get_group(): return grp_commerce diff --git a/connect/cli/plugins/commerce/utils.py b/connect/cli/plugins/commerce/utils.py index 44e61db2..9a79cf75 100644 --- a/connect/cli/plugins/commerce/utils.py +++ b/connect/cli/plugins/commerce/utils.py @@ -9,7 +9,7 @@ from click import ClickException from connect.cli.core.terminal import console from connect.client import ClientError -from openpyxl import Workbook +from openpyxl import Workbook, load_workbook from openpyxl.styles import Alignment, Font, PatternFill from openpyxl.styles.colors import WHITE, Color @@ -145,7 +145,7 @@ def fill_transformations(ws, transformations, progress): 'Function Name', 'Description', 'Overview', - 'Input columns', + 'Input Columns', 'Output Columns', 'Position', 'Settings', @@ -280,7 +280,7 @@ def export_stream( data={ 'Stream ID': stream_id, 'Stream Name': response['name'], - 'Stream Description': response['description'], + 'Stream Description': response.get('description'), 'Stream Type': stream_type, 'Stream Category': 'Inbound' if response['owner']['id'] != active_account_id @@ -349,6 +349,8 @@ def export_stream( wb.save(output_file) + console.echo('') + console.secho( f'Stream {stream_id} exported properly to {output_file}.', fg='green', @@ -634,6 +636,20 @@ def print_results(results): ) +def print_errors(errors): # pragma: no cover + if errors: + console.confirm( + 'Are you sure you want to display errors?', + abort=True, + ) + console.echo('') + for error in errors: + console.secho( + error, + fg='red', + ) + + def clone_stream( origin_account, stream_id, @@ -761,3 +777,331 @@ def clone_stream( ) return destination_stream_id, results + + +def validate_sheet_names(wb): + for sheet in ( + 'General Information', + 'Columns', + 'Transformations', + 'Attachments', + ): + if sheet not in wb.sheetnames: + raise ClickException( + 'The file must contain `General Information`, `Columns`, `Transformations` and ' + '`Attachments` sheets.' + ) + + +def _validate_header( + current_headers, + expected_headers, + sheet_name, +): + for header in expected_headers: + if header not in current_headers: + raise ClickException( + f'The {sheet_name} sheet header does not contain `{header}` header.' + ) + + +def validate_headers(wb): + _validate_header( + [c.value for c in wb['General Information'][1]], + ( + 'Stream information', + None, + ), + 'General Information', + ) + _validate_header( + [c.value for c in wb['Columns'][1]], + ( + 'ID', + 'Name', + 'Description', + 'Type', + 'Position', + 'Required', + 'Output', + ), + 'Columns', + ) + _validate_header( + [c.value for c in wb['Transformations'][1]], + ( + 'ID', + 'Function ID', + 'Function Name', + 'Description', + 'Overview', + 'Input Columns', + 'Output Columns', + 'Position', + 'Settings', + ), + 'Transformations', + ) + _validate_header( + [c.value for c in wb['Attachments'][1]], + ( + 'ID', + 'Name', + ), + 'Attachments', + ) + + +def get_work_book(input_file): + if not os.path.exists(input_file): + raise ClickException(f'The file {input_file} does not exists.') + if 'xlsx' not in input_file: + raise ClickException(f'The file {input_file} has invalid format, must be xlsx.') + wb = load_workbook(input_file, read_only=True) + + validate_sheet_names(wb) + + validate_headers(wb) + + return wb + + +def update_general_information( + client, + collection, + stream_id, + sheet, + results, + errors, + progress, +): + task = progress.add_task('Updating general information', total=5) + stream = ( + client.ns(collection) + .streams.filter(id=stream_id) + .select( + 'context', + 'samples', + 'sources', + 'validation', + ) + .first() + ) + + if stream['status'] == 'active' or stream.get('validation', {}).get('status') == 'processing': + raise ClickException( + f'Stream {stream_id} cannot be updated because it is in "active" status ' + f'or validation is processing.', + ) + + body = {'context': {}} + updated = 0 + errors_on_update = 0 + for n in range(2, sheet.max_row + 1): + h, v = sheet[n] + if h.value == 'Stream Name' and stream['name'] != v.value: + body['name'] = v.value + updated += 1 + elif h.value == 'Stream Description' and stream.get('description') != v.value: + body['description'] = v.value + updated += 1 + elif ( + h.value == 'Product ID' + and stream.get('context', {}).get('product', {}).get('id', None) != v.value + ): + body['context']['product'] = {'id': v.value} + updated += 1 + elif ( + h.value == 'Partner ID' + and stream.get('context', {}).get('account', {}).get('id', None) != v.value + ): + body['context']['account'] = {'id': v.value} + updated += 1 + elif ( + h.value == 'Marketplace ID' + and stream.get('context', {}).get('marketplace', {}).get('id', None) != v.value + ): + body['context']['marketplace'] = {'id': v.value} + updated += 1 + + if updated: + if not body['context']: + del body['context'] + try: + client.ns(collection).streams[stream_id].update( + json=body, + ) + except ClientError as e: + errors.append(str(e)) + updated = 0 + errors_on_update = 5 + + results.append(('General information', 5, 0, updated, 0, 0, errors_on_update)) + progress.update(task, advance=5) + + +def update_transformations( + client, + collection, + stream_id, + sheet, + results, + errors, + progress, +): + task = progress.add_task('Updating transformation information', total=sheet.max_row - 1) + + updated = 0 + deleted = 0 + ids = [] + for n in range(2, sheet.max_row + 1): + id, fid, fname, descr, over, input, output, position, settings = sheet[n] + ids.append(id.value) + origin_trf = None + try: + origin_trf = client.ns(collection).streams[stream_id].transformations[id.value].get() + except ClientError: + errors.append( + f'The transformation {id.value} cannot be updated because it does not exist.' + ) + progress.update(task, advance=1) + continue + + try: + to_update = {} + if origin_trf['settings'] != json.loads(settings.value): + to_update['settings'] = json.loads(settings.value) + if origin_trf.get('description') != descr.value: + to_update['description'] = descr.value + if int(origin_trf['position']) != position.value: + to_update['position'] = int(position.value) + + if to_update: + client.ns(collection).streams[stream_id].transformations[id.value].update( + json=to_update, + ) + updated += 1 + progress.update(task, advance=1) + except ClientError: + errors.append(f'Error updating the transformation {id.value} with data {to_update}.') + + try: + current_ids = [ + t['id'] for t in list(client.ns(collection).streams[stream_id].transformations.all()) + ] + for cid in current_ids: + if cid not in ids: + client.ns(collection).streams[stream_id].transformations[cid].delete() + deleted += 1 + except ClientError as e: + errors.append(f'Error deleting the transformation {e}.') + + results.append(('Transformations', sheet.max_row - 1, 0, updated, deleted, 0, len(errors))) + + +def update_attachments( + client, + stream_id, + sheet, + results, + errors, + progress, +): + task = progress.add_task('Updating attachment files', total=sheet.max_row - 1) + + created = 0 + deleted = 0 + errored = 0 + skipped = 0 + ids = [] + for n in range(2, sheet.max_row + 1): + id, file_name = sheet[n] + try: + if ( + client.ns('media') + .ns('folders') + .collection('streams_attachments')[stream_id] + .collection('files') + .filter(id=id.value) + .count() + == 0 + ): + response = upload_attachment( + client, + stream_id, + os.path.join(stream_id, 'attachments', file_name.value), + ) + ids.append(response['id']) + created += 1 + else: + ids.append(id.value) + skipped += 1 + + except ClientError as e: + errors.append(str(e)) + errored += 1 + + for attachment in list( + client.ns('media').ns('folders').collection('streams_attachments')[stream_id].files.all() + ): + try: + if attachment['id'] not in ids: + client.ns('media').ns('folders').collection('streams_attachments')[stream_id].files[ + attachment['id'] + ].delete() + deleted += 1 + except ClientError as e: + errors.append(str(e)) + errored += 1 + progress.update(task, advance=sheet.max_row - 1) + results.append(('Attachment files', sheet.max_row - 1, created, 0, deleted, skipped, errored)) + + +def sync_stream( + account, + stream_id, + input_file, +): + wb = get_work_book(input_file) + + collection = guess_if_billing_or_pricing_stream(account.client, stream_id) + if not collection: + raise ClickException(f'Stream {stream_id} not found for the current account {account.id}.') + + results = [] + errors = [] + with console.status_progress() as (status, progress): + status.update('Updating general information', fg='blue') + update_general_information( + client=account.client, + collection=collection, + stream_id=stream_id, + sheet=wb['General Information'], + results=results, + errors=errors, + progress=progress, + ) + + status.update('Updating transformations', fg='blue') + update_transformations( + client=account.client, + collection=collection, + stream_id=stream_id, + sheet=wb['Transformations'], + results=results, + errors=errors, + progress=progress, + ) + + status.update('Updating attachments', fg='blue') + if wb['Attachments'].max_row > 1: + update_attachments( + client=account.client, + stream_id=stream_id, + sheet=wb['Attachments'], + results=results, + errors=errors, + progress=progress, + ) + + return results, errors diff --git a/docs/stream_usage.md b/docs/stream_usage.md new file mode 100644 index 00000000..431df4f8 --- /dev/null +++ b/docs/stream_usage.md @@ -0,0 +1,73 @@ +# Streams management + +The streams management area offers commands that allow users to clone and get listing of streams, +export or synchronize to/from Excel file. + +To access the group of commands related to the management of streams you must invoke the CLI with the `commerce stream` command: + +```sh +$ ccli commerce stream + +Usage: ccli commerce [OPTIONS] COMMAND [ARGS]... + +Options: + -h, --help Show this message and exit. + +Commands: + clone Create a clone of a stream. + export Export commerce billing or pricing streams. + list List commerce billing and pricing streams. + sync Synchronize a stream from an excel file. +``` + +### Clone streams + +To clone existing stream you can run: + +```sh +$ccli commerce stream clone [OPTIONS] stream_id +``` + +``` +Options: + -d, --destination_account TEXT Destination account ID + -n, --new-stream-name TEXT Cloned stream name + -v, --validate Executes the validate action after the clone. +``` + +### List streams + +To list available streams you can run: + +```sh +$ ccli commerce stream list [OPTIONS] +``` + +``` +Options: + -q, --query TEXT RQL query expression. +``` + +### Export streams + +To export stream you can run: + +```sh +$ ccli commerce stream export [OPTIONS] stream_id +``` + +``` +Options: + -o, --out FILE Output Excel file name. + -p, --output_path DIRECTORY Directory where to store the export. +``` + +### Synchronize streams + +To synchronize a stream from an Excel file you can run: + +```sh +$ ccli commerce stream sync input_file +``` + +Structure of sync file can be taken from `tests/fixtures/commerce/stream_sync.xlsx` file. diff --git a/tests/conftest.py b/tests/conftest.py index 0ca939df..a8da7d8d 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -336,6 +336,11 @@ def sample_translation_workbook(fs): return load_workbook('./tests/fixtures/translation.xlsx') +@pytest.fixture(scope='function') +def sample_stream_workbook(fs): + return load_workbook('./tests/fixtures/commerce/stream_sync.xlsx') + + @pytest.fixture(scope='function') def mocked_translation_response(): with open('./tests/fixtures/translation_response.json') as response: @@ -486,3 +491,11 @@ def client(): endpoint='https://localhost/public/v1', use_specs=False, ) + + +@pytest.fixture +def load_stream_sync(): + return load_workbook( + './tests/fixtures/commerce/stream_sync.xlsx', + read_only=True, + ) diff --git a/tests/fixtures/commerce/stream_sync.xlsx b/tests/fixtures/commerce/stream_sync.xlsx new file mode 100644 index 00000000..465557d5 Binary files /dev/null and b/tests/fixtures/commerce/stream_sync.xlsx differ diff --git a/tests/plugins/commerce/test_commands.py b/tests/plugins/commerce/test_commands.py index 8849fcd0..9ad4b049 100644 --- a/tests/plugins/commerce/test_commands.py +++ b/tests/plugins/commerce/test_commands.py @@ -820,3 +820,200 @@ def test_clone_destination_account_not_found( assert result.output == ( 'Current active account: VA-000 - Account 0\n\nError: Error obtaining the destination account id VA-666\n' ) + + +def test_sync_stream( + mocker, + ccli, + mocked_responses, + config_mocker, + load_stream_sync, +): + mocked_get_work_book = mocker.patch( + 'connect.cli.plugins.commerce.utils.get_work_book', + return_value=load_stream_sync, + ) + mocked_upload_attachment = mocker.patch( + 'connect.cli.plugins.commerce.utils.upload_attachment', + return_value={'id': 'ID'}, + ) + mocked_responses.add( + method='GET', + url='https://localhost/public/v1/billing/streams?eq(id,STR-7748-7021-7449)&limit=0&offset=0', + headers={ + 'Content-Range': 'items 0-1/1', + }, + ) + + with open('./tests/fixtures/commerce/stream_retrieve_response.json') as content: + response = json.load(content)[0] + response['status'] = 'configuring' + mocked_responses.add( + method='GET', + url='https://localhost/public/v1/billing/streams?eq(id,STR-7748-7021-7449)&select(context,samples,sources,validation)&limit=1&offset=0', + json=[response], + ) + mocked_responses.add( + method='PUT', + url='https://localhost/public/v1/billing/streams/STR-7748-7021-7449', + json=[], + ) + + mocked_responses.add( + method='GET', + url='https://localhost/public/v1/billing/streams/STR-7748-7021-7449/transformations/STRA-774-870-217-449-001', + json={'settings': {'some': 'settings'}, 'description': 'description', 'position': 50}, + ) + mocked_responses.add( + method='PUT', + url='https://localhost/public/v1/billing/streams/STR-7748-7021-7449/transformations/STRA-774-870-217-449-001', + ) + + mocked_responses.add( + method='GET', + url='https://localhost/public/v1/billing/streams/STR-7748-7021-7449/transformations/STRA-774-870-217-449-002', + json={'settings': {'some': 'settings'}, 'description': 'description', 'position': 60}, + ) + mocked_responses.add( + method='PUT', + url='https://localhost/public/v1/billing/streams/STR-7748-7021-7449/transformations/STRA-774-870-217-449-002', + ) + + mocked_responses.add( + method='GET', + url='https://localhost/public/v1/billing/streams/STR-7748-7021-7449/transformations', + json=[ + {'id': 'STRA-774-870-217-449-001'}, + {'id': 'STRA-774-870-217-449-002'}, + {'id': 'STRA-774-870-217-449-003'}, + ], + ) + mocked_responses.add( + method='DELETE', + url='https://localhost/public/v1/billing/streams/STR-7748-7021-7449/transformations/STRA-774-870-217-449-003', + ) + + mocked_responses.add( + method='GET', + url='https://localhost/public/v1/media/folders/streams_attachments/STR-7748-7021-7449/files?eq(id,ID)&limit=0&offset=0', + headers={ + 'Content-Range': 'items 0-0/0', + }, + ) + + mocked_responses.add( + method='GET', + url='https://localhost/public/v1/media/folders/streams_attachments/STR-7748-7021-7449/files?eq(id,ID-EXISTS)&limit=0&offset=0', + headers={ + 'Content-Range': 'items 0-1/1', + }, + ) + + mocked_responses.add( + method='GET', + url='https://localhost/public/v1/media/folders/streams_attachments/STR-7748-7021-7449/files?limit=100&offset=0', + json=[{'id': 'ID-EXISTS'}, {'id': 'DO-NOT-EXIST'}], + ) + mocked_responses.add( + method='DELETE', + url='https://localhost/public/v1/media/folders/streams_attachments/STR-7748-7021-7449/files/DO-NOT-EXIST', + status=204, + ) + + mocked_cmd_console = mocker.patch( + 'connect.cli.plugins.commerce.commands.console', + ) + runner = CliRunner() + cmd = [ + 'commerce', + 'stream', + 'sync', + 'STR-7748-7021-7449', + ] + result = runner.invoke( + ccli, + cmd, + ) + + mocked_upload_attachment.assert_called_with( + mocker.ANY, + 'STR-7748-7021-7449', + 'STR-7748-7021-7449/attachments/attachment.xlsx', + ) + + assert result.exit_code == 0 + assert mocked_cmd_console.echo.call_count == 2 + + +def test_sync_stream_no_stream( + mocker, + ccli, + config_mocker, +): + mocker.patch('connect.cli.plugins.commerce.commands.console') + mocker.patch('connect.cli.plugins.commerce.utils.get_work_book') + mocker.patch( + 'connect.cli.plugins.commerce.utils.guess_if_billing_or_pricing_stream', + return_value=None, + ) + runner = CliRunner() + cmd = [ + 'commerce', + 'stream', + 'sync', + 'STR-7748-7021-7449', + ] + result = runner.invoke( + ccli, + cmd, + ) + + assert result.exit_code == 1 + assert 'Stream STR-7748-7021-7449 not found for the current account VA-000.' in result.output + + +def test_sync_stream_active_stream( + mocker, + ccli, + mocked_responses, + config_mocker, + load_stream_sync, +): + mocker.patch( + 'connect.cli.plugins.commerce.utils.get_work_book', + return_value=load_stream_sync, + ) + mocker.patch( + 'connect.cli.plugins.commerce.utils.upload_attachment', + return_value={'id': 'ID'}, + ) + mocked_responses.add( + method='GET', + url='https://localhost/public/v1/billing/streams?eq(id,STR-7748-7021-7449)&limit=0&offset=0', + headers={ + 'Content-Range': 'items 0-1/1', + }, + ) + + with open('./tests/fixtures/commerce/stream_retrieve_response.json') as content: + response = json.load(content)[0] + mocked_responses.add( + method='GET', + url='https://localhost/public/v1/billing/streams?eq(id,STR-7748-7021-7449)&select(context,samples,sources,validation)&limit=1&offset=0', + json=[response], + ) + + runner = CliRunner() + cmd = [ + 'commerce', + 'stream', + 'sync', + 'STR-7748-7021-7449', + ] + result = runner.invoke( + ccli, + cmd, + ) + + assert result.exit_code == 1 + assert 'Stream STR-7748-7021-7449 cannot be updated because it is in "active"' in result.output diff --git a/tests/plugins/commerce/test_utils.py b/tests/plugins/commerce/test_utils.py index 37daccee..12576fbb 100644 --- a/tests/plugins/commerce/test_utils.py +++ b/tests/plugins/commerce/test_utils.py @@ -1,17 +1,23 @@ +import json import os import tempfile -import json import pytest from click import ClickException -from openpyxl import Workbook +from connect.client import ConnectClient, ClientError +from openpyxl import Workbook, load_workbook from connect.cli.plugins.commerce.utils import ( - fill_and_download_attachments, - guess_if_billing_or_pricing_stream, + _validate_header, clone_stream, - create_stream_from_origin, clone_transformations, + create_stream_from_origin, + fill_and_download_attachments, + get_work_book, + guess_if_billing_or_pricing_stream, + update_attachments, + update_general_information, + update_transformations, ) @@ -279,3 +285,318 @@ def test_clone_transformations_fix_file_mapping( mocked_col_mapping.assert_called() mocked_progress.add_task.assert_called_with('Processing transformations', total=1) assert mocked_progress.update.call_count == 1 + + +def test_get_work_book(): + wb = get_work_book('./tests/fixtures/commerce/stream_sync.xlsx') + + general_info = wb['General Information'] + assert general_info['B2'].value == 'STR-7748-7021-7449' + + wb.close() + + +def test_get_work_book_invalid_sheets(): + with pytest.raises(ClickException) as ce: + get_work_book('./tests/fixtures/actions_sync.xlsx') + + assert str(ce.value) == ( + 'The file must contain `General Information`, `Columns`, `Transformations` and ' + '`Attachments` sheets.' + ) + + +def test_get_work_book_non_existing_file(): + with pytest.raises(ClickException) as ce: + get_work_book('non_existing_file.xlsx') + + assert str(ce.value) == 'The file non_existing_file.xlsx does not exists.' + + +def test_get_work_book_invalid_format(): + with pytest.raises(ClickException) as ce: + get_work_book('./tests/fixtures/image.png') + + assert str(ce.value) == 'The file ./tests/fixtures/image.png has invalid format, must be xlsx.' + + +def test_validate_header_missing(): + with pytest.raises(ClickException) as ce: + _validate_header(['ID', 'Name'], ['ID', 'Name', 'Description'], 'Streams') + + assert str(ce.value) == 'The Streams sheet header does not contain `Description` header.' + + +def test_update_general_information_error(sample_stream_workbook, mocker, mocked_responses): + client = ConnectClient( + api_key='ApiKey X', + endpoint='https://localhost/public/v1', + use_specs=False, + ) + mocked_responses.add( + method='GET', + url='https://localhost/public/v1/billing/streams?' + 'eq(id,STR-7755-7115-2464)&select(context,samples,sources)&limit=1&offset=0', + json=[{'name': 'Name', 'description': 'Description', 'status': 'configuring'}], + ) + + mocked_responses.add( + method='PUT', + url='https://localhost/public/v1/billing/streams/STR-7755-7115-2464', + status=400, + ) + + results = [] + update_general_information( + client=client, + collection='billing', + stream_id='STR-7755-7115-2464', + sheet=sample_stream_workbook['General Information'], + results=results, + errors=[], + progress=mocker.MagicMock(), + ) + + sample_stream_workbook.close() + assert len(results) == 1 + assert results[0] == ('General information', 5, 0, 0, 0, 0, 5) + + +def test_update_transformations(sample_stream_workbook, mocker, mocked_responses): + client = ConnectClient( + api_key='ApiKey X', + endpoint='https://localhost/public/v1', + use_specs=False, + ) + + mocked_responses.add( + method='GET', + url='https://localhost/public/v1/billing/streams/STR-7755-7115-2464/transformations', + json=[ + {'id': 'STRA-774-870-217-449-001'}, + {'id': 'STRA-774-870-217-449-002'}, + ], + ) + mocked_responses.add( + method='GET', + url='https://localhost/public/v1/billing/streams/STR-7755-7115-2464' + '/transformations/STRA-774-870-217-449-001', + json={ + 'id': 'STRA-774-870-217-449-001', + 'settings': { + 'from': 'id', + 'regex': { + 'groups': {'1': {'name': 'first_name', 'type': 'string'}}, + 'pattern': '(?P\\w+)', + }, + }, + 'description': 'Old description', + 'position': 10000, + }, + ) + mocked_responses.add( + method='GET', + url='https://localhost/public/v1/billing/streams/STR-7755-7115-2464' + '/transformations/STRA-774-870-217-449-002', + json={ + 'id': 'STRA-774-870-217-449-002', + 'settings': { + 'additional_values': [], + 'from': 'position', + 'match_condition': True, + 'value': '200', + }, + 'description': 'edeeasd', + 'position': 20000, + }, + ) + mocked_responses.add( + method='PUT', + url='https://localhost/public/v1/billing/streams/STR-7755-7115-2464' + '/transformations/STRA-774-870-217-449-001', + status=200, + ) + + results = [] + errors = [] + update_transformations( + client=client, + collection='billing', + stream_id='STR-7755-7115-2464', + sheet=sample_stream_workbook['Transformations'], + results=results, + errors=errors, + progress=mocker.MagicMock(), + ) + + assert len(errors) == 0 + assert len(results) == 1 + assert results[0] == ('Transformations', 2, 0, 1, 0, 0, 0) + + +def test_update_transformations_not_exists(sample_stream_workbook, mocker, mocked_responses): + client = ConnectClient( + api_key='ApiKey X', + endpoint='https://localhost/public/v1', + use_specs=False, + ) + + mocked_responses.add( + method='GET', + url='https://localhost/public/v1/billing/streams/STR-7755-7115-2464' + '/transformations/STRA-774-870-217-449-001', + status=404, + ) + mocked_responses.add( + method='GET', + url='https://localhost/public/v1/billing/streams/STR-7755-7115-2464' + '/transformations/STRA-774-870-217-449-002', + status=404, + ) + mocked_responses.add( + method='GET', + url='https://localhost/public/v1/billing/streams/STR-7755-7115-2464/transformations', + json=[], + ) + + results = [] + errors = [] + update_transformations( + client=client, + collection='billing', + stream_id='STR-7755-7115-2464', + sheet=sample_stream_workbook['Transformations'], + results=results, + errors=errors, + progress=mocker.MagicMock(), + ) + + assert len(errors) == 2 + assert ( + 'The transformation STRA-774-870-217-449-001' + ' cannot be updated because it does not exist.' + ) in errors + assert ( + 'The transformation STRA-774-870-217-449-002' + ' cannot be updated because it does not exist.' + ) in errors + + assert results[0] == ('Transformations', 2, 0, 0, 0, 0, 2) + + +def test_update_transformations_with_errors(sample_stream_workbook, mocker, mocked_responses): + client = ConnectClient( + api_key='ApiKey X', + endpoint='https://localhost/public/v1', + use_specs=False, + ) + + mocked_responses.add( + method='GET', + url='https://localhost/public/v1/billing/streams/STR-7755-7115-2464' + '/transformations/STRA-774-870-217-449-001', + json={ + 'id': 'STRA-774-870-217-449-001', + 'settings': { + 'from': 'id', + 'regex': { + 'groups': {'1': {'name': 'first_name', 'type': 'string'}}, + 'pattern': '(?P\\w+)', + }, + }, + 'description': 'Old description', + 'position': 10000, + }, + ) + mocked_responses.add( + method='PUT', + url='https://localhost/public/v1/billing/streams/STR-7755-7115-2464' + '/transformations/STRA-774-870-217-449-001', + status=400, + ) + + mocked_responses.add( + method='GET', + url='https://localhost/public/v1/billing/streams/STR-7755-7115-2464' + '/transformations/STRA-774-870-217-449-002', + status=404, + ) + + mocked_responses.add( + method='GET', + url='https://localhost/public/v1/billing/streams/STR-7755-7115-2464/transformations', + json=[{'id': 'STRA-774-870-217-449-003'}], + ) + mocked_responses.add( + method='DELETE', + url='https://localhost/public/v1/billing/streams/STR-7755-7115-2464' + '/transformations/STRA-774-870-217-449-003', + status=500, + ) + + results = [] + errors = [] + update_transformations( + client=client, + collection='billing', + stream_id='STR-7755-7115-2464', + sheet=sample_stream_workbook['Transformations'], + results=results, + errors=errors, + progress=mocker.MagicMock(), + ) + + assert len(errors) == 3 + assert results[0] == ('Transformations', 2, 0, 0, 0, 0, 3) + + +def test_update_attachments_with_errors(sample_stream_workbook, mocker, mocked_responses): + client = ConnectClient( + api_key='ApiKey X', + endpoint='https://localhost/public/v1', + use_specs=False, + ) + + mocker.patch('connect.cli.plugins.commerce.utils.upload_attachment', side_effect=ClientError()) + + mocked_responses.add( + method='GET', + url='https://localhost/public/v1/media/folders/streams_attachments/STR-7755-7115-2464' + '/files?eq(id,ID)&limit=0&offset=0', + headers={'Content-Range': 'items 0-0/0'}, + status=200, + ) + mocked_responses.add( + method='GET', + url='https://localhost/public/v1/media/folders/streams_attachments/STR-7755-7115-2464' + '/files?eq(id,ID-EXISTS)&limit=0&offset=0', + headers={'Content-Range': 'items 0-0/0'}, + status=200, + ) + + mocked_responses.add( + method='GET', + url='https://localhost/public/v1/media/folders/streams_attachments/' + 'STR-7755-7115-2464/files', + json=[{'id': 'MLF-123'}], + ) + mocked_responses.add( + method='DELETE', + url='https://localhost/public/v1/media/folders/streams_attachments/STR-7755-7115-2464' + '/files/MLF-123', + status=400, + ) + + results = [] + errors = [] + update_attachments( + client=client, + stream_id='STR-7755-7115-2464', + sheet=sample_stream_workbook['Attachments'], + results=results, + errors=errors, + progress=mocker.MagicMock(), + ) + + assert len(errors) == 3 + assert results[0] == ('Attachment files', 2, 0, 0, 0, 0, 3)