From bb1e9db52d8e88ff3e911ec72b4ff5104b7a056c Mon Sep 17 00:00:00 2001 From: "armando.moreno" Date: Thu, 27 Jun 2024 14:55:37 +0100 Subject: [PATCH] Version 0.4 ready: _ Working API version with initial integration with dataspace --- dotenv | 9 +++ helpers/database_interactions.py | 27 ++++---- helpers/dataspace_interactions.py | 104 ++++++++++++++++++++++++++++++ helpers/main_helpers.py | 49 ++++---------- main.py | 18 ++++++ requirements.txt | 98 +++++++++++++++++++++++++++- schemas/output_schemas.py | 9 +-- threads/run_milp_thread.py | 76 +++++++++++++--------- 8 files changed, 300 insertions(+), 90 deletions(-) create mode 100644 dotenv diff --git a/dotenv b/dotenv new file mode 100644 index 0000000..d4274c2 --- /dev/null +++ b/dotenv @@ -0,0 +1,9 @@ +# Client connector info: +TOKEN=insert_data_provider_service_key_here +API_KEY=insert_api_key_here +ACCESS_URL=insert_access_url_here +CONNECTOR_ID=insert_connector_id_here +AGENT_ID=insert_agent_id_here + +# Metadata Broker info: +METADATA_BROKER_URL=insert_metadata_broker_url_here \ No newline at end of file diff --git a/helpers/database_interactions.py b/helpers/database_interactions.py index e3ba8b8..e6835f3 100644 --- a/helpers/database_interactions.py +++ b/helpers/database_interactions.py @@ -43,25 +43,16 @@ def connect_to_sqlite_db() -> (sqlite3.Connection, sqlite3.Cursor): ) ''') - # Create the Individual_Costs, for main individual outputs - curs.execute(''' - CREATE TABLE Individual_Costs ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - order_id TEXT, - meter_id TEXT, - individual_cost REAL, - individual_savings REAL, - FOREIGN KEY(order_id) REFERENCES Orders(order_id) - ) - ''') + # Create the Meter_Costs, for outputs that are dependent on the meter ID but not time-varying curs.execute(''' - CREATE TABLE Meter_Costs ( + CREATE TABLE Member_Costs ( id INTEGER PRIMARY KEY AUTOINCREMENT, order_id TEXT, meter_id TEXT, - meter_cost REAL, - meter_savings REAL, + member_cost REAL, + member_cost_compensation REAL, + member_savings REAL, FOREIGN KEY(order_id) REFERENCES Orders(order_id) ) ''') @@ -73,13 +64,17 @@ def connect_to_sqlite_db() -> (sqlite3.Connection, sqlite3.Cursor): id INTEGER PRIMARY KEY AUTOINCREMENT, order_id TEXT, meter_id TEXT, - individual_cost REAL, - individual_savings REAL, + installation_cost REAL, + installation_cost_compensation REAL, + installation_savings REAL, installed_pv REAL, + pv_investment_cost REAL, installed_storage REAL, + storage_investment_cost REAL, total_pv REAL, total_storage REAL, contracted_power REAL, + contracted_power_cost REAL, retailer_exchange_costs REAL, sc_tariffs_costs REAL, FOREIGN KEY(order_id) REFERENCES Orders(order_id) diff --git a/helpers/dataspace_interactions.py b/helpers/dataspace_interactions.py index c18447b..df5efb0 100644 --- a/helpers/dataspace_interactions.py +++ b/helpers/dataspace_interactions.py @@ -3,6 +3,10 @@ from helpers.other import haversine from datetime import datetime from typing import Union +from collections import OrderedDict +from dotenv import dotenv_values +from loguru import logger +from tsg_client.controllers import TSGController from schemas.input_schemas import ( MeterByArea, @@ -19,6 +23,106 @@ MeterIDs ) +def load_dotenv() -> OrderedDict: + """ + Load environment variables + :return: dictionary with environment variables + """ + return dotenv_values('.env') + + +def dataspace_connection(config: OrderedDict) -> TSGController: + """ + Set up a connection to the dataspace through a dedicated TSG connector + :param config: dictionary with environment variables + :return: the TSG connector + """ + # Set up the TSG connector + conn = TSGController( + api_key=config['API_KEY'], + connector_id=config['CONNECTOR_ID'], + access_url=config['ACCESS_URL'], + agent_id=config['AGENT_ID'], + metadata_broker_url='https://broker.enershare.dataspac.es/' + + ) + + logger.info('Successfully connected to the TSG connector!') + logger.info(f'Connector info: vvv\n {conn}') # print connection details + + return conn + + + +def retrieve_data(conn: TSGController, config: OrderedDict) -> pd.DataFrame: + """ + Retrieve consumption data from CEVE through the dataspace + :param conn: TSG connector previously set up + :param config: dictionary with environment variables + :return: dataframe with retrieved data from CEVE + """ + # Get external connector info (self-descriptions): + EXTERNAL_CONNECTOR = { + 'CONNECTOR_ID': 'urn:ids:enershare:connectors:connector-sentinel', + 'ACCESS_URL': 'https://connector-sentinel.enershare.inesctec.pt', + 'AGENT_ID': 'urn:ids:enershare:participants:INESCTEC-CPES' + } + # Get authorization token + AUTH = {'Authorization': 'Token {}'.format(config['TOKEN'])} + + # Get the external connector's self-description + self_description = conn.get_connector_selfdescription( + access_url=EXTERNAL_CONNECTOR['ACCESS_URL'], + connector_id=EXTERNAL_CONNECTOR['CONNECTOR_ID'], + agent_id=EXTERNAL_CONNECTOR['AGENT_ID'] + ) + + # Get the OpenAPI specs + api_version = '1.0.0' + open_api_specs = conn.get_openapi_specs(self_description, api_version) + endpoint = '/dataspace/inesctec/observed/ceve_living-lab/metering/energy' + data_app_agent_id = open_api_specs[0]['agent'] + + # Define the request parameters + user_id = '64b7080d1efc' + date_start = '2024-06-22 10:10' + date_end = '2024-06-22 10:30' + params = { + 'shelly_id': user_id, + 'phase': 'total', + 'parameter': 'instant_active_power', + 'start_date': date_start, + 'end_date': date_end, + } + + # Execute external OpenAPI request: + logger.info(f""" + Performing a request to: + - Agent ID: {data_app_agent_id} + - API Version: {api_version} + - Endpoint: {endpoint} + """) + + response = conn.openapi_request( + headers=AUTH, + external_access_url=EXTERNAL_CONNECTOR['ACCESS_URL'], + data_app_agent_id=data_app_agent_id, + api_version=api_version, + endpoint=endpoint, + params=params, + method='get' + ) + + data = pd.DataFrame(response.json()['data']) + + logger.info(f'> Connector {EXTERNAL_CONNECTOR["CONNECTOR_ID"]} RESPONSE:') + logger.info(f'Status Code: {response.status_code}') + logger.info(f'Retrieved data: vvv\n{data}') + + return data + + + DATETIME_FORMAT = '%Y-%m-%dT%H:%M:%SZ' diff --git a/helpers/main_helpers.py b/helpers/main_helpers.py index 0b032e9..7585dfc 100644 --- a/helpers/main_helpers.py +++ b/helpers/main_helpers.py @@ -127,45 +127,23 @@ def milp_return_structure(cursor: sqlite3.Cursor, # INDIVIDUAL COSTS ################################################################################################# # Retrieve the individual costs calculated for the order ID cursor.execute(''' - SELECT * FROM Individual_Costs WHERE order_id = ? + SELECT * FROM Member_Costs WHERE order_id = ? ''', (order_id,)) - individual_costs = cursor.fetchall() + member_costs = cursor.fetchall() # Convert to dataframe for easy manipulation - individual_costs_df = pd.DataFrame(individual_costs) - individual_costs_df.columns = ['index', 'order_id', 'meter_id', 'individual_cost', 'individual_savings'] - del individual_costs_df['index'] - del individual_costs_df['order_id'] + member_costs_df = pd.DataFrame(member_costs) + member_costs_df.columns = ['index', 'order_id', 'meter_id', 'member_cost', 'member_cost_compensation', 'member_savings'] + del member_costs_df['index'] + del member_costs_df['order_id'] # Create final dictionary substructure - individual_investments_outputs_dict = { - 'individual_costs': individual_costs_df.to_dict('records') + member_costs_outputs_dict = { + 'member_costs': member_costs_df.to_dict('records') } # Update the return dictionary - milp_return.update(individual_investments_outputs_dict) - - - # METER COSTS ################################################################################################# - # Retrieve the individual costs calculated for the order ID - cursor.execute(''' - SELECT * FROM Meter_Costs WHERE order_id = ? - ''', (order_id,)) - meter_costs = cursor.fetchall() - - # Convert to dataframe for easy manipulation - meter_costs_df = pd.DataFrame(meter_costs) - meter_costs_df.columns = ['index', 'order_id', 'meter_id', 'meter_cost', 'meter_savings'] - del meter_costs_df['index'] - del meter_costs_df['order_id'] - - # Create final dictionary substructure - meter_investments_outputs_dict = { - 'meter_costs': meter_costs_df.to_dict('records') - } - - # Update the return dictionary - milp_return.update(meter_investments_outputs_dict) + milp_return.update(member_costs_outputs_dict) @@ -178,18 +156,19 @@ def milp_return_structure(cursor: sqlite3.Cursor, # Convert to dataframe for easy manipulation meter_investments_outputs_df = pd.DataFrame(meter_investments_outputs) - meter_investments_outputs_df.columns = ['index', 'order_id', 'meter_id', 'individual_cost', 'individual_savings', 'installed_pv', - 'installed_storage', 'total_pv', 'total_storage', 'contracted_power', 'retailer_exchange_costs', 'sc_tariffs_costs'] + meter_investments_outputs_df.columns = ['index', 'order_id', 'meter_id', 'installation_cost', 'installation_cost_compensation', 'installation_savings', + 'installed_pv', 'pv_investment_cost', 'installed_storage', 'storage_investment_cost', 'total_pv', 'total_storage', + 'contracted_power', 'contracted_power_cost', 'retailer_exchange_costs', 'sc_tariffs_costs'] del meter_investments_outputs_df['index'] del meter_investments_outputs_df['order_id'] # Create final dictionary substructure - individual_costs_dict = { + meter_investments_outputs_dict = { 'meter_investments_outputs': meter_investments_outputs_df.to_dict('records') } # Update the return dictionary - milp_return.update(individual_costs_dict) + milp_return.update(meter_investments_outputs_dict) diff --git a/main.py b/main.py index 22de931..78686b3 100644 --- a/main.py +++ b/main.py @@ -32,6 +32,13 @@ TimeseriesDataNotFound, MeterIDs ) +from helpers.dataspace_interactions import ( + dataspace_connection, + load_dotenv, + retrieve_data +) + + # from rec_sizing.custom_types.collective_milp_pool_types import ( # BackpackCollectivePoolDict, @@ -45,6 +52,17 @@ version='0.2.0' ) +# Set up logging +set_stdout_logger() +app.state.handler = set_logfile_handler('logs') + +# DATASPACE INTERACTIONS ############################################################################################### +# Load environment variables +config = load_dotenv() +# Connect to dataspace +dataspace_connection = dataspace_connection(config) +# Retrieve CEVE data +CEVE_data = retrieve_data(dataspace_connection, config) # Runs when the API is started: set loggers and create / connect to SQLite database #################################### @app.on_event('startup') diff --git a/requirements.txt b/requirements.txt index 3aa0ea9..123fc99 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,98 @@ +alabaster==0.7.16 +annotated-types==0.6.0 +anyio==3.7.1 +argcomplete==3.4.0 +Babel==2.14.0 +bcrypt==4.1.2 +build==1.2.1 +CacheControl==0.14.0 +certifi==2024.2.2 +charset-normalizer==3.3.2 +cleo==2.1.0 +click==8.1.7 +colorama==0.4.6 +contourpy==1.2.0 +crashtest==0.4.1 +cycler==0.12.1 +distlib==0.3.8 +docutils==0.20.1 +dulwich==0.21.7 fastapi==0.104.1 +fastjsonschema==2.20.0 +filelock==3.15.3 +fonttools==4.49.0 +h11==0.14.0 +idna==3.7 +imagesize==1.4.1 +iniconfig==2.0.0 +installer==0.7.0 +jaraco.classes==3.4.0 +Jinja2==3.1.4 +joblib==1.3.2 +keyring==24.3.1 +kiwisolver==1.4.5 +loguru==0.7.2 +markdown-it-py==3.0.0 +MarkupSafe==2.1.5 +matplotlib==3.8.3 +mdit-py-plugins==0.4.0 +mdurl==0.1.2 +more-itertools==10.3.0 +msgpack==1.0.8 +myst-parser==2.0.0 +numpy==1.26.4 +packaging==24.0 +pandas==2.1.4 +pexpect==4.9.0 +pillow==10.2.0 +pipx==1.6.0 +pkginfo==1.11.1 +platformdirs==4.2.2 +pluggy==1.5.0 +poetry==1.8.3 +poetry-core==1.9.0 +poetry-plugin-export==1.8.0 +ptyprocess==0.7.0 +PuLP==2.7.0 pydantic==2.5.2 -pydantic-extra-types==2.1.0 +pydantic_core==2.14.5 +Pygments==2.17.2 +pyparsing==3.1.2 +pyproject_hooks==1.1.0 +pytest==8.1.1 +python-dateutil==2.9.0.post0 +python-dotenv==1.0.1 +pytz==2024.1 +pywin32-ctypes==0.2.2 +PyYAML==6.0.1 +rapidfuzz==3.9.3 +requests==2.32.2 +requests-toolbelt==1.0.0 +ruff==0.3.7 +setuptools==70.1.0 +shellingham==1.5.4 +six==1.16.0 +sniffio==1.3.1 +snowballstemmer==2.2.0 +Sphinx==7.3.7 +sphinx-rtd-theme==2.0.0 +sphinxcontrib-applehelp==1.0.8 +sphinxcontrib-devhelp==1.0.6 +sphinxcontrib-htmlhelp==2.0.5 +sphinxcontrib-jquery==4.1 +sphinxcontrib-jsmath==1.0.1 +sphinxcontrib-qthelp==1.0.7 +sphinxcontrib-serializinghtml==1.1.10 +starlette==0.27.0 +tomlkit==0.12.5 +trove-classifiers==2024.5.22 +-e git+https://github.com/CPES-Power-and-Energy-Systems/tsg-client.git@d1f5c2a6c9466b03703d6aa68b8fc2f0b6f14cbd#egg=tsg_client +typing_extensions==4.10.0 +tzdata==2024.1 +urllib3==2.2.1 +userpath==1.9.2 uvicorn==0.24.0.post1 -loguru~=0.7.2 -pandas~=2.1.4 +virtualenv==20.26.2 +wheel==0.41.2 +win32-setctime==1.1.0 + diff --git a/schemas/output_schemas.py b/schemas/output_schemas.py index 4328b78..44582c8 100644 --- a/schemas/output_schemas.py +++ b/schemas/output_schemas.py @@ -91,9 +91,9 @@ class IndividualCosts(BaseModel): examples=['Meter#1'] ) individual_cost: float = Field( - description='The total cost (operation + investment) for the optimization horizon calculated for ' - 'the individual/member, without considering the cost for degradation of the BESS, in €. ' - 'It includes the costs of the shared assets' + description='The total cost (operation + investment) for the optimization horizon calculated for the individual/member, ' + 'without considering the cost for degradation of the BESS, in €. It includes' + 'the costs of the shared assets' ) individual_savings: float = Field( description='Total savings obtained for that meter ID, in €.
' @@ -104,7 +104,6 @@ class IndividualCosts(BaseModel): 'meter.' ) - class MeterCosts(BaseModel): meter_id: str = Field( description='The string that identifies the meter of the REC.', @@ -123,7 +122,6 @@ class MeterCosts(BaseModel): 'meter.' ) - class InvestmentsPerMeter(BaseModel): meter_id: str = Field( description='The string that identifies the meter of the REC.', @@ -169,7 +167,6 @@ class InvestmentsPerMeter(BaseModel): description='The total grid access costs when self-consuming in the REC, in €.' ) - class InputsPerMeterAndDatetime(BaseModel): meter_id: str = Field( description='A string that unequivocally identifies a meter of the REC.', diff --git a/threads/run_milp_thread.py b/threads/run_milp_thread.py index 9ef1db5..f30dcbc 100644 --- a/threads/run_milp_thread.py +++ b/threads/run_milp_thread.py @@ -17,6 +17,7 @@ def run_dual_thread(user_params: Union[SizingInputs, SizingInputsWithShared], logger.info('[THREAD] Fetching data from dataspace.') data_df, list_of_datetimes, missing_ids, missing_dts = fetch_dataspace(user_params) meter_ids = set(data_df['meter_id']) + print('meter_ids ', meter_ids) print('data_df: \n', data_df) print('missing_ids', missing_ids) print('missing_dts', missing_dts) @@ -56,9 +57,19 @@ def run_dual_thread(user_params: Union[SizingInputs, SizingInputsWithShared], results = run_pre_collective_pool_milp(inputs) print('results', results.keys()) print('results', results['e_alc']) - # Create the INPUTS_OWNERSHIP_PP dictionary INPUTS_OWNERSHIP_PP = {'ownership': {}} + if hasattr(user_params, 'shared_meter_id'): + for meter in [i for i in meter_ids if i != user_params.shared_meter_id]: + # Add the percentage for the meter + INPUTS_OWNERSHIP_PP['ownership'][meter] = {meter: 1.0} + else: + for meter in meter_ids: + # Add the percentage for the meter + INPUTS_OWNERSHIP_PP['ownership'][meter] = {meter: 1.0} + + print('''INPUTS_OWNERSHIP_PP['ownership'][meter] = {meter: 1.0}''', INPUTS_OWNERSHIP_PP) + if hasattr(user_params, 'shared_meter_id'): # Add ownership for each meter shared_meter = user_params.shared_meter_id @@ -66,16 +77,14 @@ def run_dual_thread(user_params: Union[SizingInputs, SizingInputsWithShared], meter_id = ownership.meter_id percentage = ownership.percentage - # Add the percentage for the meter - INPUTS_OWNERSHIP_PP['ownership'][meter_id] = {meter_id: 1.0} # Add shared meter ownership - shared_meter_ownership = {} for i, ownership in enumerate(user_params.ownerships, start=1): meter_id = ownership.meter_id percentage = ownership.percentage # Calculate the shared meter ownership percentage + shared_meter_ownership={} shared_meter_ownership[meter_id] = percentage / 100 shared_meter_key = user_params.shared_meter_id @@ -110,45 +119,52 @@ def run_dual_thread(user_params: Union[SizingInputs, SizingInputsWithShared], results['milp_status'], round(results_pp['obj_value'], 2) )) - - for meter_id in shared_meter_ownership.keys(): - curs.execute(''' - INSERT INTO Individual_Costs (order_id, meter_id, individual_cost, individual_savings) - VALUES (?, ?, ?, ?) - ''', ( - id_order, - meter_id, - round(results_pp['installation_cost_compensations'][meter_id] - + shared_meter_ownership[meter_id] - * results_pp['installation_cost_compensations'][shared_meter], 2), - 0 - )) + #todo: this + if hasattr(user_params, 'shared_meter_id'): + for meter_id in [i for i in meter_ids if i != user_params.shared_meter_id]: + curs.execute(''' + INSERT INTO Member_Costs (order_id, meter_id, member_cost, member_cost_compensation, member_savings) + VALUES (?, ?, ?, ?, ?) + ''', ( + id_order, + meter_id, + round(results_pp['member_cost'][meter_id],2), + round(results_pp['member_cost_compensations'][meter_id],2), + 0 + )) + else: + for meter_id in meter_ids: + curs.execute(''' + INSERT INTO Member_Costs (order_id, meter_id, member_cost, member_cost_compensation, member_savings) + VALUES (?, ?, ?, ?, ?) + ''', ( + id_order, + meter_id, + round(results_pp['member_cost'][meter_id], 2), + round(results_pp['member_cost_compensations'][meter_id], 2), + 0 + )) for meter_id in meter_ids: curs.execute(''' - INSERT INTO Meter_Costs (order_id, meter_id, meter_cost, meter_savings) - VALUES (?, ?, ?, ?) - ''', ( - id_order, - meter_id, - round(results_pp['installation_cost_compensations'][meter_id], 2), - 0 - )) - - curs.execute(''' - INSERT INTO Meter_Investment_Outputs (order_id, meter_id, individual_cost, individual_savings, installed_pv, - installed_storage, total_pv, total_storage, contracted_power, retailer_exchange_costs, sc_tariffs_costs) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + INSERT INTO Meter_Investment_Outputs (order_id, meter_id, installation_cost, installation_cost_compensation, installation_savings, + installed_pv, pv_investment_cost, installed_storage, storage_investment_cost, total_pv, total_storage, + contracted_power, contracted_power_cost, retailer_exchange_costs, sc_tariffs_costs) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ''', ( id_order, meter_id, results_pp['installation_cost_compensations'][meter_id], + results_pp['installation_cost_compensations'][meter_id], 0, results_pp['p_gn_new'][meter_id], + results_pp['PV_investments_cost'][meter_id], results_pp['e_bn_new'][meter_id], + results_pp['batteries_investments_cost'][meter_id], results_pp['p_gn_total'][meter_id], results_pp['e_bn_total'][meter_id], results_pp['p_cont'][meter_id], + results_pp['contractedpower_cost'][meter_id], sum(results_pp['e_sup'][meter_id]), sum(results_pp['e_slc_pool'][meter_id]) ))