Version 0.4 ready:

_ Working API version with initial integration with dataspace

dotenv

@@ -0,0 +1,9 @@
+# Client connector info:
+TOKEN=insert_data_provider_service_key_here
+API_KEY=insert_api_key_here
+ACCESS_URL=insert_access_url_here
+CONNECTOR_ID=insert_connector_id_here
+AGENT_ID=insert_agent_id_here
+
+# Metadata Broker info:
+METADATA_BROKER_URL=insert_metadata_broker_url_here

helpers/database_interactions.py

@@ -43,25 +43,16 @@ def connect_to_sqlite_db() -> (sqlite3.Connection, sqlite3.Cursor):
         )
         ''')
 
-        # Create the Individual_Costs, for main individual outputs
-        curs.execute('''
-        CREATE TABLE Individual_Costs (
-        id INTEGER PRIMARY KEY AUTOINCREMENT,
-        order_id TEXT,
-        meter_id TEXT,
-        individual_cost REAL,
-        individual_savings REAL,
-        FOREIGN KEY(order_id) REFERENCES Orders(order_id)
-        )
-        ''')
+
         # Create the Meter_Costs, for outputs that are dependent on the meter ID but not time-varying
         curs.execute('''
-        CREATE TABLE Meter_Costs (
+        CREATE TABLE Member_Costs (
         id INTEGER PRIMARY KEY AUTOINCREMENT,
         order_id TEXT,
         meter_id TEXT,
-        meter_cost REAL,
-        meter_savings REAL,
+        member_cost REAL,
+        member_cost_compensation REAL,
+        member_savings REAL,
         FOREIGN KEY(order_id) REFERENCES Orders(order_id)
         )
         ''')
@@ -73,13 +64,17 @@ def connect_to_sqlite_db() -> (sqlite3.Connection, sqlite3.Cursor):
         id INTEGER PRIMARY KEY AUTOINCREMENT,
         order_id TEXT,
         meter_id TEXT,
-        individual_cost REAL,
-        individual_savings REAL,
+        installation_cost REAL,
+        installation_cost_compensation REAL,
+        installation_savings REAL,
         installed_pv REAL,
+        pv_investment_cost REAL,
         installed_storage REAL,
+        storage_investment_cost REAL,
         total_pv REAL,
         total_storage REAL,
         contracted_power REAL,
+        contracted_power_cost REAL,
         retailer_exchange_costs REAL,
         sc_tariffs_costs REAL,
         FOREIGN KEY(order_id) REFERENCES Orders(order_id)

helpers/dataspace_interactions.py

@@ -3,6 +3,10 @@
 from helpers.other import haversine
 from datetime import datetime
 from typing import Union
+from collections import OrderedDict
+from dotenv import dotenv_values
+from loguru import logger
+from tsg_client.controllers import TSGController
 
 from schemas.input_schemas import (
 	MeterByArea,
@@ -19,6 +23,106 @@
 	MeterIDs
 )
 
+def load_dotenv() -> OrderedDict:
+	"""
+	Load environment variables
+	:return: dictionary with environment variables
+	"""
+	return dotenv_values('.env')
+
+
+def dataspace_connection(config: OrderedDict) -> TSGController:
+	"""
+	Set up a connection to the dataspace through a dedicated TSG connector
+	:param config: dictionary with environment variables
+	:return: the TSG connector
+	"""
+	# Set up the TSG connector
+	conn = TSGController(
+		api_key=config['API_KEY'],
+		connector_id=config['CONNECTOR_ID'],
+		access_url=config['ACCESS_URL'],
+		agent_id=config['AGENT_ID'],
+		metadata_broker_url='https://broker.enershare.dataspac.es/'
+
+	)
+
+	logger.info('Successfully connected to the TSG connector!')
+	logger.info(f'Connector info: vvv\n {conn}')  # print connection details
+
+	return conn
+
+
+
+def retrieve_data(conn: TSGController, config: OrderedDict) -> pd.DataFrame:
+	"""
+	Retrieve consumption data from CEVE through the dataspace
+	:param conn: TSG connector previously set up
+	:param config: dictionary with environment variables
+	:return: dataframe with retrieved data from CEVE
+	"""
+	# Get external connector info (self-descriptions):
+	EXTERNAL_CONNECTOR = {
+		'CONNECTOR_ID': 'urn:ids:enershare:connectors:connector-sentinel',
+		'ACCESS_URL': 'https://connector-sentinel.enershare.inesctec.pt',
+		'AGENT_ID': 'urn:ids:enershare:participants:INESCTEC-CPES'
+	}
+	# Get authorization token
+	AUTH = {'Authorization': 'Token {}'.format(config['TOKEN'])}
+
+	# Get the external connector's self-description
+	self_description = conn.get_connector_selfdescription(
+		access_url=EXTERNAL_CONNECTOR['ACCESS_URL'],
+		connector_id=EXTERNAL_CONNECTOR['CONNECTOR_ID'],
+		agent_id=EXTERNAL_CONNECTOR['AGENT_ID']
+	)
+
+	# Get the OpenAPI specs
+	api_version = '1.0.0'
+	open_api_specs = conn.get_openapi_specs(self_description, api_version)
+	endpoint = '/dataspace/inesctec/observed/ceve_living-lab/metering/energy'
+	data_app_agent_id = open_api_specs[0]['agent']
+
+	# Define the request parameters
+	user_id = '64b7080d1efc'
+	date_start = '2024-06-22 10:10'
+	date_end = '2024-06-22 10:30'
+	params = {
+		'shelly_id': user_id,
+		'phase': 'total',
+		'parameter': 'instant_active_power',
+		'start_date': date_start,
+		'end_date': date_end,
+	}
+
+	# Execute external OpenAPI request:
+	logger.info(f"""
+		Performing a request to:
+		- Agent ID: {data_app_agent_id}
+		- API Version: {api_version}
+		- Endpoint: {endpoint}
+		""")
+
+	response = conn.openapi_request(
+		headers=AUTH,
+		external_access_url=EXTERNAL_CONNECTOR['ACCESS_URL'],
+		data_app_agent_id=data_app_agent_id,
+		api_version=api_version,
+		endpoint=endpoint,
+		params=params,
+		method='get'
+	)
+
+	data = pd.DataFrame(response.json()['data'])
+
+	logger.info(f'> Connector {EXTERNAL_CONNECTOR["CONNECTOR_ID"]} RESPONSE:')
+	logger.info(f'Status Code: {response.status_code}')
+	logger.info(f'Retrieved data: vvv\n{data}')
+
+	return data
+
+
+
 
 DATETIME_FORMAT = '%Y-%m-%dT%H:%M:%SZ'
 

helpers/main_helpers.py

@@ -127,45 +127,23 @@ def milp_return_structure(cursor: sqlite3.Cursor,
 	# INDIVIDUAL COSTS #################################################################################################
 	# Retrieve the individual costs calculated for the order ID
 	cursor.execute('''
-		SELECT * FROM Individual_Costs WHERE order_id = ?
+		SELECT * FROM Member_Costs WHERE order_id = ?
 	''', (order_id,))
-	individual_costs = cursor.fetchall()
+	member_costs = cursor.fetchall()
 
 	# Convert to dataframe for easy manipulation
-	individual_costs_df = pd.DataFrame(individual_costs)
-	individual_costs_df.columns = ['index', 'order_id', 'meter_id', 'individual_cost', 'individual_savings']
-	del individual_costs_df['index']
-	del individual_costs_df['order_id']
+	member_costs_df = pd.DataFrame(member_costs)
+	member_costs_df.columns = ['index', 'order_id', 'meter_id', 'member_cost', 'member_cost_compensation', 'member_savings']
+	del member_costs_df['index']
+	del member_costs_df['order_id']
 
 	# Create final dictionary substructure
-	individual_investments_outputs_dict = {
-		'individual_costs': individual_costs_df.to_dict('records')
+	member_costs_outputs_dict = {
+		'member_costs': member_costs_df.to_dict('records')
 	}
 
 	# Update the return dictionary
-	milp_return.update(individual_investments_outputs_dict)
-
-
-	# METER COSTS #################################################################################################
-	# Retrieve the individual costs calculated for the order ID
-	cursor.execute('''
-		SELECT * FROM Meter_Costs WHERE order_id = ?
-	''', (order_id,))
-	meter_costs = cursor.fetchall()
-
-	# Convert to dataframe for easy manipulation
-	meter_costs_df = pd.DataFrame(meter_costs)
-	meter_costs_df.columns = ['index', 'order_id', 'meter_id', 'meter_cost', 'meter_savings']
-	del meter_costs_df['index']
-	del meter_costs_df['order_id']
-
-	# Create final dictionary substructure
-	meter_investments_outputs_dict = {
-		'meter_costs': meter_costs_df.to_dict('records')
-	}
-
-	# Update the return dictionary
-	milp_return.update(meter_investments_outputs_dict)
+	milp_return.update(member_costs_outputs_dict)
 
 
 
@@ -178,18 +156,19 @@ def milp_return_structure(cursor: sqlite3.Cursor,
 
 	# Convert to dataframe for easy manipulation
 	meter_investments_outputs_df = pd.DataFrame(meter_investments_outputs)
-	meter_investments_outputs_df.columns = ['index', 'order_id', 'meter_id', 'individual_cost', 'individual_savings', 'installed_pv',
-				'installed_storage', 'total_pv', 'total_storage', 'contracted_power', 'retailer_exchange_costs', 'sc_tariffs_costs']
+	meter_investments_outputs_df.columns = ['index', 'order_id', 'meter_id', 'installation_cost', 'installation_cost_compensation', 'installation_savings',
+				 'installed_pv', 'pv_investment_cost', 'installed_storage', 'storage_investment_cost', 'total_pv', 'total_storage',
+				 'contracted_power', 'contracted_power_cost', 'retailer_exchange_costs', 'sc_tariffs_costs']
 	del meter_investments_outputs_df['index']
 	del meter_investments_outputs_df['order_id']
 
 	# Create final dictionary substructure
-	individual_costs_dict = {
+	meter_investments_outputs_dict = {
 		'meter_investments_outputs': meter_investments_outputs_df.to_dict('records')
 	}
 
 	# Update the return dictionary
-	milp_return.update(individual_costs_dict)
+	milp_return.update(meter_investments_outputs_dict)
 
 
 

main.py

@@ -32,6 +32,13 @@
 	TimeseriesDataNotFound,
 	MeterIDs
 )
+from helpers.dataspace_interactions import (
+	dataspace_connection,
+	load_dotenv,
+	retrieve_data
+)
+
+
 
 # from rec_sizing.custom_types.collective_milp_pool_types import (
 # 	BackpackCollectivePoolDict,
@@ -45,6 +52,17 @@
 	version='0.2.0'
 )
 
+# Set up logging
+set_stdout_logger()
+app.state.handler = set_logfile_handler('logs')
+
+# DATASPACE INTERACTIONS ###############################################################################################
+# Load environment variables
+config = load_dotenv()
+# Connect to dataspace
+dataspace_connection = dataspace_connection(config)
+# Retrieve CEVE data
+CEVE_data = retrieve_data(dataspace_connection, config)
 
 # Runs when the API is started: set loggers and create / connect to SQLite database ####################################
 @app.on_event('startup')

requirements.txt

@@ -1,6 +1,98 @@
+alabaster==0.7.16
+annotated-types==0.6.0
+anyio==3.7.1
+argcomplete==3.4.0
+Babel==2.14.0
+bcrypt==4.1.2
+build==1.2.1
+CacheControl==0.14.0
+certifi==2024.2.2
+charset-normalizer==3.3.2
+cleo==2.1.0
+click==8.1.7
+colorama==0.4.6
+contourpy==1.2.0
+crashtest==0.4.1
+cycler==0.12.1
+distlib==0.3.8
+docutils==0.20.1
+dulwich==0.21.7
 fastapi==0.104.1
+fastjsonschema==2.20.0
+filelock==3.15.3
+fonttools==4.49.0
+h11==0.14.0
+idna==3.7
+imagesize==1.4.1
+iniconfig==2.0.0
+installer==0.7.0
+jaraco.classes==3.4.0
+Jinja2==3.1.4
+joblib==1.3.2
+keyring==24.3.1
+kiwisolver==1.4.5
+loguru==0.7.2
+markdown-it-py==3.0.0
+MarkupSafe==2.1.5
+matplotlib==3.8.3
+mdit-py-plugins==0.4.0
+mdurl==0.1.2
+more-itertools==10.3.0
+msgpack==1.0.8
+myst-parser==2.0.0
+numpy==1.26.4
+packaging==24.0
+pandas==2.1.4
+pexpect==4.9.0
+pillow==10.2.0
+pipx==1.6.0
+pkginfo==1.11.1
+platformdirs==4.2.2
+pluggy==1.5.0
+poetry==1.8.3
+poetry-core==1.9.0
+poetry-plugin-export==1.8.0
+ptyprocess==0.7.0
+PuLP==2.7.0
 pydantic==2.5.2
-pydantic-extra-types==2.1.0
+pydantic_core==2.14.5
+Pygments==2.17.2
+pyparsing==3.1.2
+pyproject_hooks==1.1.0
+pytest==8.1.1
+python-dateutil==2.9.0.post0
+python-dotenv==1.0.1
+pytz==2024.1
+pywin32-ctypes==0.2.2
+PyYAML==6.0.1
+rapidfuzz==3.9.3
+requests==2.32.2
+requests-toolbelt==1.0.0
+ruff==0.3.7
+setuptools==70.1.0
+shellingham==1.5.4
+six==1.16.0
+sniffio==1.3.1
+snowballstemmer==2.2.0
+Sphinx==7.3.7
+sphinx-rtd-theme==2.0.0
+sphinxcontrib-applehelp==1.0.8
+sphinxcontrib-devhelp==1.0.6
+sphinxcontrib-htmlhelp==2.0.5
+sphinxcontrib-jquery==4.1
+sphinxcontrib-jsmath==1.0.1
+sphinxcontrib-qthelp==1.0.7
+sphinxcontrib-serializinghtml==1.1.10
+starlette==0.27.0
+tomlkit==0.12.5
+trove-classifiers==2024.5.22
+-e git+https://github.com/CPES-Power-and-Energy-Systems/tsg-client.git@d1f5c2a6c9466b03703d6aa68b8fc2f0b6f14cbd#egg=tsg_client
+typing_extensions==4.10.0
+tzdata==2024.1
+urllib3==2.2.1
+userpath==1.9.2
 uvicorn==0.24.0.post1
-loguru~=0.7.2
-pandas~=2.1.4
+virtualenv==20.26.2
+wheel==0.41.2
+win32-setctime==1.1.0
+