Changes

	modified:   README.md
	modified:   greenbutton.json
        Added util_format
        Two utilities avilable sce-tou & pep
	modified:   greenbutton_import.py
        Moved csv parsing into module utilparse.py
        All new csv formats will be in that module
	modified:   requirements.txt
        cleanup of non-used modules
	new file:   utilparse.py
        Added new tags (day and year) for better dashboards without using complex date queries
           example, use the tags Saturday & Sunday for just weekend
        Changed measurement to energy - kwhrs was ambigous for a measurement
        Changed field 'value' to 'kwr'
        Added new parsing format for Pepco utility
            Pepco provides hoursly cost so added that as a field

Author: gauthig

README.md

@@ -4,7 +4,7 @@
 
 # Green button import to Influxdb
 Parse SCE Green Button download files only for now. 
-<br>Green button data from utilities is not always the same format so this is tuned to SCE's net metering format.  Yes, Even SCE has several formats based on customer plan.
+<br>Green button data from utilities is not always the same format , two utilities rae now inlcuded.
 If someone provides sample data for other green button formats (including SCE non Net Metering) I will incorporate.  
 
 Looking for contributors for other utility formats, code enhancements, bugs, documentation ....
@@ -15,7 +15,9 @@ If you would like to contribute anything including just identifying enhancements
 Features
 - [X] Parse SCE NetMetering version of Green Button CSV file - Solar
  File is grouped by day by Received (Sent to Utility) or Delivered (from Utiliy)
-- [X] Parse SCE non-Netmetering version of Green Button CSV file - Normal Residential Customers without solar
+- [X] Provide feature to load different utility formats 
+-       SCE (Southern Cal.) Time of use / Net Metering , Pepco (Maryland)
+- [ ] Parse SCE non-Netmetering version of Green Button CSV file - Normal Residential Customers without solar
  File is group by day and has Consumed header tag
 - [X] Send data to influxdb (v1, non-ssl) - should be on private network and not public accessible due to no SSL
 - [X] Grafana dashboard to analyze Energy Delivery and Generation

__pycache__/utilparse.cpython-38.pyc

@@ -1,11 +1,12 @@
 {
-        "host": "127.0.0.1",
+        "util_format": "sce-tou",
+        "host": "influxdbv1.mydomain.org",
         "port": 8086,
-        "user": "admin",
-        "password": "testadmin",
+        "user": "test",
+        "password": "test",
         "dbname": "energy",
         "ssl_enable": false,
         "ssl_verify": false,
-	"timezone": "America/Los_Angeles",
+        "influx_ver": "V1",
         "batchsize": 10000
  }

greenbutton.json

@@ -13,15 +13,16 @@
 # Input - SCE Green Data (TOU Style)
 # Output - Inluxdb mesurements  (V1)
 #
+# utilparse.py must be in same directory as this program
 
 __author__ = 'Garrett Gauthier'
 __copyright__ = 'Copyright 2022, Garrett Gauthier'
 __author__ = 'Garrett Gauthier'
 __copyright__ = 'Copyright 2022, Garrett Gauthier'
 __credits__ = ['Garrett Gauthier', 'Others soon?']
 __license__ = 'GPL'
-__version__ = '1.1'
-__versiondate__ = '08/18/2022'
+__version__ = '1.5'
+__versiondate__ = '12/29/2022'
 __maintainer__ = 'gauthig@github'
 __github__ = 'https://github.com/gauthig/scegreenbutton'
 __email__ = 'garrett-g@outlook.com'
@@ -37,6 +38,7 @@
 import argparse
 import json
 import pytz
+from utilparse import parse_data
 
 
 metricsout = []
@@ -47,70 +49,6 @@ def get_config_value(key, default_value):
         return config[key]
     return default_value
 
-def parse_data(input_file, verbose):
-    if verbose:
-        print('Starting parse_data')
-
-
-
-    point = []
-    tag_values = []
-    rows_generated = 0
-    rows_delivered = 0
-    tag = ''
-    pmult = 0
-    dt_format = '%Y-%m-%d %H:%M:%S'
-    infile = open(input_file, mode='r')
-    csv_reader = csv.reader(infile, delimiter=',')
-    if verbose:
-        print('File: ', infile)
-
-    for row in csv_reader:
-        if len(row) > 0:
-            if 'Received' in row[0]:
-                tag = 'generated'
-            elif 'Delivered' in row[0]:
-                tag = 'delivered'
-            elif 'Consumption' in row[0]:
-                tag = 'delivered'
-            elif 'to' in row[0]:
-                sce_timestamp = row[0].split('to')
-                # SCE adds non-ASCII charter before the to field, need to strip
-                sce_timestamp = [
-                    item.replace('\xa0', '') for item in sce_timestamp
-                ]
-                dt_local = datetime.strptime(sce_timestamp[0], dt_format)
-                dt_utc = dt_local.astimezone(pytz.UTC)
-                dt_utc = dt_utc.strftime("%Y-%m-%d %H:%M:%S")
-                # use local time instead of UTC as you want customer month
-                dt_month = dt_local.strftime('%B')
-                if tag == 'generated':
-                    rows_generated = rows_generated + 1
-                    pmult = -1
-                elif tag == 'delivered':
-                    rows_delivered = rows_delivered + 1
-                    pmult = 1
-
-                point = {
-                    "measurement": "kwhrs",
-                    "tags": {
-                        "type": tag,
-                        "month": dt_month
-                    },
-                    "time": dt_utc,
-                    "fields": {
-                        "value": float(row[1]) * pmult
-                    }
-                }
-
-                
-                if verbose:
-                    print(point)
-
-                metricsout.append(point)
-
-    return (rows_delivered, rows_generated)
-
 
 def write_csv():
     textout = ''
@@ -173,9 +111,7 @@ def send_data(hostname, port, user, password,
 
     parser = \
         argparse.ArgumentParser(description="""Loads Green Button csv file
-        and send formated results to influxdb. 
-        Used for Net Metering format only (solar)"""
-                               )
+        and send formated results to influxdb. """ )
     parser.add_argument('--version',
                         help='display version number',
                         action='store_true')
@@ -216,11 +152,18 @@ def send_data(hostname, port, user, password,
     )
     args = parser.parse_args()
 
+    if config['util_format'] not in ('sce-tou','pep'):
+        print('\n*** json file error ***')
+        print('Only valid Utility formats are: sce-tou, pep.')
+        print('Current json value for util_format is ', config['util_format'])
+        print('Change util_format or create an enhancement request in github to create a new utility format')
+        sys.exit()
+
     if args.version:
         print('sceinfluxdb.py - version', __version__)
-        sys.exit()
+        sys.exit(-1)
 
-    (delivered, generated) = parse_data(args.file, args.verbose)
+    (delivered, generated, metricsout) = parse_data(args.file, args.verbose, config['util_format'], metricsout)
 
     if args.csv:
         write_csv()

greenbutton_import.py

@@ -1,4 +1,6 @@
 pytz
 influxdb
 argparse
-datetime
+datetime
+
+

requirements.txt

@@ -0,0 +1,140 @@
+
+#utilparse.py
+#module to parse all metrics based on utility format
+#Notes
+#  UNIX timestamp for influxdb converted to UTC - grafana will convert back to local
+#  tags are using local (input file) time zone
+#  
+#  Interval times are end times only, start is assumed in a timebased database
+
+from datetime import datetime
+import sys
+import csv
+import pytz
+import re
+
+rows_generated = 0
+rows_delivered = 0
+
+pmult = 0
+
+def parse_data(input_file, verbose, util_format, metricsout):
+    if verbose:
+        print('Starting parse_data')
+
+    infile = open(input_file, mode='r')
+    csv_reader = csv.reader(infile, delimiter=',')
+    if verbose:
+        print('File: ', infile)
+
+    if util_format == 'sce-tou':
+        sce_tou_parse(csv_reader, verbose, metricsout)
+    elif util_format == 'pep':
+        pep_parse(csv_reader, verbose, metricsout)
+    return (rows_delivered, rows_generated, metricsout)
+
+
+def sce_tou_parse(csv_reader, verbose, metricsout):
+    global rows_generated 
+    global rows_delivered 
+    point = []
+    tag_values = []
+    tag = ''
+    pmult = 0
+
+    for row in csv_reader:
+        if len(row) > 0:
+            if 'Received' in row[0]:
+                tag = 'generated'
+            elif 'Delivered' in row[0]:
+                tag = 'delivered'
+            elif 'Consumption' in row[0]:
+                tag = 'delivered'
+            elif 'to' in row[0]:
+                util_timestamp = row[0].split('to')
+                # SCE adds non-ASCII charter before the to field, need to strip
+                util_timestamp = [
+                    item.replace('\xa0', '') for item in util_timestamp
+                ]
+                dt_local = datetime.strptime(util_timestamp[0], '%Y-%m-%d %H:%M:%S')
+                dt_utc = dt_local.astimezone(pytz.UTC)
+                dt_utc = dt_utc.strftime("%Y-%m-%d %H:%M:%S")
+                # use local time instead of UTC as you want customer month
+                dt_month = dt_local.strftime('%B')
+                if tag == 'generated':
+                    rows_generated = rows_generated + 1
+                    pmult = -1
+                elif tag == 'delivered':
+                    rows_delivered = rows_delivered + 1
+                    pmult = 1
+                if float(row[1]) != 0 :
+                    point = {
+                        "measurement": "energy",
+                        "tags": {
+                            "type": tag,
+                            "month": dt_local.strftime('%B'),
+                            "day": dt_local.strftime('%A'),
+                            "year": dt_local.strftime('%Y'),
+                        },
+                        "time": dt_utc,
+                        "fields": {
+                            "kwh": float(row[1]) * pmult
+                        }
+                }
+           
+                if verbose:
+                    print(point)
+
+                metricsout.append(point)
+
+def pep_parse(csv_reader, verbose, metricsout):
+    global rows_generated 
+    global rows_delivered 
+    point = []
+    tag_values = []
+    tag = ''
+    pmult = 0
+
+    for row in csv_reader:
+        if len(row) > 0:
+            if 'Electric usage' in row[0]:
+                if float(row[4]) < 0 :
+                    tag = 'generated'
+                else : 
+                    tag = 'delivered'
+            
+                util_timestamp = row[1]
+                util_timestamp = str(util_timestamp) + ' ' + str(row[3]) + str(':00')
+                dt_local = datetime.strptime(util_timestamp ,"%Y-%m-%d %H:%M:%S")
+                dt_utc = dt_local.astimezone(pytz.UTC)
+                dt_utc = dt_utc.strftime("%Y-%m-%d %H:%M:%S")
+                # use local time instead of UTC as you want customer month
+            
+                if tag == 'generated':
+                    rows_generated = rows_generated + 1
+                    cost = float(re.sub(r'[^0-9]', '', row[6]))/-100 
+                elif tag == 'delivered':
+                    rows_delivered = rows_delivered + 1
+                    cost = float(re.sub(r'[^0-9]', '', row[6]))/100 
+        
+                point = {
+                    "measurement": "energy",
+                    "tags": {
+                        "type": tag,
+                        "month": dt_local.strftime('%B'),
+                        "day": dt_local.strftime('%A'),
+                        "year": dt_local.strftime('%Y'),
+                    },
+                    "time": dt_utc,
+                    "fields": {
+                        "kwh": float(row[4]),
+                        "cost" : cost
+                    }
+                }
+
+                if verbose:
+                    print(point)
+
+                metricsout.append(point)
+
+    return ()