model update

Author: ethgasstation

egs/egs_ref.py

@@ -432,13 +432,17 @@ def update_txblock(self, txpool_block, blockdata, predictiontable, gprecs):
         txpool_block = txpool_block.join(predictiontable.predictiondf, how='left', on='round_gp_10gwei')
         txpool_block['safelow'] = gprecs['safeLow']
         txpool_block['average'] = gprecs['average']
+        txpool_block['nomine'] = gprecs['nomine']
         console.info("updating " + str(len(txpool_block)) + " transactions")
         self.df = self.df.combine_first(txpool_block)
 
 
-    def write_to_sql(self):
+    def write_to_sql(self, txpool):
         """writes to sql, prevent buffer overflow errors"""
         console.info("writing to mysql....this can take awhile")
+        prev_block = self.process_block-1
+        txp = txpool.txpool_df.loc[txpool.txpool_df['block']==prev_block]
+        self.df['txpool'] = self.df.index.isin(txp.index).astype(int)
         self.df.reset_index(inplace=True)
         length = len(self.df)
         chunks = int(np.ceil(length/1000))
@@ -481,6 +485,7 @@ def __init__(self, name, current_block, start_block, end_block, max_gas, alltx,
         self.alltx = alltx
         self.txpool = txpool
         self.safe = None
+        self.nomine_gp = None
         self.init_df()
 
     def init_df(self):
@@ -511,6 +516,8 @@ def init_df(self):
             self.print_length()
             self.df = pd.DataFrame()
             self.safe = None
+            self.nomine_gp = None
+
 
     def get_txpool(self):
         df = self.df
@@ -521,7 +528,10 @@ def get_txpool(self):
         else:
             safe = df.loc[(df['total'] >= 2) & (df['mined'] >=1) & (df['pct_remaining'] < 10)]
         safe_gp = safe.index.min()
+        nomine= df.loc[(df['mined'] == 0) & (df.index < safe_gp)]
+        nomine_gp = nomine.index.max()
         self.safe = safe_gp
+        self.nomine_gp = nomine_gp
 
 
     def print_length(self):
@@ -548,7 +558,6 @@ def init_predictiontable(self):
         """makes prediction table for number of blocks to confirmation"""
         hashpower = self.blockdata.hashpower
         hpower = self.alltx.pctmined_gp_last100
-        avg_timemined = self.blockdata.block_time
         txpool_by_gp = self.txpool.txpool_by_gp
         submitted_5mago = self.recentdf
         submitted_30mago = self.remotedf
@@ -568,9 +577,38 @@ def init_predictiontable(self):
             predictTable['pct_remaining30m'] = submitted_30mago['pct_remaining']
             predictTable['pct_mined_30m'] =  submitted_30mago['pct_mined']
             predictTable['total_seen_30m'] =  submitted_30mago['total']
+
+        self.predictiondf = predictTable
 
-        #to-do - fix
-        predictTable['expectedWait'] = predictTable.apply(predict, axis=1)
+    def get_predicted_wait(self, gpreport):
+        """make wait time based on the data"""
+        predictTable = self.predictiondf
+        avg_timemined = self.blockdata.block_time
+        gprecs = gpreport.gprecs
+        predictTable['average'] = gprecs['average']
+        predictTable['safelow'] = gprecs['safeLow']
+        predictTable['nomine'] = gprecs['nomine']
+        predictTable['avgdiff'] = (predictTable.index >= predictTable['average']).astype(int)
+        predictTable['intercept'] = INTERCEPT 
+        predictTable['hpa_coef'] = HPA_COEF
+        predictTable['avgdiff_coef'] = AVGDIFF_COEF
+        predictTable['tx_atabove_coef'] = TXATABOVE_COEF
+        predictTable['int2'] = INTERCEPT2
+        predictTable['hpa_coef2'] = HPA_COEF2
+
+        
+        
+        
+        if gprecs['nomine']:
+            predictTable['sum'] = (predictTable['intercept'] + (predictTable['hashpower_accepting'] * predictTable['hpa_coef'])  + (predictTable['avgdiff'] * predictTable['avgdiff_coef']) + (predictTable['tx_atabove'] * predictTable['tx_atabove_coef']))
+            predictTable['expectedWait'] = predictTable['sum'].apply(lambda x: np.exp(x))
+            predictTable.loc[predictTable.index <= gprecs['nomine'], 'expectedWait'] = 1000
+        else:
+            predictTable['sum'] = (predictTable['int2'] + (predictTable['hashpower_accepting'] * predictTable['hpa_coef2']))
+            predictTable['expectedWait'] = predictTable['sum'].apply(lambda x: np.exp(x))
+            predictTable['unsafe'] = predictTable['hashpower_accepting'].apply(lambda x: 1 if x < 30 else 0)
+            predictTable.loc[predictTable['unsafe'] ==1, 'expectedWait'] = 1000
+        predictTable['expectedWait'] = predictTable['expectedWait'].apply(lambda x: x if x>=2 else 2)
         predictTable['expectedTime'] = predictTable['expectedWait'].apply(lambda x: np.round((x * avg_timemined / 60), decimals=2))
         self.predictiondf = predictTable
 
@@ -602,7 +640,6 @@ def __init__(self, predictiontable, blockdata, submitted_recent, submmited_remot
         self.array5m = array5m
         self.array30m = array30m
         self.gprecs = None
-        self.minlow = 1
         self.make_gasprice_report()
 
 
@@ -613,7 +650,6 @@ def make_gasprice_report(self):
         speed = self.blockdata.speed
         array5m = self.array5m
         array30m = self.array30m
-        minlow = self.minlow
         block = self.block
 
         if self.submitted_remote.safe:
@@ -626,44 +662,35 @@ def make_gasprice_report(self):
         else:
             average = self.blockdata.avg
 
-        def get_wait(gasprice):
-            try:
-                wait =  prediction_table.loc[prediction_table['gasprice']==gasprice, 'expectedTime'].values[0]
-            except:
-                wait = 0
-            wait = round(wait, 1)
-            return float(wait)
-
         console.info('safelow: ' + str(safelow))
         console.info('avg: ' +str(average))
-        if np.isnan(safelow):
-            safelow = MAX_GP
-        if np.isnan(average):
-            average = MAX_GP
-        array30m.append(safelow)
-        array5m.append(average)
 
         gprecs = {}
-        gprecs['safeLow'] = np.percentile(array30m, 50)
-        gprecs['average'] = np.percentile(array5m, 50)
         gprecs['fast'] = self.blockdata.fast
-        if np.isnan(gprecs['fast']):
-            gprecs['fast'] = MAX_GP
         gprecs['fastest'] = self.blockdata.fastest
-        if np.isnan(gprecs['fastest']):
-            gprecs['fastest'] = MAX_GP
+        if self.submitted_remote.nomine_gp:
+            gprecs['nomine'] = self.submitted_remote.nomine_gp.astype(float)
+        else:
+            gprecs['nomine'] = self.submitted_remote.nomine_gp
+
+        for rec in [safelow, average, gprecs['fast'], gprecs['fastest']]:
+            if np.isnan(rec):
+                rec = MAX_GP
+            
+        
+        array30m.append(safelow)
+        array5m.append(average)
 
+        
+        gprecs['safeLow'] = np.percentile(array30m, 50)
+        gprecs['average'] = np.percentile(array5m, 50)
+        
         if (gprecs['fast'] < gprecs['average']):
             gprecs['fast'] = gprecs['average']
 
         if (gprecs['safeLow'] > gprecs['average']):
             gprecs['safeLow'] = gprecs['average']
 
-        gprecs['safeLowWait'] = get_wait(gprecs['safeLow'])
-        gprecs['avgWait'] = get_wait(gprecs['average'])
-
-        gprecs['fastWait'] = get_wait(gprecs['fast'])
-        gprecs['fastestWait'] = get_wait(gprecs['fastest'])
         gprecs['block_time'] = block_time
         gprecs['blockNum'] = block
         gprecs['speed'] = speed
@@ -677,11 +704,24 @@ def get_wait(gasprice):
         self.array5m = array5m
         self.array30m = array30m
 
+    def get_wait(self, prediction_table):
+        def lookup(gasprice):
+            try:
+                wait =  prediction_table.at[prediction_table.index[gasprice], 'expectedTime']
+            except Exception as e :
+                print(e)
+            wait = round(wait, 1)
+            return float(wait)
+        self.gprecs['safeLowWait'] = lookup(self.gprecs['safeLow'])
+        self.gprecs['avgWait'] = lookup(self.gprecs['average'])
+        self.gprecs['fastWait'] = lookup(self.gprecs['fast'])
+        self.gprecs['fastestWait'] = lookup(self.gprecs['fastest'])
 
     def write_to_json(self):
         """write json data"""
         global exporter
         try:
+            print(self.gprecs)
             exporter.write_json('ethgasAPI', self.gprecs)
         except Exception as e:
             console.error("write_to_json: Exception caught: " + str(e))

egs/main.py

@@ -53,6 +53,10 @@ def master_control(args):
             array5m = gaspricereport.array5m 
             #hold recent safelow gp rec
             array30m = gaspricereport.array30m 
+            #make predicted wait times
+            if txpool.got_txpool:
+                predictiontable.get_predicted_wait(gaspricereport)
+                gaspricereport.get_wait(predictiontable.predictiondf)
             #updates tx submitted at current block with data from predictiontable, gpreport- this is for storing in mysql for later optional stats models.
             alltx.update_txblock(txpool.txpool_block, blockdata, predictiontable, gaspricereport.gprecs) 
 
@@ -81,7 +85,7 @@ def master_control(args):
             #write to mysql
             response = input ("save transactions to mysql (y/n)?")
             if (response.lower() == 'y'):
-                alltx.write_to_sql()
+                alltx.write_to_sql(txpool)
                 blockdata.write_to_sql()
                 sys.exit()
             else:

egs/modelparams/constants.py

@@ -11,25 +11,13 @@
 #interaction term with highgas offered and hashpower. not currently using
 #highgas offefred coefficient
 
-INTERCEPT = 7.5375
-HPA_COEF = -0.0801
-TXATABOVE_COEF = 0.0003 
-INTERACT_COEF = 0 
-HIGHGAS_COEF = 0.3532
+INTERCEPT = 4.4683
+HPA_COEF = -0.0136
+TXATABOVE_COEF = .0009
+AVGDIFF_COEF = -2.0235
 
-
-#a second model when data from past 5 min is available
-INT2 = 5.5751
-HPA2 = -0.0454
-TXATAB2 = 0
-HIGHGAS2 = 0.6658
-S5MAGO = 0.0176
-
-#high gas offered is defined based as a percentage of the gas limit
-#highgas2 is the only one that matters right now
-
-HIGHGAS1 = .037
-HIGHGAS2 = .15
+INTERCEPT2 = 6.9238
+HPA_COEF2 = -0.0670
 
 # Highest gas price tracked
 MAX_GP = 5000