Merge pull request #57 from rjsears/pool_update

Pool update
rjsears · Jul 9, 2021 · a50876a · a50876a
2 parents 5f1f439 + 31b41e7
commit a50876a
Show file tree

Hide file tree

Showing 9 changed files with 71 additions and 29 deletions.
diff --git a/README.md b/README.md
@@ -1,7 +1,7 @@
  <h2 align="center">
   <a name="chia_drive_logo" href="https://github.com/rjsears/chia_plot_manager"><img src="https://github.com/rjsears/chia_plot_manager/blob/main/images/chia_plot_manager_new.png" alt="Chia Plot Manager"></a><br>
 
-  Chia Plot, Drive Manager, Coin Monitor & Auto Drive (V0.92 - May 31th, 2021)
+  Chia Plot, Drive Manager, Coin Monitor & Auto Drive (V0.93 - July 8th, 2021)
   </h2>
   <p align="center">
 Multi Server Chia Plot and Drive Management Solution
@@ -754,6 +754,13 @@ strategy above, it is super easy to add more drives.
 
 ### <a name="changelog"></a>Changelog
 
+<b>V0.93 2021-07-08</b>
+   - Added ability to identify plots as `portable` (set `pooling: True` in config file)
+     and all new plots will be prepended with `portable.`. Eventually I will add in 
+     the ability to overwrite old plots with new plots during new pooling plot creation.
+     This ASSUMES 1.2.0 and has nothing to do with the plotting itself, just (as always)
+     with the management of the plots. 
+
 <b>V0.9 2021-05-28</b>
    - Added full multi-harvester capabilities to ```plot_manager.py``` along
      with host checks for all harvesters prior to attempting a plot move.

diff --git a/chianas/drive_manager.py b/chianas/drive_manager.py
@@ -3,7 +3,7 @@
 # -*- coding: utf-8 -*-
 
 __author__ = 'Richard J. Sears'
-VERSION = "0.92 (2021-06-16)"
+VERSION = "0.93 (2021-07-08)"
 
 """
 NOTE NOTE NOTE NOTE NOTE NOTE NOTE
@@ -47,6 +47,8 @@
 
 
  Updates
+   V0.93 2021-07-08
+   - Added in code to various parts of project to impliment pools.
  
    V0.92 2021-05-31
    - Converted to a central YAML config file

diff --git a/chianas/drivemanager_classes.py b/chianas/drivemanager_classes.py
@@ -2,11 +2,11 @@
 # -*- coding: utf-8 -*-
 
 """
-Part of drive_manager. These classes are for reading and updating out yaml
+Part of drive_manager. These classes are for reading and updating our yaml
 config file.
 """
 
-VERSION = "V0.92 (2021-06-07)"
+VERSION = "V0.93 (2021-07-08)"
 
 import os
 import yaml
@@ -76,13 +76,17 @@ class DriveManager:
         log.debug("Please check file path and try again.")
         exit()
     else:
-        def __init__(self, configured, hostname, chia_log_file, chia_config_file, remote_harvester_reports, remote_harvesters,
-                     notifications, pb, email, sms, daily_update, new_plot_drive, per_plot, local_plotter, temp_dirs, temp_dirs_critical,
-                     temp_dirs_critical_alert_sent, dst_dirs, dst_dirs_critical, dst_dirs_critical_alert_sent, warnings, emails, phones,
-                     twilio_from, twilio_account, twilio_token, pb_api, current_internal_drive, current_plotting_drive, total_plot_highwater_warning,
-                     total_plots_alert_sent, current_total_plots_midnight, current_total_plots_daily, offlined_drives, logging, log_level):
+        def __init__(self, configured, hostname, pools, replace_non_pool_plots, fill_empty_drives_first, chia_log_file, chia_config_file, 
+                     remote_harvester_reports, remote_harvesters, notifications, pb, email, sms, daily_update, new_plot_drive, per_plot, 
+                     local_plotter, temp_dirs, temp_dirs_critical, temp_dirs_critical_alert_sent, dst_dirs, dst_dirs_critical, 
+                     dst_dirs_critical_alert_sent, warnings, emails, phones, twilio_from, twilio_account, twilio_token, pb_api, 
+                     current_internal_drive, current_plotting_drive, total_plot_highwater_warning, total_plots_alert_sent, 
+                     current_total_plots_midnight, current_total_plots_daily, offlined_drives, logging, log_level):
             self.configured = configured
             self.hostname = hostname
+            self.pools = pools
+            self.replace_non_pool_plots = replace_non_pool_plots
+            self.fill_empty_drives_first = fill_empty_drives_first
             self.chia_log_file = chia_log_file
             self.chia_config_file = chia_config_file
             self.remote_harvester_reports = remote_harvester_reports
@@ -125,6 +129,9 @@ def read_configs(cls):
                 return cls(
                     configured=server['configured'],
                     hostname=server['hostname'],
+                    pools=server['pools']['active'],
+                    replace_non_pool_plots=server['pools']['replace_non_pool_plots'],
+                    fill_empty_drives_first=server['pools']['fill_empty_drives_first'],
                     chia_log_file=server['chia_log_file'],
                     chia_config_file=server['chia_config_file'],
                     remote_harvester_reports=server['remote_harvester_reports']['active'],

diff --git a/chianas/move_local_plots.py b/chianas/move_local_plots.py
@@ -3,7 +3,7 @@
 # -*- coding: utf-8 -*-
 
 __author__ = 'Richard J. Sears'
-VERSION = "0.92 (2021-06-07)"
+VERSION = "0.93 (2021-07-08)"
 
 # This script is part of my plot management set of tools. This
 # script is used to move plots from one location to another on
@@ -91,28 +91,32 @@ def process_plot():
         if not process_control('check_status', 0):
             plot_to_process = get_list_of_plots()
             if plot_to_process and not testing:
+                plot_source = plot_dir + '/' + plot_to_process
+                if chianas.pools:
+                    plot_destination = chianas.current_internal_drive + '/' + 'portable.' + plot_to_process
+                else:
+                    plot_destination = chianas.current_internal_drive + '/' + plot_to_process
                 process_control('set_status', 'start')
-                plot_path = plot_dir + '/' + plot_to_process
-                log.info(f'Processing Plot: {plot_path}')
+                log.info(f'Processing Plot: {plot_source}')
                 log.debug(f'Current Internal Plotting Drive is: {chianas.current_internal_drive}')
-                log.debug(f'Starting Copy of {plot_path} to {chianas.current_internal_drive}')
+                log.debug(f'Starting Copy of {plot_source} to {plot_destination}')
                 start_time = timer()
                 try:
-                    shutil.copy2(plot_path, chianas.current_internal_drive)
+                    shutil.copy2(plot_source, plot_destination)
                 except:
                     log.debug(f'ERROR: There was a problem copying: {plot_dir}!')
                     exit()
                 end_time = timer()
-                if verify_plot_move(chianas.current_internal_drive, plot_path, plot_to_process):
+                if verify_plot_move(plot_source, plot_destination):
                     log.info('Plot Sizes Match, we have a good plot move!')
                     log.info(f'Total Elapsed Time: {end_time - start_time:.2f} seconds or {(end_time - start_time)/60:.2f} Minutes')
                 else:
                     log.debug('FAILURE - Plot sizes DO NOT Match')
                     process_control('set_status', 'stop')  #Set to stop so it will attempt to run again in the event we want to retry....
                     main() # Try Again - no need to do anything with the file, shutil.copy2 will overwrite an existing file.
                 process_control('set_status', 'stop')
-                os.remove(plot_path)
-                log.info(f'Removing: {plot_path}')
+                os.remove(plot_source)
+                log.info(f'Removing: {plot_source}')
             elif testing:
                 log.debug('Testing Only - Nothing will be Done!')
             else:
@@ -121,11 +125,11 @@ def process_plot():
             return
 
 
-def verify_plot_move(current_plotting_drive, plot_path, plot_to_process):
+def verify_plot_move(plot_source, plot_destination):
     log.debug('verify_plot_move() Started')
-    log.debug (f'Verifing: {current_plotting_drive}/{plot_to_process}')
-    original_plot_size = os.path.getsize(plot_path)
-    copied_plot_size = os.path.getsize(current_plotting_drive + '/' + plot_to_process)
+    log.debug (f'Verifing: {plot_source}')
+    original_plot_size = os.path.getsize(plot_source)
+    copied_plot_size = os.path.getsize(plot_destination)
     log.debug(f'Original Plot Size Reported as: {original_plot_size}')
     log.debug(f'Copied Plot Size Reported as: {copied_plot_size}')
     if original_plot_size == copied_plot_size:
@@ -189,6 +193,7 @@ def check_drive_activity():
         return True
 
 def main():
+    log.debug(f'move_local_plots.py: Version {VERSION}')
     are_we_configured()
     process_plot()
 

diff --git a/chianas/plot_manager.skel.yaml b/chianas/plot_manager.skel.yaml
@@ -1,15 +1,26 @@
-# v0.92 2021-06-07
+# v0.93 2021-07-08
 # Once you have made the necessary modifications to this file, change this to
 # True.
 configured: False
 
 # Enter the hostname of this server:
 hostname: chianas01
 
+# Are we plotting for pools? This has nothing to do with the actual plotting of
+# plots but rather just naming of the new plots and eventually the replacing of 
+# old plots with portable plots.
+pools:
+  active: False
+  # Do we want to replace non-pool plots with new plots
+  replace_non_pool_plots: True
+  # Should we fill up empty drive space before replacing old non-pool plots?
+  fill_empty_drives_first: True
+
 # Enter Logging Information
 logging: True
 log_level: DEBUG
 
+
 # Where is your chia log file located? Remember to set the logging level
 # in your chia config to INFO. By default, it is set to WARNING.
 chia_log_file: not_set

diff --git a/chiaplot/plot_manager.py b/chiaplot/plot_manager.py
@@ -2,7 +2,7 @@
 # -*- coding: utf-8 -*-
 
 __author__ = 'Richard J. Sears'
-VERSION = "0.92 (2021-06-04)"
+VERSION = "0.93 (2021-07-08)"
 
 """
 Simple python script that helps to move my chia plots from my plotter to
@@ -14,6 +14,9 @@
 the incoming plots will be placed. This script simply sends those plots when
 they are ready to send.
 Updates
+  v0.93 2021-08-21
+  - Added ability to identify pool plots by prepending 'portable.' to the plot name
+    so we can manage them at a later time.
   v0.9 2021-05-28
   - Rewritten logging to support path autodetection, support for multiple
     NAS/Harvesters. Chooses Harvester with the most available plots on it
@@ -133,6 +136,8 @@ def process_plot():
             plot_path = plot_dir + plot_to_process
             log.info(f'Processing Plot: {plot_path}')
             log.debug(f'{nas_server} reports remote mount as {remote_mount}')
+            if chiaplot.pools:
+                plot_to_process = 'portable.'+plot_to_process
             subprocess.call([f'{script_path.joinpath("send_plot.sh")}', plot_path, plot_to_process, nas_server])
             try:
                 subprocess.call(['ssh', nas_server, f'{script_path.joinpath("utilities/kill_nc.sh")}'])  # make sure all of the nc processes are dead on the receiving end
@@ -425,7 +430,7 @@ def check_temp_drive_utilization():
         chiaplot.toggle_alert_sent('temp_dirs_critical_alert_sent')
         notify('INFORMATION: Directory Utilization', 'INFORMATION: Your Temp Directory is now below High Capacity Warning\nPlotting will Continue')
     else:
-        log.debug('Temp Drive(s) check complete. ALl OK!')
+        log.debug('Temp Drive(s) check complete. All OK!')
 
 def check_dst_drive_utilization():
     """

diff --git a/chiaplot/plot_manager.skel.yaml b/chiaplot/plot_manager.skel.yaml
@@ -5,11 +5,14 @@ configured: False
 # Enter the hostname of this server:
 hostname: chiaplot01
 
+# Are we plotting for pools (prepends `portable.` to the plot name)
+pools: True
+
 # Enter Logging Information
 logging: True
 log_level: DEBUG
 
-# I use Plotman to do my plotting, but this should work for anything. This
+# I use MadMax to do my plotting, but this should work for anything. This
 # has NOTHING to do with setting up your plotting configuration and is
 # only used for monitoring drive space for notifications. Set to True if
 # locally plotting and configure the rest of the settings.

diff --git a/chiaplot/plotmanager_classes.py b/chiaplot/plotmanager_classes.py
@@ -2,11 +2,11 @@
 # -*- coding: utf-8 -*-
 
 """
-Part of drive_manager. These classes are for reading and updating out yaml
+Part of drive_manager. These classes are for reading and updating our yaml
 config file.
 """
 
-VERSION = "V0.92 (2021-06-04)"
+VERSION = "V0.93 (2021-07-08)"
 
 import os
 import yaml
@@ -36,13 +36,14 @@ class PlotManager:
         log.debug("Please check file path and try again.")
         exit()
     else:
-        def __init__(self, configured, hostname, remote_harvesters,
+        def __init__(self, configured, hostname, pools, remote_harvesters,
                      notifications, pb, email, sms, temp_dirs, temp_dirs_critical, network_interface,
                      dst_dirs, dst_dirs_critical, dst_dirs_critical_alert_sent,temp_dirs_critical_alert_sent,
                      warnings, emails, phones, twilio_from, twilio_account,
                      twilio_token, pb_api, logging, log_level):
             self.configured = configured
             self.hostname = hostname
+            self.pools = pools
             self.remote_harvesters = remote_harvesters
             self.network_interface = network_interface
             self.notifications = notifications
@@ -72,6 +73,7 @@ def read_configs(cls):
                 return cls(
                     configured=server['configured'],
                     hostname=server['hostname'],
+                    pools=server['pools'],
                     remote_harvesters=server['remote_harvesters'],
                     network_interface=server['network_interface'],
                     notifications=server['notifications']['active'],

diff --git a/install.sh b/install.sh
@@ -1,6 +1,6 @@
 #! /bin/bash
 
-# Version V0.92 2021-06-07
+# Version V0.93 2021-07-08
 
 # Simple Install script for NEW clean Ubuntu 20.04 install, updates
 # the system with various tools and tings required to run the various