#1033 - slack bot warn when subgraph gets stuck (#1034)

* Check subgraph sync

* Run check script every 30 minutes

* Run cron every 30 minutes and make random jokes

* Intentionally fail to test

* Convert token amount to int

* set +e

* int token amount

* Catch errors

* Debug output

* Fix bash

* Use || true

* Use floats

* Check for Error

* test print

* print("Debug print")

* Test

* Remove debug prints

* Warning test

* Revert debug thing

* Remove checkout ref

* Convert to wei

* Convert to UnixTimeS

* Add missing import

* Force fail

* Remove forced errors

* Fix tests

* Formatting

* Mock block number

* Fix mock path

* Linter

.github/workflows/check_mainnet.yml

@@ -2,7 +2,7 @@ name: Run Mainnet Check and Notify Slack
 
 on:
   schedule:
-    - cron: "17 * * * *"
+    - cron: "*/30 * * * *"
   workflow_dispatch:
 
 jobs:
@@ -11,8 +11,6 @@ jobs:
     steps:
       - name: Checkout Repository
         uses: actions/checkout@v2
-        with:
-          ref: ba241caa6554d6b0444a501ec0f884f43d59abf2
 
       - name: Set up Python
         uses: actions/setup-python@v2
@@ -32,28 +30,16 @@ jobs:
 
       - name: Notify Slack
         run: |
-          output=$(python pdr check_network ppss.yaml sapphire-mainnet | grep -E 'FAIL|WARNING|error' || true)
-          
-          if [ -z "$output" ]; then
+          output=$(python pdr check_network ppss.yaml sapphire-mainnet 2>&1 || true)
+          echo "$output"          
+          filtered_output=$(echo "$output" | grep -E 'FAIL|ERROR|error|Error' || true)
+          if [ -z "$filtered_output" ]; then
             echo "No output, so no message will be sent to Slack"
           else
-            trivia_response=$(curl -s "https://opentdb.com/api.php?amount=1&difficulty=medium&type=multiple&category=18")
-            question=$(echo "$trivia_response" | jq -r '.results[0].question')
-            correct_answer=$(echo "$trivia_response" | jq -r '.results[0].correct_answer')
-            incorrect_answers=$(echo "$trivia_response" | jq -r '.results[0].incorrect_answers[]' | paste -sd "," -)
-
-            IFS=',' read -r -a answers <<< "$correct_answer,$incorrect_answers"
-            shuffled_answers=($(shuf -e "${answers[@]}"))
-
+            joke_response=$(curl -s "https://v2.jokeapi.dev/joke/Any?type=single")
+            joke=$(echo "$joke_response" | jq -r '.joke')
 
-            options=("A" "B" "C" "D")
-            shuffled_options=()
-            for i in "${!shuffled_answers[@]}"; do
-              shuffled_options+=("${options[$i]}: ${shuffled_answers[$i]}")
-            done
+            message="🙀 Mainnet Check script failed: \n${filtered_output}\n\nHere's a joke:\n$joke\n"
 
-            trivia_message="Trivia question: $question\nOptions:\n${shuffled_options[0]}\n${shuffled_options[1]}\n${shuffled_options[2]}\n${shuffled_options[3]}\n"
-            
-            message="Mainnet Check script failed: \n${output}\n\n${trivia_message}"
             curl -X POST -H 'Content-type: application/json' --data '{"text":"'"$message"'"}' ${{ secrets.SLACK_WEBHOOK_URL_MAINNET }}
           fi

pdr_backend/analytics/check_network.py

@@ -10,8 +10,9 @@
 from pdr_backend.subgraph.subgraph_consume_so_far import get_consume_so_far_per_contract
 from pdr_backend.util.constants import S_PER_DAY, S_PER_WEEK
 from pdr_backend.util.constants_opf_addrs import get_opf_addresses
-from pdr_backend.util.currency_types import Eth
+from pdr_backend.util.currency_types import Eth, Wei
 from pdr_backend.util.time_types import UnixTimeS
+from pdr_backend.subgraph.subgraph_sync import block_number_is_synced
 
 _N_FEEDS = 20  # magic number alert. FIX ME, shouldn't be hardcoded
 logger = logging.getLogger("check_network")
@@ -47,10 +48,10 @@ def check_dfbuyer(
     dfbuyer_addr: str,
     contract_query_result: dict,
     subgraph_url: str,
-    token_amt: int,
+    token_amt: float,
 ):
     cur_ut = UnixTimeS.now()
-    start_ut = int((cur_ut // S_PER_WEEK) * S_PER_WEEK)
+    start_ut = UnixTimeS(int((cur_ut // S_PER_WEEK) * S_PER_WEEK))
 
     contracts_sg_dict = contract_query_result["data"]["predictContracts"]
     contract_addresses = [
@@ -76,7 +77,22 @@ def check_dfbuyer(
 
 
 @enforce_types
-def get_expected_consume(for_ut: int, token_amt: int) -> Union[float, int]:
+def check_subgraph(web3_pp):
+    current_block = web3_pp.w3.eth.block_number
+    threshold = 50
+    check_block_number = current_block - threshold
+
+    is_synced = block_number_is_synced(web3_pp.subgraph_url, check_block_number)
+    if not is_synced:
+        logger.error(
+            "Subgraph is out of sync, checked block %d, current block: %d",
+            check_block_number,
+            current_block,
+        )
+
+
+@enforce_types
+def get_expected_consume(for_ut: int, token_amt: float) -> Union[float, int]:
     """
     @arguments
       for_ut -- unix time, in ms, in UTC time zone
@@ -89,7 +105,7 @@ def get_expected_consume(for_ut: int, token_amt: int) -> Union[float, int]:
     week_start_ut = (math.floor(for_ut / S_PER_WEEK)) * S_PER_WEEK
     time_passed = for_ut - week_start_ut
     n_weeks = int(time_passed / S_PER_DAY) + 1
-    return n_weeks * amt_per_feed_per_week
+    return int(n_weeks * amt_per_feed_per_week)
 
 
 @enforce_types
@@ -168,7 +184,7 @@ def check_network_main(ppss: PPSS, lookback_hours: int):
     addresses = get_opf_addresses(web3_pp.network)
     for name, address in addresses.items():
         ocean_bal = OCEAN.balanceOf(address)
-        native_bal = web3_pp.get_token_balance(address)
+        native_bal = Wei(web3_pp.get_token_balance(address))
 
         ocean_warning = (
             " LOW OCEAN BALANCE!"
@@ -177,7 +193,7 @@ def check_network_main(ppss: PPSS, lookback_hours: int):
         )
         native_warning = " LOW NATIVE BALANCE!" if native_bal < Eth(10).to_wei() else ""
 
-        lfunc = logger.warning if ocean_warning or native_warning else logger.info
+        lfunc = logger.error if ocean_warning or native_warning else logger.info
 
         lfunc(
             "%s: OCEAN: %.2f%s, Native: %.2f%s",
@@ -196,4 +212,7 @@ def check_network_main(ppss: PPSS, lookback_hours: int):
     # If token_amt is not a multiple of 60, adjust it to the next multiple of 60
     if token_amt % 60 != 0:
         token_amt = ((token_amt // 60) + 1) * 60
+
     check_dfbuyer(dfbuyer_addr, result, web3_pp.subgraph_url, token_amt)
+
+    check_subgraph(web3_pp)

pdr_backend/analytics/test/test_check_network.py

@@ -35,7 +35,7 @@ def test_check_dfbuyer(  # pylint: disable=unused-argument
     dfbuyer_addr = "0x1"
     contract_query_result = {"data": {"predictContracts": [{"id": "0x1"}]}}
     subgraph_url = "test_dfbuyer"
-    token_amt = 3
+    token_amt = 3.0
     check_dfbuyer(dfbuyer_addr, contract_query_result, subgraph_url, token_amt)
 
     target_str = (
@@ -56,7 +56,7 @@ def test_check_dfbuyer(  # pylint: disable=unused-argument
 def test_get_expected_consume():
     # Test case 1: Beginning of week
     for_ut = S_PER_WEEK  # Start of second week
-    token_amt = 140
+    token_amt = 140.0
     expected = token_amt / 7 / _N_FEEDS  # Expected consume for one interval
     assert get_expected_consume(for_ut, token_amt) == expected
 
@@ -103,8 +103,9 @@ def test_check_network_main(  # pylint: disable=unused-argument
     mock_token.return_value.balanceOf.return_value = Eth(1000).to_wei()
 
     mock_w3 = Mock()  # pylint: disable=not-callable
-    mock_w3.eth.get_balance.return_value = Eth(1000).to_wei()
+    mock_w3.eth.get_balance.return_value = 1000.0 * 1e18
     ppss.web3_pp.web3_config.w3 = mock_w3
+    ppss.web3_pp.w3.eth.block_number = 100
     check_network_main(ppss, lookback_hours=24)
 
     mock_get_opf_addresses.assert_called_once_with("sapphire-mainnet")

pdr_backend/cli/arg_timeframe.py

@@ -54,8 +54,7 @@ def __init__(self, timeframes: Union[List[str], List[ArgTimeframe]]):
         for timeframe in timeframes:
             if isinstance(timeframe, str):
                 frame = ArgTimeframe(timeframe)
-
-            frames.append(frame)
+                frames.append(frame)
 
         super().__init__(frames)
 

system_tests/test_check_network_system.py

@@ -29,7 +29,8 @@ def test_check_network(mock_print_stats, mock_check_dfbuyer, caplog):
 
     mock_web3_pp.OCEAN_Token = mock_token
     mock_web3_pp.NativeToken = mock_token
-    mock_web3_pp.get_token_balance.return_value = Wei(100)
+    mock_web3_pp.get_token_balance.return_value = 100
+    mock_web3_pp.w3.eth.block_number = 100
 
     mock_query_subgraph = Mock()
     mock_query_subgraph.return_value = {