Skip to content

Commit cc129a7

Browse files
committed
Adds scraper progress bar
1 parent 949e302 commit cc129a7

File tree

2 files changed

+4
-3
lines changed

2 files changed

+4
-3
lines changed

ffbot/scraper.py

+3-3
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66
import pandas as pd
77
import requests
88
from requests.adapters import HTTPAdapter
9+
from tqdm import tqdm
910
from urllib3.util import Retry
1011
from user_agent import generate_user_agent
1112

@@ -78,8 +79,6 @@ def scrape(league):
7879
df = pd.DataFrame(data, columns=["ID", "Team"])
7980

8081
# Scrape projections
81-
logger.info("Scraping weekly forecasts...")
82-
8382
def get_projections(row):
8483
# New session every 100 players
8584
global s
@@ -138,7 +137,8 @@ def get_projections(row):
138137

139138
return row
140139

141-
df = df.apply(get_projections, axis=1)
140+
tqdm.pandas(desc="Scraping weekly forecasts")
141+
df = df.progress_apply(get_projections, axis=1)
142142

143143
# Reorder columns
144144
columns = list(df.columns)

requirements.txt

+1
Original file line numberDiff line numberDiff line change
@@ -4,4 +4,5 @@ lxml
44
pandas
55
pulp
66
requests
7+
tqdm
78
user_agent

0 commit comments

Comments
 (0)