Merge branch 'main' into 13-add-ui

Author: amarvin

.github/workflows/python-package.yml

@@ -13,33 +13,37 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.8", "3.9", "3.10"]
+        python-version: ["3.8", "3.9", "3.10", "3.11"]
 
     steps:
     - uses: actions/checkout@v2
     - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v2
+      uses: actions/setup-python@v4
       with:
         python-version: ${{ matrix.python-version }}
+        cache: 'pip' # caching pip dependencies
+        cache-dependency-path: |
+          **/setup.cfg
+          **/requirements*.txt
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip
         if [ -f requirements-test.txt ]; then pip install -r requirements-test.txt; fi
         if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
+    - name: Sort imports with isort
+      run: isort . --check --color --diff
+    - name: Code format with black
+      run: black . --check --color --diff
     - name: Lint with flake8
       run: |
         # stop the build if there are Python syntax errors or undefined names
         flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
         # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
         flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
-    - name: Code format with black
-      run: |
-        black . --check --color --diff
     - name: Test with pytest
       run: |
         pytest --cov=ffbot --cov-report=xml
     - name: "Upload coverage to Codecov"
       uses: codecov/codecov-action@v2
       with:
         fail_ci_if_error: true
-  

.github/workflows/pythonpublish.yml

@@ -12,7 +12,7 @@ jobs:
     - name: Set up Python
       uses: actions/setup-python@v1
       with:
-        python-version: '3.7'
+        python-version: '3.11'
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip

.gitignore

@@ -109,3 +109,4 @@ chromedriver.exe
 .idea
 credentials.json
 *.mps
+.vscode

.pre-commit-config.yaml

@@ -0,0 +1,17 @@
+repos:
+-   repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.5.0
+    hooks:
+    -   id: trailing-whitespace
+        exclude_types: [python]
+    -   id: end-of-file-fixer
+    -   id: check-yaml
+    -   id: check-added-large-files
+-   repo: https://github.com/PyCQA/isort
+    rev: 5.13.2
+    hooks:
+    -   id: isort
+-   repo: https://github.com/psf/black
+    rev: 24.3.0
+    hooks:
+    -   id: black

README.md

@@ -1,9 +1,10 @@
 # fantasy-football-bot (ffbot)
 
 [![PyPI Latest Release](https://img.shields.io/pypi/v/ffbot.svg)](https://pypi.org/project/ffbot/)
-[![PyPI downloads](https://pepy.tech/badge/ffbot)](https://pepy.tech/project/ffbot)
+[![PyPI downloads](https://static.pepy.tech/badge/ffbot)](https://pepy.tech/project/ffbot)
 [![License](https://img.shields.io/github/license/amarvin/fantasy-football-bot)](https://github.com/amarvin/fantasy-football-bot/blob/main/LICENSE)
 [![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black)
+[![Imports: isort](https://img.shields.io/badge/%20imports-isort-%231674b1?style=flat&labelColor=ef8336)](https://pycqa.github.io/isort/)
 [![codecov](https://codecov.io/gh/amarvin/fantasy-football-bot/branch/main/graph/badge.svg?token=CH6M9DR7VX)](https://codecov.io/gh/amarvin/fantasy-football-bot)
 
 Automate playing Yahoo Fantasy Football
@@ -23,14 +24,16 @@ pip install ffbot
 ### Scrape player forecasts
 
 To connect to your Yahoo league and team, you need your league ID and team ID.
-Visit your team at https://football.fantasysports.yahoo.com/f1/, and the url will also include your league and team ID.
+Visit your team at <https://football.fantasysports.yahoo.com/f1/>, and the url will also include your league and team ID.
 
 ```python
 >>> LEAGUE = 123456
 >>> TEAM = 1
 >>> POSITIONS = "QB, WR, WR, WR, RB, RB, TE, W/R/T, K, DEF, BN, BN, BN, BN, IR"
 >>> week = ffbot.current_week()
 >>> df = ffbot.scrape(LEAGUE)
+>>> # If playing an Individual Defensive Player (IDP) league, then scrape additional players with:
+>>> # df = ffbot.scrape(LEAGUE, is_IDP=True)
 Scraping all QB...
 Scraping all WR...
 Scraping all RB...
@@ -73,10 +76,11 @@ Only one Waiver claim (for Jordon Howard) increases discounted points.
 
 ## Contribution
 
-Please add Issues or submit Pull requests!
+Please add Issues or submit Pull Requests!
 
-For local development, install optional testing dependencies using
+For local development, install optional testing dependencies and pre-commit hooks using
 
 ```sh
 pip install ffbot[test]
+pre-commit install
 ```

demo.py

@@ -1,6 +1,5 @@
 import ffbot
 
-
 # Yahoo league/team id
 # Visit your team at https://football.fantasysports.yahoo.com/f1/, and the url will also include your league and team ID
 LEAGUE = 123456
@@ -10,6 +9,8 @@
 
 # Scrape data for current and available players, and their point forecasts for each week
 df = ffbot.scrape(LEAGUE)
+# If playing an Individual Defensive Player (IDP) league, then scrape additional players with:
+# df = ffbot.scrape(LEAGUE, is_IDP=True)
 
 # Optional save data to CSV, and load latest data
 # ffbot.save(df, week)

ffbot/__init__.py

@@ -1,6 +1,6 @@
 from .constants import VERSION
 from .optimizer import optimize  # noqa: F401,E402
 from .scraper import current_week, scrape  # noqa: F401,E402
-from .utils import save, load  # noqa: F401,E402
+from .utils import load, save  # noqa: F401,E402
 
 __version__ = VERSION

ffbot/constants.py

@@ -1 +1 @@
-VERSION = "1.1.0"
+VERSION = "1.2.4"

ffbot/optimizer.py

@@ -1,19 +1,19 @@
 from collections import Counter
 
 import pandas as pd
+from loguru import logger
 from pulp import (
+    PULP_CBC_CMD,
     LpBinary,
     LpContinuous,
     LpMaximize,
     LpProblem,
     LpStatus,
-    lpSum,
     LpVariable,
-    PULP_CBC_CMD,
+    lpSum,
     value,
 )
 
-
 IR_STATUSES = {
     "COVID",  # e.g. COVID-19
     "IR",  # e.g. IR, IR-R
@@ -33,19 +33,36 @@ def optimize(df, week, team, positions):
     # Game rules
     positions = [x.strip() for x in positions.split(",")]
     PossiblePositions = dict(
-        QB={"QB"},
-        WR={"WR", "W/R/T"},
-        RB={"RB", "W/R/T"},
-        TE={"TE", "W/R/T"},
+        QB={"QB", "Q/W/R/T"},
+        WR={"WR", "W/R/T", "W/T", "W/R", "Q/W/R/T"},
+        RB={"RB", "W/R/T", "W/R", "Q/W/R/T"},
+        TE={"TE", "W/R/T", "W/T", "Q/W/R/T"},
         K={"K"},
-        DEF={"DEF"},
+        DEF={"DEF", "D/ST"},
+        # IDP positions
+        CB={"CB", "D", "DB"},
+        DE={"D", "DE", "DL"},
+        DT={"D", "DL", "DT"},
+        LB={"D", "LB"},
+        S={"D", "DB", "S"},
     )
+    #  Remove positions not available in this league
+    for key, values in PossiblePositions.items():
+        PossiblePositions[key] = set(
+            position for position in values if position in positions
+        )
+    PossiblePositions = {
+        key: values for key, values in PossiblePositions.items() if values
+    }
+    #  Players that can play multiple positions
     for position in df["Position"].unique():
         if position not in PossiblePositions:
             # There is a player that can play multiple positions, so consider those options too
             PossiblePositions[position] = set()
             for n in position.split(","):
-                PossiblePositions[position].update(PossiblePositions[n.strip()])
+                n = n.strip()
+                if n in PossiblePositions:
+                    PossiblePositions[position].update(PossiblePositions[n])
     PositionMax = Counter(positions)
     POSITIONS = PositionMax.keys()
 
@@ -100,6 +117,7 @@ def optimize(df, week, team, positions):
         for n in POSITIONS
         if (p, n) in PlayerPosition
     ]
+    logger.info("Optimizer pre-processed data")
 
     # Define optimization problem
     prob = LpProblem("football", LpMaximize)
@@ -153,6 +171,7 @@ def optimize(df, week, team, positions):
                 lpSum(assign[p, t, n] for p in PLAYERS if (p, n) in PlayerPosition)
                 <= PositionMax[n]
             )
+    logger.info("Optimizer starting...")
 
     # Solve optimization problem
     solutions_headers = ["Add", "Drop", "Total points", "Discounted points", "VOR"]
@@ -317,4 +336,5 @@ def optimize(df, week, team, positions):
     df_opt = pd.DataFrame(solutions, columns=solutions_headers)
     df_opt = df_opt.round(2)
     df_opt.fillna("", inplace=True)
+    logger.info("Optimizer finished")
     return df_opt

ffbot/scraper.py

@@ -1,24 +1,30 @@
 from datetime import datetime
+from io import StringIO
 
-from bs4 import BeautifulSoup as bs
-from loguru import logger
 import numpy as np
 import pandas as pd
 import requests
+from bs4 import BeautifulSoup as bs
+from loguru import logger
 from requests.adapters import HTTPAdapter
 from tqdm import tqdm
 from urllib3.util import Retry
 from user_agent import generate_user_agent
 
-
 # A public league for current week and player IDs
-PUBLIC_LEAGUE = 39452
+PUBLIC_LEAGUE = 16
+PUBLIC_LEAGUE_IDP = 762
+SEARCH_PLAYER_GROUPS = ["QB", "WR", "RB", "TE", "K", "DEF"]
+SEARCH_PLAYER_GROUPS_IDP = ["QB", "WR", "RB", "TE", "K", "D", "DB", "DL", "LB"]
 
 
 def create_session():
     """Create requests session with retries and random user-agent"""
     s = requests.Session()
     s.headers = {
+        "Accept": "text/html",
+        "Accept-Encoding": "gzip, deflate, br",
+        "Accept-Language": "en-US",
         "Content-Type": "application/x-www-form-urlencoded;charset=utf-8",
         "User-Agent": generate_user_agent(),
     }
@@ -30,18 +36,19 @@ def create_session():
     return s
 
 
-def scrape(league):
+def scrape(league, is_IDP: bool = False):
     """Scrape data
 
     :param league: league ID
+    :param is_IDP: (bool) is this a individual defense player (IDP) league?
     """
 
     # Start timer
     startTime = datetime.now()
 
     # Scrape player IDs and teams from a public league
     data = set()
-    groups = ["QB", "WR", "RB", "TE", "K", "DEF"]
+    groups = SEARCH_PLAYER_GROUPS_IDP if is_IDP else SEARCH_PLAYER_GROUPS
     s = create_session()
     for group in groups:
         logger.info("Scraping all {}...".format(group))
@@ -51,7 +58,7 @@ def scrape(league):
             s.headers["User-Agent"] = generate_user_agent()
             r = s.get(
                 "https://football.fantasysports.yahoo.com/f1/{}/players".format(
-                    PUBLIC_LEAGUE
+                    PUBLIC_LEAGUE_IDP if is_IDP else PUBLIC_LEAGUE
                 ),
                 params=dict(
                     count=i * 25,
@@ -64,12 +71,14 @@ def scrape(league):
             i += 1
             soup = bs(r.text, "lxml")
             table = soup.select_one("#players-table table")
+            if not table:
+                break
             rows = table.select("tbody tr")
             if not rows:
                 break
             for row in rows:
                 td = row.select("td")[1]
-                ID = td.select_one("span.player-status a")["data-ys-playerid"]
+                ID = td.select_one(".player-status a")["data-ys-playerid"]
                 ID = int(ID)
                 team = td.select_one(".ysf-player-name span").text
                 team = team.split()[0]
@@ -115,7 +124,7 @@ def get_projections(row):
         row["% Owned"] = playerinfo.select_one("dd.owned").text.split()[0]
 
         # Weekly projections
-        df2 = pd.read_html(html)[0]
+        df2 = pd.read_html(StringIO(html))[0]
         for _, row2 in df2.iterrows():
             week = "Week {}".format(row2["Week"])
             points = row2["Fan Pts"]
@@ -149,7 +158,15 @@ def get_projections(row):
     columns = ["Week {}".format(i) for i in range(current_week(), 18)]
     df["Remaining"] = df[columns].sum(axis=1)
     available = df.loc[df["Owner ID"].isnull()]
-    means = available.groupby(["Position"])["Remaining"].nlargest(3).mean(level=0)
+    means = (
+        available.groupby(["Position"])["Remaining"].nlargest(3).groupby(level=0).mean()
+    )
+    for positions in means.index:
+        if "," in positions:
+            for position in positions.split(","):
+                position = position.strip()
+                if position not in means:
+                    means[position] = means[positions]
     df["VOR"] = df.apply(
         lambda row: row["Remaining"]
         - max(means[n.strip()] for n in row["Position"].split(",")),

ffbot/utils.py

@@ -1,7 +1,7 @@
+import re
 from datetime import datetime
 from os import listdir, makedirs
 from os.path import exists, getctime, isfile, join, split
-import re
 
 import pandas as pd
 

pyproject.toml

@@ -0,0 +1,2 @@
+[tool.isort]
+profile = "black"

requirements-test.txt

@@ -1,4 +1,6 @@
 black
 flake8
+isort[colors]
+pre-commit
 pytest
 pytest-cov

setup.py

@@ -1,9 +1,9 @@
 from io import open
 from os import path
-from setuptools import setup, find_packages
 
-from ffbot.constants import VERSION
+from setuptools import find_packages, setup
 
+from ffbot.constants import VERSION
 
 here = path.abspath(path.dirname(__file__))
 
@@ -32,7 +32,7 @@ def get_requirements(kind: str = None):
     author="Alex Marvin",
     author_email="alex.marvin@gmail.com",
     classifiers=[
-        "Development Status :: 3 - Alpha",
+        "Development Status :: 5 - Production/Stable",
         "License :: OSI Approved :: MIT License",
         "Programming Language :: Python :: 3",
     ],

tests/test_optimizer.py

@@ -1,8 +1,9 @@
 import pandas as pd
 
-from . import POSITIONS, SCRAPER_FILE, TEAM
 import ffbot
 
+from . import POSITIONS, SCRAPER_FILE, TEAM
+
 
 def test_optimize():
     df, week = ffbot.load(SCRAPER_FILE)