feat: Bulk build failure wiki (#948)

In order to have a `bulk` branch-specific Build Failures wiki page, need
to allow for a git range to be specified for which recipes will be
included. This uses the same method of determining whether a recipe is
included as the build command. (i.e., it's not diffing the build failure
files between branches.)

Also added handling for recipes that exist but have no corresponding
package in the channel. That avoids errors from new Bioconductor recipes
during a bulk update.

Author: aliciaaevans

bioconda_utils/build_failure.py

@@ -1,4 +1,5 @@
 import os
+import sys
 import time
 from typing import Optional, Union
 import subprocess as sp
@@ -13,6 +14,8 @@
 import pandas as pd
 import networkx as nx
 
+from .githandler import BiocondaRepo
+
 from bioconda_utils.recipe import Recipe
 from bioconda_utils import graph, utils
 from bioconda_utils.githandler import GitHandler
@@ -179,7 +182,7 @@ def category(self, value):
         self.inner["category"] = value
 
 
-def collect_build_failure_dataframe(recipe_folder, config, channel, link_fmt="txt", link_prefix=""):
+def collect_build_failure_dataframe(recipe_folder, config, channel, link_fmt="txt", link_prefix="", git_range=None):
     def get_build_failure_records(recipe):
         return filter(
             BuildFailureRecord.exists, 
@@ -190,6 +193,21 @@ def has_build_failure(recipe):
         return any(get_build_failure_records(recipe))
 
     recipes = list(utils.get_recipes(recipe_folder))
+
+    if git_range:
+        if not git_range or len(git_range) > 2:
+            sys.exit("--git-range may have only one or two arguments")
+        other = git_range[0]
+        ref = "HEAD" if len(git_range) == 1 else git_range[1]
+        repo = BiocondaRepo(recipe_folder)
+        changed_recipes = repo.get_recipes_to_build(ref, other)
+        logger.info("Constraining to %s git modified recipes%s.", len(changed_recipes),
+                    utils.ellipsize_recipes(changed_recipes, recipe_folder))
+        recipes = [recipe for recipe in recipes if recipe in set(changed_recipes)]
+        if len(recipes) != len(changed_recipes):
+            logger.info("Overlap was %s recipes%s.", len(recipes),
+                        utils.ellipsize_recipes(recipes, recipe_folder))
+
     dag, _ = graph.build(recipes, config)
 
     def get_data():
@@ -214,10 +232,11 @@ def get_data():
             recs = list(get_build_failure_records(recipe))
 
             failures = ", ".join(utils.format_link(rec.path, link_fmt, prefix=link_prefix, label=rec.platform) for rec in recs)
+            categories = ", ".join(rec.category for rec in recs)
             skiplisted = any(rec.skiplist for rec in recs)
             prs = utils.format_link(f"https://github.com/bioconda/bioconda-recipes/pulls?q=is%3Apr+is%3Aopen+{package}", link_fmt, label="show")
-            yield (recipe, downloads, descendants, skiplisted, failures, prs)
+            yield (recipe, downloads, descendants, skiplisted, categories, failures, prs)
 
-    data = pd.DataFrame(get_data(), columns=["recipe", "downloads", "depending", "skiplisted", "build failures", "pull requests"])
+    data = pd.DataFrame(get_data(), columns=["recipe", "downloads", "depending", "skiplisted", "category", "build failures", "pull requests"])
     data.sort_values(by=["depending", "downloads"], ascending=False, inplace=True)
     return data

bioconda_utils/cli.py

@@ -1075,7 +1075,11 @@ def annotate_build_failures(recipes, skiplist=False, reason=None, category=None,
 @arg('--channel', help="Channel with packages to check", default="bioconda")
 @arg('--output-format', help="Output format", choices=['txt', 'markdown'], default="txt")
 @arg('--link-prefix', help="Prefix for links to build failures", default='')
-def list_build_failures(recipe_folder, config, channel=None, output_format=None, link_prefix=None):
+@arg('--git-range', nargs='+',
+     help='''Git range (e.g. commits or something like
+     "master HEAD" to check commits in HEAD vs master, or just "HEAD" to
+     include uncommitted changes).''')
+def list_build_failures(recipe_folder, config, channel=None, output_format=None, link_prefix=None, git_range=None):
     """List recipes with build failure records"""
 
     df = collect_build_failure_dataframe(
@@ -1084,6 +1088,7 @@ def list_build_failures(recipe_folder, config, channel=None, output_format=None,
         channel,
         link_fmt=output_format,
         link_prefix=link_prefix,
+        git_range=git_range
     )
     if output_format == "markdown":
         fmt_writer = pandas.DataFrame.to_markdown

bioconda_utils/utils.py

@@ -1645,4 +1645,6 @@ def yaml_remove_invalid_chars(text: str, valid_chars_re=re.compile(r"[^ \t\n\w\d
 def get_package_downloads(channel, package):
     """Use anaconda API to obtain download counts."""
     data = requests.get(f"https://api.anaconda.org/package/{channel}/{package}").json()
-    return sum(rec["ndownloads"] for rec in data["files"])
+    if "files" in data:
+        return sum(rec["ndownloads"] for rec in data["files"])
+    return 0