Package Size Analyzer (AI-5021)

.github/workflows/measure-disk-usage.yml

@@ -0,0 +1,125 @@
+name: Measure Disk Usage
+
+on:
+  push:
+    branches:
+    - master 
+env:
+  PYTHON_VERSION: "3.12"
+
+jobs:
+  measure-disk-usage:
+    runs-on: ubuntu-22.04
+    steps:
+    - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+      with:
+        fetch-depth: 0
+    - name: Set up Python ${{ env.PYTHON_VERSION }}
+      uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
+      with:
+        python-version: ${{ env.PYTHON_VERSION }}
+    - name: Install ddev
+      run: |
+        pip install -e ./datadog_checks_dev[cli]
+        pip install -e ./ddev
+
+    - name: Configure ddev
+      run: |
+        ddev config set repos.core .
+        ddev config set repo core
+    - name: Measure disk usage (uncompressed)
+      run: | 
+        mkdir -p status_visualizations
+        ddev size status --csv > size-uncompressed.csv
+        ddev size status  --save_to_png_path status_visualizations/uncompressed.png > size-uncompressed.txt
+        cat size-uncompressed.txt
+        echo "# Size (uncompressed)" >> $GITHUB_STEP_SUMMARY
+        echo '```' >> $GITHUB_STEP_SUMMARY
+        cat size-uncompressed.txt >> $GITHUB_STEP_SUMMARY
+        echo '```' >> $GITHUB_STEP_SUMMARY
+
+    - name: Measure disk usage (compressed)
+      run: |
+        mkdir -p status_visualizations
+        ddev size status --csv --compressed > size-compressed.csv
+        ddev size status --compressed  --save_to_png_path status_visualizations/compressed.png > size-compressed.txt
+        cat size-compressed.txt
+        echo "# Size (compressed)" >> $GITHUB_STEP_SUMMARY
+        echo '```' >> $GITHUB_STEP_SUMMARY
+        cat size-compressed.txt >> $GITHUB_STEP_SUMMARY
+        echo '```' >> $GITHUB_STEP_SUMMARY
+
+
+    - name: Measure disk usage differences from last commit (uncompressed)
+      if: false  # Disabled for now: size difference can be misleading due to dependencies not being built in the same PR
+      run: | 
+        mkdir -p diff_visualizations
+        BEFORE=$(git rev-parse HEAD^)
+        AFTER=$(git rev-parse HEAD)
+        ddev size diff $BEFORE $AFTER --csv > diff-uncompressed.csv
+        ddev size diff $BEFORE $AFTER  --save_to_png_path diff_visualizations/diff-uncompressed-linux.png > diff-uncompressed.txt
+        cat diff-uncompressed.txt
+        echo "# Size diff (uncompressed)" >> $GITHUB_STEP_SUMMARY
+        echo '```' >> $GITHUB_STEP_SUMMARY
+        cat diff-uncompressed.txt >> $GITHUB_STEP_SUMMARY
+        echo '```' >> $GITHUB_STEP_SUMMARY
+
+    - name: Measure disk usage differences from last commit (compressed)
+      if: false  # Disabled for now: size difference can be misleading due to dependencies not being built in the same PR
+      run: | 
+        mkdir -p diff_visualizations
+        BEFORE=$(git rev-parse HEAD^)
+        AFTER=$(git rev-parse HEAD)
+        ddev size diff $BEFORE $AFTER --compressed --csv > diff-compressed.csv
+        ddev size diff $BEFORE $AFTER --compressed  --save_to_png_path diff_visualizations/diff-compressed-linux.png > diff-compressed.txt
+        cat diff-compressed.txt
+        echo "# Size diff (compressed)" >> $GITHUB_STEP_SUMMARY
+        echo '```' >> $GITHUB_STEP_SUMMARY
+        cat diff-compressed.txt >> $GITHUB_STEP_SUMMARY
+        echo '```' >> $GITHUB_STEP_SUMMARY
+
+    - name: Upload file sizes (uncompressed)
+      uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
+      with:
+        name: size-uncompressed.csv
+        path: size-uncompressed.csv
+        if-no-files-found: error
+
+    - name: Upload file sizes (compressed)
+      uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
+      with:
+        name: size-compressed.csv
+        path: size-compressed.csv
+        if-no-files-found: error
+
+    - name: Upload file sizes diff (uncompressed)
+      if: false # Disabled for now: size difference can be misleading due to dependencies not being built in the same PR
+      uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
+      with:
+        name: diff-uncompressed.csv
+        path: diff-uncompressed.csv
+        if-no-files-found: error     
+
+    - name: Upload file sizes diff (compressed)
+      if: false  # Disabled for now: size difference can be misleading due to dependencies not being built in the same PR
+      uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
+      with:
+        name: diff-compressed.csv
+        path: diff-compressed.csv
+        if-no-files-found: error
+
+    - name: Upload status PNGs
+      uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
+      with:
+        name: size-visuals
+        path: status_visualizations/
+        if-no-files-found: error
+
+    - name: Upload diff PNGs
+      if: false # Disabled for now: size difference can be misleading due to dependencies not being built in the same PR
+      uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
+      with:
+        name: diff-visuals
+        path: diff_visualizations/
+        if-no-files-found: error
+

.github/workflows/slapr.yml

@@ -27,4 +27,4 @@ jobs:
         SLAPR_EMOJI_APPROVED: "approved2"
         SLAPR_EMOJI_CHANGES_REQUESTED: "changes_requested"
         SLAPR_EMOJI_MERGED: "merged"
-        SLAPR_EMOJI_CLOSED: "closed"
+        SLAPR_EMOJI_CLOSED: "closed"

ddev/changelog.d/20128.added

@@ -0,0 +1,4 @@
+Added new commands to track and analyze size changes in integrations and dependencies:
+- **`ddev size status`**: Shows current sizes of all modules.
+- **`ddev size diff [COMMIT_BEFORE] [COMMIT_AFTER]`**: Compares size changes between two commits.
+- **`ddev size timeline {integration | dependency} [INTEGRATION_NAME/DEPENDENCY_NAME]`**: Visualizes the size evolution of a module over time.

ddev/pyproject.toml

@@ -40,6 +40,9 @@ dependencies = [
     "tomli-w",
     "tomlkit",
     "tqdm",
+    "requests",
+    "matplotlib",
+    "squarify"
 ]
 dynamic = ["version"]
 

ddev/src/ddev/cli/__init__.py

@@ -18,6 +18,7 @@
 from ddev.cli.env import env
 from ddev.cli.meta import meta
 from ddev.cli.release import release
+from ddev.cli.size import size
 from ddev.cli.status import status
 from ddev.cli.test import test
 from ddev.cli.validate import validate
@@ -149,6 +150,7 @@ def ddev(
 ddev.add_command(status)
 ddev.add_command(test)
 ddev.add_command(validate)
+ddev.add_command(size)
 
 __management_command = os.environ.get('PYAPP_COMMAND_NAME', '')
 if __management_command:

ddev/src/ddev/cli/size/__init__.py

@@ -0,0 +1,30 @@
+# (C) Datadog, Inc. 2022-present
+# All rights reserved
+# Licensed under a 3-clause BSD style license (see LICENSE)
+
+import click
+
+from ddev.cli.size.diff import diff
+from ddev.cli.size.status import status
+from ddev.cli.size.timeline import timeline
+
+
+@click.group()
+def size():
+    """
+    Analyze the download size of integrations and dependencies in various modes.
+
+    This command provides tools to inspect the current status, compare commits and monitor size changes of modules
+    across different commits, platforms, and Python versions.
+
+    """
+
+    pass
+
+
+size.add_command(status)
+size.add_command(diff)
+size.add_command(timeline)
+
+if __name__ == "__main__":
+    size()