.gitlab-ci.yml

# Copyright (c) 2023 NVIDIA CORPORATION.  All rights reserved.
# NVIDIA CORPORATION and its licensors retain all intellectual property
# and proprietary rights in and to this software, related documentation
# and any modifications thereto.  Any use, reproduction, disclosure or
# distribution of this software and related documentation without an express
# license agreement from NVIDIA CORPORATION is strictly prohibited.

# ==============================================================================
# CI/CD Pipeline Configuration
# ==============================================================================

include:
  - local: /.gitlab/ci/common.yml
  - project: "omniverse/devplat/gitlab/templates/common/compliance"
    file: "modules/omniverse-repo-compliance.gitlab-ci.yml"
    ref: v1_latest

workflow:
  rules:
    - if: $CI_PROJECT_ROOT_NAMESPACE != "omniverse" # Prevent pipelines that can't access the runners
      when: never
    - if: $CI_PIPELINE_SOURCE == 'merge_request_event'
    - if: $CI_MERGE_REQUEST_TARGET_BRANCH_SHA
    - if: $CI_PIPELINE_SOURCE == "schedule" || $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH 
      auto_cancel:
        on_new_commit: none
        on_job_failure: none
    - if: $CI_COMMIT_TAG # Run for tagged releases
    - if: $CI_COMMIT_BRANCH =~ /^release-.*/
    - if: $CI_PIPELINE_SOURCE == "web" # Run if triggered from the UI

variables:
  PM_PACKAGES_ROOT: '$CI_PROJECT_DIR/packman-repo'
  PIP_CACHE_DIR: '$CI_PROJECT_DIR/.cache/pip'
  CUDA_BIN: '$CI_PROJECT_DIR/_build/target-deps/cuda/bin'
  CUDA: '$CI_PROJECT_DIR/_build/target-deps/cuda'
  CUDA_HOME: '$CI_PROJECT_DIR/_build/target-deps/cuda'
  PYTHON: '$CI_PROJECT_DIR/_build/target-deps/python/python'
  LINBUILD: '$CI_PROJECT_DIR/_build/host-deps/linbuild/linbuild.sh'
  WARP_CACHE_ROOT: '$CI_PROJECT_DIR/.cache/warp' # Used by the parallel test runner
  GIT_DEPTH: 1
  DEFAULT_PYTHON:
    value: "3.9.18+nv1"
    options:
      - "3.11.8+nv1"
      - "3.10.13+nv3"
      - "3.9.18+nv1"
      - "3.8.18+nv1"
    description: "The default Python version used in the main testing jobs."

stages:
  - build
  - test
  - child pipelines
  - package
  - deploy

# ==============================================================================
# Build Jobs (Release)
# ==============================================================================

linux-aarch64 build:
  stage: build
  image: ubuntu:20.04
  extends:
    - .runner-build-linux-aarch64
    - .save_warp_bin_artifact
  before_script:
    - echo -e "\\e[0Ksection_start:`date +%s`:install_dependencies[collapsed=true]\\r\\e[0KInstalling dependencies"
    - apt-get update && apt-get install build-essential curl --no-install-recommends -y
    - >
      curl -k -H "Authorization: Bearer $ARTIFACTORY_ACCESS_TOKEN"
      $ARTIFACTORY_BASE_URL/$ARTIFACTORY_LIBMATHDX_PATH/libmathdx-Linux-aarch64-0.1.1.tar.gz
      -o libmathdx.tar.gz
    - mkdir -p _build/target-deps
    - tar -xzf libmathdx.tar.gz -C _build/target-deps
    - export LIBMATHDX_HOME="$CI_PROJECT_DIR/_build/target-deps/libmathdx"
    - gcc --version
    - echo -e "\\e[0Ksection_end:`date +%s`:install_dependencies\\r\\e[0K"
  script:
    - ./tools/ci/building/build-linux-aarch64/build.sh --no-docker
    - mkdir -p warp/bin/linux-aarch64
    - mv warp/bin/warp.so warp/bin/linux-aarch64
    - mv warp/bin/warp-clang.so warp/bin/linux-aarch64

linux-x86_64 build:
  stage: build
  image: ubuntu:20.04
  extends:
    - .save_warp_bin_artifact
    - .runner-build-linux-x86_64
  before_script:
    - echo -e "\\e[0Ksection_start:`date +%s`:install_dependencies[collapsed=true]\\r\\e[0KInstalling dependencies"
    - apt-get update && apt-get install build-essential curl --no-install-recommends -y
    - >
      curl -k -H "Authorization: Bearer $ARTIFACTORY_ACCESS_TOKEN"
      $ARTIFACTORY_BASE_URL/$ARTIFACTORY_LIBMATHDX_PATH/libmathdx-Linux-x86_64-0.1.1.tar.gz
      -o libmathdx.tar.gz
    - mkdir -p _build/target-deps
    - tar -xzf libmathdx.tar.gz -C _build/target-deps
    - export LIBMATHDX_HOME="$CI_PROJECT_DIR/_build/target-deps/libmathdx"
    - gcc --version
    - echo -e "\\e[0Ksection_end:`date +%s`:install_dependencies\\r\\e[0K"
  script:
    - ./tools/ci/building/build-linux-x86_64/build.sh --no-docker
    - mkdir -p warp/bin/linux-x86_64
    - mv warp/bin/warp.so warp/bin/linux-x86_64
    - mv warp/bin/warp-clang.so warp/bin/linux-x86_64

windows-x86_64 build:
  stage: build
  extends:
    - .save_warp_bin_artifact
    - .runner-build-windows-x86_64
  before_script:
    - powershell -command "Get-Volume | Format-Table -AutoSize"
    - |
      $headers = @{
          "Authorization" = "Basic $([Convert]::ToBase64String([Text.Encoding]::ASCII.GetBytes("warp-cicd:${env:LIBMATHDX_REGISTRY_TOKEN}")))"
      }
      $url = "${CI_API_V4_URL}/projects/141992/packages/generic/libmathdx/0-1-1-0/libmathdx-win64-x86_64-0.1.1.zip"
      $output = "libmathdx.zip"
      Invoke-WebRequest -Uri $url -Headers $headers -OutFile $output -ErrorAction Stop
    - Expand-Archive -Path libmathdx.zip -DestinationPath _build/target-deps -Force
    - Get-ChildItem -Path "${env:CI_PROJECT_DIR}\_build\target-deps"
    - $env:LIBMATHDX_HOME = "${env:CI_PROJECT_DIR}\_build\target-deps\libmathdx"
  script:
    - ./tools/ci/building/build-windows-x86_64/build.bat

mac-x86_64 build:
  stage: build
  extends:
    - .save_warp_bin_artifact
    - .runner-build-macos-universal
    - .macos_warp_tags
  script:
    - ./tools/ci/building/build-linux-x86_64/build.sh

# ==============================================================================
# Linting Jobs
#
# The jobs here are meant to assist with code quality analysis.
# They can run immediately without waiting for the build jobs to complete.
# ==============================================================================

ruff lint:
  stage: test
  image: python:3.11-slim
  needs: []
  extends:
    - .runner-utility-linux-x86_64
  before_script:
    - python -m pip install --upgrade pip
    - pip install ruff --constraint docs/requirements.txt
  script:
    - ruff check --output-format full  --exit-zero # Just to get something in the log
    - ruff check --output-format gitlab > gl-code-quality-report.json
  artifacts:
    reports:
      codequality: gl-code-quality-report.json

ruff format:
  stage: test
  image: python:3.11-slim
  needs: []
  extends:
    - .runner-utility-linux-x86_64
  before_script:
    - python -m pip install --upgrade pip
    - pip install ruff --constraint docs/requirements.txt
  script:
    - ruff format --diff

osec:sonarqube:
  variables:
    # Disable C/C++ analyzer until project specific work is done to enable it.
    # See: https://confluence.nvidia.com/display/OMNIVERSE/SonarQube+Gitlab+CI+Integration#C+Project+Enablement+Additions
    SONAR_EXTRA_ARGS: "-Dsonar.c.file.suffixes=- -Dsonar.cpp.file.suffixes=- -Dsonar.objc.file.suffixes=-"

# ==============================================================================
# Main Unit Testing Jobs
#
# The jobs here will always be run when the pipeline is triggered. The jobs that
# compute code coverage run slower than jobs that do not. The minimal jobs were
# added to test the user experience without any optional Python packages.
# ==============================================================================

.test_common_main:
  stage: test
  coverage: '/(?i)total.*? (100(?:\.0+)?\%|[1-9]?\d(?:\.\d+)?\%)$/'
  artifacts:
    when: always
    paths:
      - rspec.xml
      - coverage.xml
    reports:
      junit: rspec.xml
      coverage_report:
        coverage_format: cobertura
        path: coverage.xml
  rules:
    - if: $CI_PIPELINE_SOURCE == "schedule" || $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH 
    - if: $CI_COMMIT_TAG
    - if: $CI_COMMIT_BRANCH =~ /^release-.*/
    - if: $CI_PIPELINE_SOURCE == "web"
    - if: $CI_PIPELINE_SOURCE == 'merge_request_event'
      changes:
        - .gitlab/ci/*
        - warp/**/*
        - pyproject.toml
        - tools/**/*
        - deps/*
        - build_lib.py
        - build_llvm.py
    - when: manual # If not auto-triggered, allow any pipeline to run this job manually
      allow_failure: true

linux-aarch64 test:
  image: ubuntu:22.04
  needs: [linux-aarch64 build]
  extends:
    - .test_common_main
  allow_failure: true # Unsure of stability and availability of devplat ARM GPU runners
  before_script:
    - echo -e "\\e[0Ksection_start:`date +%s`:install_dependencies[collapsed=true]\\r\\e[0KInstalling dependencies"
    - !reference [.snippets, install-python+warp-aarch64]
    - python -m pip install coverage[toml]
    - echo -e "\\e[0Ksection_end:`date +%s`:install_dependencies\\r\\e[0K"
  script:
    - python -m warp.tests --junit-report-xml rspec.xml --coverage --coverage-xml coverage.xml -s autodetect --failfast
  tags:  # TODO: Use .omni_devplat_arm_docker_gpu
    - arch/arm
    - os/linux
    - group/devplat
    - platform/horde
    - gpu/any

linux-aarch64 test jetson:
  image: ubuntu:22.04
  needs: [linux-aarch64 build]
  extends:
    - .test_common_main
  rules:
    - if: $CI_PIPELINE_SOURCE == "schedule" || $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH 
    - if: $CI_COMMIT_TAG
    - if: $CI_COMMIT_BRANCH =~ /^release-.*/
    - when: manual # If not auto-triggered, allow any pipeline to run this job manually
  allow_failure: true # Unsure of stability of Jetson runner for now
  before_script:
    - echo -e "\\e[0Ksection_start:`date +%s`:install_dependencies[collapsed=true]\\r\\e[0KInstalling dependencies"
    - !reference [.snippets, install-python+warp-aarch64]
    - python -m pip install coverage[toml]
    - python -m pip install -U "jax[cuda12]"
    - echo -e "\\e[0Ksection_end:`date +%s`:install_dependencies\\r\\e[0K"
  script:
    - python -m warp.tests --junit-report-xml rspec.xml --coverage --coverage-xml coverage.xml -s autodetect --failfast
  tags:
    - gpu/orin

linux-x86_64 test:
  needs: [linux-x86_64 build]
  extends:
    - .omni_nvks_gpu_2x
    - .test_common_main
  before_script:
    - echo -e "\\e[0Ksection_start:`date +%s`:install_dependencies[collapsed=true]\\r\\e[0KInstalling dependencies"
    - df -h
    # Move compiled binaries out of platform-specific directory
    - mv warp/bin/linux-x86_64/warp.so warp/bin/
    - mv warp/bin/linux-x86_64/warp-clang.so warp/bin/
    - tools/packman/packman install -l _build/target-deps/python python ${DEFAULT_PYTHON}-linux-x86_64
    - export PATH="$CUDA_BIN:$PATH"
    - $PYTHON -m venv _venv
    - source _venv/bin/activate
    - python -m pip install --upgrade pip
    - python -m pip install --upgrade usd-core coverage[toml] blosc
    - python -m pip install --upgrade torch --extra-index-url https://download.pytorch.org/whl/cu121
    - python -m pip install -U "jax[cuda12]"
    - python -m pip install -e .
    - echo -e "\\e[0Ksection_end:`date +%s`:install_dependencies\\r\\e[0K"
    # HACK: disable P2P tests due to misbehaving agents
    - export WARP_DISABLE_P2P_TESTS=1
  script:
    - python -m warp.tests --junit-report-xml rspec.xml --coverage --coverage-xml coverage.xml -s autodetect --failfast

# Note that for throughput reasons this runs on a single-GPU runner
windows-x86_64 test:
  stage: test
  needs: [windows-x86_64 build]
  extends:
    - .runner-test-windows-x86_64-gpu
    - .test_common_main
  before_script:
    - !reference [.snippets, define-powershell-GetTime]
    - Write-Output "$([char]27)[0Ksection_start:$(GetTime):install_dependencies[collapsed=true]$([char]13)$([char]27)[0KInstalling dependencies"
    - powershell -command "Get-Volume | Format-Table -AutoSize"
    - $python_name = $DEFAULT_PYTHON + "-windows-x86_64"
    - tools/packman/packman.cmd install -l _build/target-deps/python python $python_name
    - '& $env:CI_PROJECT_DIR\_build\target-deps\python\python.exe -m venv _venv'
    - .\_venv\Scripts\Activate.ps1
    - python -m pip install --upgrade pip
    - python -m pip install --upgrade usd-core coverage[toml] numpy blosc
    - python -m pip install --upgrade torch --extra-index-url https://download.pytorch.org/whl/cu121
    - python -m pip install -e .
    - Write-Output "$([char]27)[0Ksection_end:$(GetTime):install_dependencies$([char]13)$([char]27)[0K"
  script:
    - python -m warp.tests --junit-report-xml rspec.xml --coverage --coverage-xml coverage.xml -s autodetect --failfast

mac-x86_64 test:
  stage: test
  needs: [mac-x86_64 build]
  extends:
    - .runner-test-macos-universal
    - .test_common_main
    - .macos_warp_tags
  before_script:
    - echo -e "\\e[0Ksection_start:`date +%s`:install_dependencies[collapsed=true]\\r\\e[0KInstalling dependencies"
    - !reference [.snippets, install-python+warp-macos]
    - python -m pip install --upgrade usd-core coverage[toml]
    - echo -e "\\e[0Ksection_end:`date +%s`:install_dependencies\\r\\e[0K"
  script:
    - python -m warp.tests --maxjobs 4 --junit-report-xml rspec.xml --coverage --coverage-xml coverage.xml -s autodetect --failfast

# Test the Linux release build in an environment with minimal Python dependencies installed
linux-x86_64 test minimal:
  stage: test
  needs: [linux-x86_64 build]
  extends:
    - .omni_nvks_gpu_2x
    - .save_test_report_artifact
  rules:
    - if: $CI_PIPELINE_SOURCE == "schedule" || $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH 
    - if: $CI_COMMIT_TAG
    - if: $CI_COMMIT_BRANCH =~ /^release-.*/
    - if: $CI_PIPELINE_SOURCE == "web"
    - if: $CI_PIPELINE_SOURCE == 'merge_request_event'
      changes:
        - .gitlab/ci/*
        - warp/**/*
        - pyproject.toml
        - tools/**/*
        - deps/*
        - build_lib.py
        - build_llvm.py
    - when: manual # If not auto-triggered, allow any pipeline to run this job manually
      allow_failure: true
  before_script:
    - echo -e "\\e[0Ksection_start:`date +%s`:install_dependencies[collapsed=true]\\r\\e[0KInstalling dependencies"
    - df -h
    # Move compiled binaries out of platform-specific directory
    - mv warp/bin/linux-x86_64/warp.so warp/bin/
    - mv warp/bin/linux-x86_64/warp-clang.so warp/bin/
    - tools/packman/packman install -l _build/target-deps/python python ${DEFAULT_PYTHON}-linux-x86_64
    - $PYTHON -m venv _venv
    - source _venv/bin/activate
    - python -m pip install --upgrade pip
    - python -m pip install -e .
    - echo -e "\\e[0Ksection_end:`date +%s`:install_dependencies\\r\\e[0K"
    # HACK: disable P2P tests due to misbehaving agents
    - export WARP_DISABLE_P2P_TESTS=1
  script:
    - python -m warp.tests --junit-report-xml rspec.xml -s autodetect --failfast

# Optional multi-GPU Windows job (likely longer queue time)
windows-x86_64 test mgpu:
  stage: test
  needs: [windows-x86_64 build]
  extends:
    - .save_test_report_artifact
  rules:
    - if: $CI_PIPELINE_SOURCE == "schedule" || $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH 
    - if: $CI_COMMIT_TAG
    - when: manual # Can be triggered in all other scenarios
      allow_failure: true
  before_script:
    - !reference [.snippets, define-powershell-GetTime]
    - Write-Output "$([char]27)[0Ksection_start:$(GetTime):install_dependencies[collapsed=true]$([char]13)$([char]27)[0KInstalling dependencies"
    - powershell -command "Get-Volume | Format-Table -AutoSize"
    - $python_name = $DEFAULT_PYTHON + "-windows-x86_64"
    - tools/packman/packman.cmd install -l _build/target-deps/python python $python_name
    - '& $env:CI_PROJECT_DIR\_build\target-deps\python\python.exe -m venv _venv'
    - .\_venv\Scripts\Activate.ps1
    - python -m pip install --upgrade pip
    - python -m pip install --upgrade usd-core numpy
    - python -m pip install --upgrade torch --extra-index-url https://download.pytorch.org/whl/cu121
    - python -m pip install -e .
    - Write-Output "$([char]27)[0Ksection_end:$(GetTime):install_dependencies$([char]13)$([char]27)[0K"
  script:
    - python -m warp.tests --junit-report-xml rspec.xml -s autodetect --failfast
  tags:
    - os/windows
    - gpu/2x-A5000

# Note: This will no longer be needed once Warp auto-initializes upon import
linux-x86_64 test warp-init:
  stage: test
  needs: [linux-x86_64 build]
  extends:
    - .omni_nvks_gpu
    - .save_test_report_artifact
  rules:
    - if: $CI_PIPELINE_SOURCE == "schedule" || $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH 
    - if: $CI_COMMIT_TAG
    - if: $CI_COMMIT_BRANCH =~ /^release-.*/
    - if: $CI_PIPELINE_SOURCE == "web"
    - if: $CI_PIPELINE_SOURCE == 'merge_request_event'
      changes:
        - .gitlab/ci/*
        - warp/**/*
        - pyproject.toml
        - tools/**/*
        - deps/*
        - build_lib.py
        - build_llvm.py
    - when: manual # If not auto-triggered, allow any pipeline to run this job manually
      allow_failure: true
  timeout: 10m
  before_script:
    - echo -e "\\e[0Ksection_start:`date +%s`:install_dependencies[collapsed=true]\\r\\e[0KInstalling dependencies"
    - df -h
    # Move compiled binaries out of platform-specific directory
    - mv warp/bin/linux-x86_64/warp.so warp/bin/
    - mv warp/bin/linux-x86_64/warp-clang.so warp/bin/
    - tools/packman/packman install -l _build/target-deps/python python ${DEFAULT_PYTHON}-linux-x86_64
    - $PYTHON -m venv _venv
    - source _venv/bin/activate
    - python -m pip install --upgrade pip
    - python -m pip install -e .
    - echo -e "\\e[0Ksection_end:`date +%s`:install_dependencies\\r\\e[0K"
  script:
    - python -m warp.tests --junit-report-xml rspec.xml -s autodetect --disable-process-pooling --disable-concurrent-futures --level test -p 'test_implicit_init.py'

# The only purpose of this job is to make sure documentation can be built on Windows.
# The output does not get published anywhere, but the website can be viewed in the
# artifacts.
windows-x86_64 docs:
  stage: test
  needs: [windows-x86_64 build]
  extends:
    - .runner-utility-windows-x86_64
  artifacts:
    paths:
      - public
  before_script:
    - !reference [.snippets, define-powershell-GetTime]
    - Write-Output "$([char]27)[0Ksection_start:$(GetTime):install_dependencies[collapsed=true]$([char]13)$([char]27)[0KInstalling dependencies"
    - powershell -command "Get-Volume | Format-Table -AutoSize"
    - $python_name = "3.12.6+nv1-windows-x86_64"
    - tools/packman/packman.cmd install -l _build/target-deps/python python $python_name
    - '& $env:CI_PROJECT_DIR\_build\target-deps\python\python.exe -m venv _venv'
    - .\_venv\Scripts\Activate.ps1
    - python -m pip install --upgrade pip
    - python -m pip install -r docs/requirements.txt
    - Write-Output "$([char]27)[0Ksection_end:$(GetTime):install_dependencies$([char]13)$([char]27)[0K"
  script:
    - python build_docs.py --quick
    - mv docs/_build/html/ ./public/
  after_script:
    - echo "View the website at https://$CI_PROJECT_ROOT_NAMESPACE.$CI_PAGES_DOMAIN/-/$CI_PROJECT_NAME/-/jobs/$CI_JOB_ID/artifacts/public/index.html"

# This job tests code snippets in the documentation
doctest:
  stage: test
  image: python:3.11-slim
  needs: [linux-x86_64 build]
  extends:
    - .omni_nvks_gpu
  before_script:
    - echo -e "\\e[0Ksection_start:`date +%s`:install_dependencies[collapsed=true]\\r\\e[0KSet up docs environment"
    - df -h
    - apt-get update && apt-get install make --no-install-recommends -y
    # Move compiled binaries out of platform-specific directory
    - mv warp/bin/linux-x86_64/warp.so warp/bin/
    - mv warp/bin/linux-x86_64/warp-clang.so warp/bin/
    - python -m pip install --upgrade pip
    - python -m pip install -r docs/requirements.txt
    - echo -e "\\e[0Ksection_end:`date +%s`:install_dependencies\\r\\e[0K"
  script:
    - sphinx-build -b doctest docs docs/_build/doctest

# Benchmark this commit with the current main branch using Airspeed Velocity
# Only runs on merge requests
linux-x86_64 asv:
  stage: test
  needs: []
  image: ubuntu:22.04
  extends:
    - .omni_nvks_gpu
  variables:
    GIT_DEPTH: "50"
  rules:
    - if: $CI_PIPELINE_SOURCE == 'merge_request_event'
      changes:
        - .gitlab-ci.yml
        - benchmarks/**/*
        - warp/**/*
        - pyproject.toml
        - deps/*
        - build_lib.py
        - build_llvm.py
  before_script:
    - echo -e "\\e[0Ksection_start:`date +%s`:install_dependencies[collapsed=true]\\r\\e[0KInstalling dependencies"
    - df -h
    - apt-get update && apt-get install python3 python3.10-venv git-lfs build-essential curl ca-certificates --no-install-recommends -y
    - tools/packman/packman pull --platform linux-x86_64 deps/cuda-toolkit-deps.packman.xml --verbose --include-tag "cuda-11"
    - export PATH="$CUDA_BIN:$PATH"
    - python3 -m venv _venv
    - source _venv/bin/activate
    - python -m pip install --upgrade pip
    - python -m pip install asv
    - export GIT_LFS_SKIP_SMUDGE=1
    - git config --global --add safe.directory '*' 
    - git fetch origin main
    # Make sure the main branch can be checked out by asv
    - git checkout -f main
    - git checkout -f $CI_COMMIT_SHA
    - echo -e "\\e[0Ksection_end:`date +%s`:install_dependencies\\r\\e[0K"
  script:
    - cd benchmarks
    - asv machine --yes
    - asv continuous --interleave-rounds --append-samples --no-only-changed -f 1.15 main $CI_COMMIT_SHA
  allow_failure: True # Can make success required after we have tested this job more

# ==============================================================================
# Child pipelines
#
# The child pipelines defined here are only run in specific
# circumstances. Most developers don't need to worry about the
# jobs in this section.
# ==============================================================================

# Trigger additional (no code coverage) pipelines testing more Python versions
# Workaround from https://gitlab.com/gitlab-org/gitlab/-/issues/284086
trigger python 3.X test pipelines:
  stage: test
  image: busybox
  extends:
    - .runner-utility-linux-x86_64
  rules:
    - if: $CI_PIPELINE_SOURCE == "schedule"
    - if: $CI_COMMIT_TAG
    - if: $CI_COMMIT_BRANCH =~ /^release-.*/
    - when: manual # Can be triggered in all other scenarios
      allow_failure: true
  variables:
    GIT_STRATEGY: none
  script:
    - echo "Run this job to test additional Python versions."

python 3.8 test:
  stage: child pipelines
  needs: [trigger python 3.X test pipelines]
  trigger:
    include: /.gitlab/ci/additional-tests.yml
  extends:
    - .trigger_common
  variables:
    DEFAULT_PYTHON: "3.8.18+nv1"

# Trigger debug build and test (no code coverage) pipelines
# Workaround from https://gitlab.com/gitlab-org/gitlab/-/issues/284086
trigger debug build and test pipeline:
  stage: test
  image: busybox
  extends:
    - .runner-utility-linux-x86_64
  needs: []
  rules:
    - if: $CI_PIPELINE_SOURCE == "schedule"
    - if: $CI_COMMIT_TAG
    - if: $CI_COMMIT_BRANCH =~ /^release-.*/
    - when: manual # Can be triggered in all other scenarios
      allow_failure: true
  variables:
    GIT_STRATEGY: none
  script:
    - echo "Run this job to test Warp compiled in debug mode."

# Uses the same Python version as the main pipeline.
debug build and test:
  stage: child pipelines
  needs: [trigger debug build and test pipeline]
  trigger:
    include: /.gitlab/ci/debug-build-and-test.yml
  extends:
    - .trigger_common

# Trigger CUDA 11 pipelines
# Workaround from https://gitlab.com/gitlab-org/gitlab/-/issues/284086
trigger cuda 11 pipeline:
  stage: test
  image: busybox
  extends:
    - .runner-utility-linux-x86_64
  needs: []
  rules:
    - if: $CI_PIPELINE_SOURCE == "schedule"
    - if: $CI_COMMIT_TAG
    - if: $CI_COMMIT_BRANCH =~ /^release-.*/
    - when: manual # Can be triggered in all other scenarios
      allow_failure: true
  variables:
    GIT_STRATEGY: none
  script:
    - echo "Run this job to test Warp compiled with CUDA 11."

# Uses the same Python version as the main pipeline.
cuda 11 build and test:
  stage: child pipelines
  needs: [trigger cuda 11 pipeline]
  trigger:
    include: /.gitlab/ci/cuda-11-build-and-test.yml
  extends:
    - .trigger_common

# ==============================================================================
# Packaging Jobs
#
# Kit and PyPI jobs produce deployment artifacts and are not run in MRs unless
# manually started.
# ==============================================================================

# Creates wheel files for PyPI
create pypi wheels:
  stage: package
  needs:
    - linux-aarch64 build
    - linux-x86_64 build
    - windows-x86_64 build
    - mac-x86_64 build
  extends:
    - .runner-utility-linux-x86_64
  rules:
    - if: $CI_PIPELINE_SOURCE == "schedule"
    - if: $CI_COMMIT_TAG
    - if: $CI_COMMIT_BRANCH =~ /^release-.*/
    - if: $CI_PIPELINE_SOURCE == 'merge_request_event'
      changes:
        - pyproject.toml
        - setup.py
    - when: manual # If not auto-triggered, allow any pipeline to run this job manually
      allow_failure: true
  before_script:
    # Move binaries into platform-specific folders. Already done in the build jobs for Linux.
    - mkdir -p warp/bin/windows-x86_64
    - mv warp/bin/warp.dll warp/bin/windows-x86_64/
    - mv warp/bin/warp-clang.dll warp/bin/windows-x86_64/
    - mkdir -p warp/bin/macos-universal
    - mv warp/bin/libwarp.dylib warp/bin/macos-universal/
    - mv warp/bin/libwarp-clang.dylib warp/bin/macos-universal/
    - python3 -m pip install --upgrade pip
    - python3 -m pip install build
  script:
    - sed -i "s/^\(.*\)$/\1+cu12/" VERSION.md  # Modify VERSION.md with +cu12
    - python3 -m build --wheel -C--build-option=-Pwindows-x86_64
    - python3 -m build --wheel -C--build-option=-Plinux-x86_64
    - python3 -m build --wheel -C--build-option=-Plinux-aarch64
    - sed -i "s/^\(.*\)+cu12$/\1+cpu/" VERSION.md  # Modify VERSION.md with +cu12 replaced by +cpu
    - python3 -m build --wheel -C--build-option=-Pmacos-universal
    - sed -i "s/^\(.*\)+cpu$/\1/" VERSION.md  # Revert VERSION.md changes
    - mv dist dist-github
    # Now make the wheels meant for PyPI publishing
    - python3 -m build --wheel -C--build-option=-Pwindows-x86_64
    - python3 -m build --wheel -C--build-option=-Plinux-x86_64
    - python3 -m build --wheel -C--build-option=-Plinux-aarch64
    - python3 -m build --wheel -C--build-option=-Pmacos-universal
    - find . -type f -exec chmod 664 {} +
    - find . -type d -exec chmod 775 {} +
  artifacts:
    name: $CI_COMMIT_REF_SLUG-$CI_COMMIT_SHORT_SHA
    expose_as: "Python Wheels"
    paths:
      - "dist/"
      - "dist-github/"
    when: always

create dev wheels:
  stage: package
  needs:
    - linux-aarch64 build
    - linux-x86_64 build
    - windows-x86_64 build
  extends:
    - .runner-utility-linux-x86_64
  rules:
    - if: $CI_PIPELINE_SOURCE == "schedule" && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
    - when: never
  before_script:
    # Move binaries into platform-specific folders. Already done in the build jobs for Linux.
    - mkdir -p warp/bin/windows-x86_64
    - mv warp/bin/warp.dll warp/bin/windows-x86_64/
    - mv warp/bin/warp-clang.dll warp/bin/windows-x86_64/
    - python3 -m pip install --upgrade pip
    - python3 -m pip install build
  script:
    - python3 tools/ci/publishing/set_nightly_version.py  # Modify VERSION.md with dev version
    - python3 -m build --wheel -C--build-option=-Pwindows-x86_64
    - python3 -m build --wheel -C--build-option=-Plinux-x86_64
    - python3 -m build --wheel -C--build-option=-Plinux-aarch64
    - find . -type f -exec chmod 664 {} +
    - find . -type d -exec chmod 775 {} +
    - cp VERSION.md dist/
  artifacts:
    name: $CI_COMMIT_REF_SLUG-$CI_COMMIT_SHORT_SHA-dev
    paths:
      - "dist/"

# ==============================================================================
# Deployment Jobs
#
# This section currently contains jobs that publish files to the internal
# GitLab service.
# ==============================================================================

publish wheels to testpypi registry:
  stage: deploy
  image: python:3.11-slim
  needs: ["create pypi wheels"]
  extends:
    - .runner-utility-linux-x86_64
  rules:
    - if: $CI_COMMIT_BRANCH =~ /release-.*/ || $CI_COMMIT_TAG
      when: manual
      allow_failure: true
  environment:
    name: staging
    url: https://test.pypi.org/project/warp-lang/
  before_script:
    - python3 -m pip install --upgrade pip
    - python3 -m pip install --upgrade build twine
  script:
    - python3 -m twine upload --verbose --skip-existing --non-interactive --repository testpypi dist/* -u __token__ -p $TESTPYPI_DEPLOY_KEY

publish wheels to pypi registry:
  stage: deploy
  image: python:3.11-slim
  needs: ["create pypi wheels"]
  extends:
    - .runner-utility-linux-x86_64
  rules:
    - if: $CI_COMMIT_TAG
      when: manual
      allow_failure: true
  environment:
    name: production
    url: https://pypi.org/project/warp-lang/
  before_script:
    - python3 -m pip install --upgrade pip
    - python3 -m pip install --upgrade build twine
  script:
    - python3 -m twine upload --verbose --skip-existing --non-interactive dist/* -u __token__ -p $PYPI_DEPLOY_KEY

publish wheels to gitlab pypi registry:
  stage: deploy
  image: python:3.11-slim
  needs: ["create pypi wheels"]
  extends:
    - .runner-utility-linux-x86_64
  rules:
    - if: $CI_COMMIT_BRANCH =~ /^release-.*/ || $CI_COMMIT_TAG
      when: manual
      allow_failure: true
  before_script:
    - python3 -m pip install --upgrade pip
    - python3 -m pip install --upgrade build twine
  script:
    - TWINE_PASSWORD=${CI_JOB_TOKEN} TWINE_USERNAME=gitlab-ci-token python3 -m twine upload --verbose --skip-existing --non-interactive --repository-url ${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/pypi dist-github/*

# Uploads the wheels to the internal GitLab package registry in the Warp project
# Generated files will be in a branch/tag-specific folder
publish wheels to gitlab package registry:
  stage: deploy
  needs: ["create pypi wheels"]
  extends:
    - .runner-utility-linux-x86_64
  rules:
    - if: $CI_COMMIT_TAG
    - if: $CI_COMMIT_BRANCH =~ /^release-.*/
    - when: manual # If not auto-triggered, allow any pipeline to run this job manually
      allow_failure: true
  before_script:
    - apt-get update && apt-get install curl --no-install-recommends -y
  script:
    - |
      for file in $(find . -name '*.whl'); do
          filename=$(basename -- "$file")
          curl --header "JOB-TOKEN: $CI_JOB_TOKEN" --upload-file "$file" "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/generic/warp/${CI_COMMIT_REF_SLUG}/${filename}"
      done
    - echo "See the published files at $CI_PROJECT_URL/-/packages"

publish dev wheels to artifactory:
  stage: deploy
  image: releases-docker.jfrog.io/jfrog/jfrog-cli-v2-jf
  needs:
    - job: create dev wheels
      optional: true
  extends:
    - .runner-utility-linux-x86_64
  rules: # Should be consistent with create dev wheels
    - if: $CI_PIPELINE_SOURCE == "schedule" && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
  allow_failure: true
  before_script:
    - export VERSION=$(head -n 1 dist/VERSION.md)
  script:
    - cd dist
    - >
      jf rt u --fail-no-op --url=$ARTIFACTORY_BASE_URL --access-token $ARTIFACTORY_SVC_ACCESS_TOKEN
      --target-props 'component_name=warp-lang;release_approver=ershi;release_status=ready'
      '*.whl' sw-warp-pypi-local/warp/$CI_COMMIT_REF_NAME/$VERSION/
    # Set build-specific properties on individual artifacts
    - >
      jf rt sp --fail-no-op --url=$ARTIFACTORY_BASE_URL --access-token $ARTIFACTORY_SVC_ACCESS_TOKEN
      sw-warp-pypi-local/warp/$CI_COMMIT_REF_NAME/$VERSION/*-win_amd64.whl
      'arch=x86_64;os=windows'
    - >
      jf rt sp --fail-no-op --url=$ARTIFACTORY_BASE_URL --access-token $ARTIFACTORY_SVC_ACCESS_TOKEN
      sw-warp-pypi-local/warp/$CI_COMMIT_REF_NAME/$VERSION/*-manylinux2014_aarch64.whl
      'arch=aarch64;os=linux'
    - >
      jf rt sp --fail-no-op --url=$ARTIFACTORY_BASE_URL --access-token $ARTIFACTORY_SVC_ACCESS_TOKEN
      sw-warp-pypi-local/warp/$CI_COMMIT_REF_NAME/$VERSION/*-manylinux2014_x86_64.whl
      'arch=x86_64;os=linux'
    # Set additional common properties on all artifacts
    - >
      jf rt sp --fail-no-op --url=$ARTIFACTORY_BASE_URL --access-token $ARTIFACTORY_SVC_ACCESS_TOKEN
      sw-warp-pypi-local/warp/$CI_COMMIT_REF_NAME/$VERSION/*.whl
      version=$VERSION
    - >
      jf rt sp --fail-no-op --url=$ARTIFACTORY_BASE_URL --access-token $ARTIFACTORY_SVC_ACCESS_TOKEN
      sw-warp-pypi-local/warp/$CI_COMMIT_REF_NAME/$VERSION/*.whl
      branch=$CI_COMMIT_REF_NAME

publish tag wheels to artifactory:
  stage: deploy
  image: releases-docker.jfrog.io/jfrog/jfrog-cli-v2-jf
  needs: ["create pypi wheels"]
  extends:
    - .runner-utility-linux-x86_64
  rules:
    - if: $CI_COMMIT_TAG
  allow_failure: true
  before_script:
    - export VERSION=$(head -n 1 VERSION.md)
  script:
    - cd dist
    - >
      jf rt u --fail-no-op --url=$ARTIFACTORY_BASE_URL --access-token $ARTIFACTORY_SVC_ACCESS_TOKEN
      --target-props 'component_name=warp-lang;release_approver=ershi;release_status=ready'
      '*.whl' sw-warp-pypi-local/warp/$CI_COMMIT_REF_NAME/$VERSION/
    # Set build-specific properties on individual artifacts
    - >
      jf rt sp --fail-no-op --url=$ARTIFACTORY_BASE_URL --access-token $ARTIFACTORY_SVC_ACCESS_TOKEN
      sw-warp-pypi-local/warp/$CI_COMMIT_REF_NAME/$VERSION/*-win_amd64.whl
      'arch=x86_64;os=windows'
    - >
      jf rt sp --fail-no-op --url=$ARTIFACTORY_BASE_URL --access-token $ARTIFACTORY_SVC_ACCESS_TOKEN
      sw-warp-pypi-local/warp/$CI_COMMIT_REF_NAME/$VERSION/*-manylinux2014_aarch64.whl
      'arch=aarch64;os=linux'
    - >
      jf rt sp --fail-no-op --url=$ARTIFACTORY_BASE_URL --access-token $ARTIFACTORY_SVC_ACCESS_TOKEN
      sw-warp-pypi-local/warp/$CI_COMMIT_REF_NAME/$VERSION/*-manylinux2014_x86_64.whl
      'arch=x86_64;os=linux'
    - >
      jf rt sp --fail-no-op --url=$ARTIFACTORY_BASE_URL --access-token $ARTIFACTORY_SVC_ACCESS_TOKEN
      sw-warp-pypi-local/warp/$CI_COMMIT_REF_NAME/$VERSION/*-macosx_*_universal2.whl
      'arch=universal;os=macosx'
    # Set additional common properties on all artifacts
    - >
      jf rt sp --fail-no-op --url=$ARTIFACTORY_BASE_URL --access-token $ARTIFACTORY_SVC_ACCESS_TOKEN
      sw-warp-pypi-local/warp/$CI_COMMIT_REF_NAME/$VERSION/*.whl
      version=$VERSION
    - >
      jf rt sp --fail-no-op --url=$ARTIFACTORY_BASE_URL --access-token $ARTIFACTORY_SVC_ACCESS_TOKEN
      sw-warp-pypi-local/warp/$CI_COMMIT_REF_NAME/$VERSION/*.whl
      branch=$CI_COMMIT_REF_NAME

.build-docs-common:
  stage: deploy
  image: python:3.11-slim
  needs: [linux-x86_64 build]
  extends:
    - .runner-utility-linux-x86_64
  artifacts:
    paths:
      - public
  before_script:
    - echo -e "\\e[0Ksection_start:`date +%s`:install_dependencies[collapsed=true]\\r\\e[0KSet up docs environment"
    - df -h
    - apt-get update && apt-get install make --no-install-recommends -y
    # Move compiled binaries out of platform-specific directory
    - mv warp/bin/linux-x86_64/warp.so warp/bin/
    - mv warp/bin/linux-x86_64/warp-clang.so warp/bin/
    - python -m pip install --upgrade pip
    - python -m pip install -r docs/requirements.txt
    - echo -e "\\e[0Ksection_end:`date +%s`:install_dependencies\\r\\e[0K"
  script:
    - python build_docs.py --quick
    - mv docs/_build/html/ ./public/

# Merge requests: Build documentation and save as an artifact
# A link to the generated documentation is added to the merge request.
merge request docs:
  extends:
    - .build-docs-common
  artifacts:
    paths:
      - warp/stubs.py
      - docs/modules/functions.rst
      - public
  rules:
    - if: $CI_PIPELINE_SOURCE == 'merge_request_event'
  timeout: 10m
  environment:
    name: review/$CI_MERGE_REQUEST_IID
    url: https://$CI_PROJECT_ROOT_NAMESPACE.$CI_PAGES_DOMAIN/-/$CI_PROJECT_NAME/-/jobs/$CI_JOB_ID/artifacts/public/index.html
  after_script:
    - echo "View the website at https://$CI_PROJECT_ROOT_NAMESPACE.$CI_PAGES_DOMAIN/-/$CI_PROJECT_NAME/-/jobs/$CI_JOB_ID/artifacts/public/index.html"


# Merge requests: Ensure that exports.h, stubs.py, functions.rst have been
# manually added to the MR if they are changed
check generated files:
  needs:
    - job: linux-x86_64 build
    - job: merge request docs
      optional: true
  stage: deploy
  artifacts:
    when: on_failure
    expose_as: 'Generated source files'
    paths:
      - warp/native/exports.h
      - warp/stubs.py
      - docs/modules/functions.rst
  rules:
    - if: $CI_PIPELINE_SOURCE == 'merge_request_event'
  timeout: 10m
  extends:
    - .runner-utility-linux-x86_64
  script:
    - >
      git diff --exit-code warp/stubs.py docs/modules/functions.rst ||
      (echo "Please run build_docs.py (or download from $CI_JOB_URL/artifacts/browse) and add modified files to your merge request." && false)
    - >
      git diff --exit-code warp/native/exports.h ||
      (echo "Please run build_lib.py  (or download from $CI_JOB_URL/artifacts/browse) and add warp/native/exports.h to your merge request." && false)

# Build documentation and publish on gitlab-master
# This only runs in the default branch pipeline. The "pages" name is special for GitLab.
pages:
  extends:
    - .build-docs-common
  rules:
    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
  timeout: 10m
  environment:
    name: GitLab Pages
    deployment_tier: staging
    url: $CI_PAGES_URL