Azure pipeline

.azurePipeline/wholeBuild.yml

@@ -0,0 +1,87 @@
+parameters:
+  pythonVersions: []
+  operatingSystems: []
+  architectures: []
+
+jobs:
+- job: 'test_and_publish'
+  strategy:
+    matrix:
+      ${{ each py in parameters.pythonVersions }}:
+        ${{ each os in parameters.operatingSystems }}:
+          ${{ each architecture in parameters.architectures }}:
+            ${{ format('{0}_Python_{1}_{2}', os, py, architecture) }}:
+              pythonVersion: ${{ py }}
+              operatingSystem: ${{ os }}
+              architecture: ${{ architecture }}
+
+  pool:
+    vmImage: $(operatingSystem)
+
+  displayName: 'Test and publish for '
+
+  steps:
+# I would not have found on my own such a step. Thanks https://dvlup.com/2019/01/03/using-powershell-to-installing-msi-in-a-devops-build/
+# The solution I found was not working: it hanged forevever, using "msiexec /i"
+# These steps are required because of at least the bitarray dependency (but maybe more). See issue #153 tracking if bitarray is still used.
+  - powershell: Invoke-WebRequest -Uri https://download.microsoft.com/download/7/9/6/796EF2E4-801B-4FC4-AB28-B59FBF6D907B/VCForPython27.msi -OutFile VCForPython27.msi
+    displayName: 'Download Microsoft Visual C++ 9.0 from https://download.microsoft.com/download/7/9/6/796EF2E4-801B-4FC4-AB28-B59FBF6D907B/VCForPython27.msi'
+    condition: eq(variables['pythonVersion'], 2.7)
+  - powershell: Start-Process VCForPython27.msi -ArgumentList "/q" -Wait 
+    displayName: 'Install Microsoft Visual C++ 9.0.'
+    condition: and(eq(variables['operatingSystem'], 'vs2017-win2016'), eq(variables['pythonVersion'], 2.7))
+
+  - task: UsePythonVersion@0
+    inputs:
+      versionSpec: '$(pythonVersion)'
+      architecture: '$(architecture)'
+
+  - script: |
+      python -m pip install -U pip
+      python -m pip install -U wheel setuptools codecov
+      python -m pip install -U -r requirements.txt
+    displayName: 'Install requirements'
+
+  - script: python setup.py sdist bdist_wheel
+    displayName: 'Package'
+
+  - script: python -m pytest --cov=clkhash --junitxml=testResults.xml --cov-report=xml:coverageReport.xml --cov-report=html:htmlcov
+    displayName: 'Test with pytest'
+    env:
+      INCLUDE_CLI: 1
+
+  - task: PublishTestResults@2
+    displayName: 'Publish test results in Azure'
+    condition: succeeded()
+    inputs:
+      testResultsFormat: 'JUnit'
+      testResultsFiles: 'testResults.xml'
+      testRunTitle: 'Test results on a vm $(operatingSystem) ($(architecture)) for Python $(pythonVersion)'
+      failTaskOnFailedTests: true
+
+  - task: PublishCodeCoverageResults@1
+    displayName: 'Publish code coverage in Azure'
+    # If the previous stage fail, we still want to run this one as the previous stage may fail because of a failing test.
+    condition: succeededOrFailed()
+    inputs:
+      codeCoverageTool: Cobertura
+      summaryFileLocation: 'coverageReport.xml'
+# Seems to create warning as this step is already creating its own html pages.
+#      reportDirectory: 'htmlcov'
+      failIfCoverageEmpty: true
+
+  - script: python -m codecov --token $(CODECOV_TOKEN) --file coverageReport.xml
+    displayName: 'Send coverage to codecov'
+
+  - script: |
+      pyinstaller cli.spec
+      .\dist\clkutil.exe --version
+    displayName: 'Build clkutil.exe'
+    condition: and(eq(variables['operatingSystem'], 'vs2017-win2016'), eq(variables['pythonVersion'], '3.7'), eq(variables['architecture'], 'x86'))
+
+  - task: PublishPipelineArtifact@0
+    displayName: 'Publish artifacts'
+    condition: and(eq(variables['pythonVersion'], '3.7'), eq(variables['architecture'], 'x86'))
+    inputs:
+      artifactName: Artifacts
+      targetPath: 'dist/'

.gitignore

@@ -22,6 +22,7 @@ lib64/
 parts/
 sdist/
 var/
+venv/
 *.egg-info/
 .installed.cfg
 

azurePipeline.yml

@@ -0,0 +1,33 @@
+trigger:
+  branches: 
+    include:
+    - master
+    exclude:
+    - '*'
+  tags:
+    include:
+    - 'v*'
+pr:
+  branches:
+    include:
+    - '*'
+
+jobs:
+- template: .azurePipeline/wholeBuild.yml  # Template reference
+  parameters:
+    pythonVersions: ['3.7', '3.6', '3.5', '2.7']
+#    operatingSystems: ['ubuntu-16.04', 'macos-10.13', 'vs2017-win2016']
+    operatingSystems: ['vs2017-win2016']
+    architectures: ['x64', 'x86']
+
+- job:
+  steps:
+# In this step, if this build is triggered by a tag starting with a 'v', it will add a tag 'Automated' to the current build.
+# This tag if put on a commit from master should trigger the release pipeline.
+  - script: echo "##vso[build.addbuildtag] Automated"
+    condition: startsWith(variables['Build.SourceBranch'], 'refs/tags/v')
+#  - script: |
+#      pip install twine
+#      twine upload -u $(PYPI_LOGIN) -p $(PYPI_PASSWORD) --skip-existing dist/*.whl
+#    displayName: 'Send artifact to Pypi'
+#    condition: only if commit is tagged... will be fun to check

docs/devops.rst

@@ -0,0 +1,60 @@
+Devops
+===========
+
+
+.. toctree::
+   :maxdepth: 2
+
+   clkhash
+
+Azure Pipeline
+--------------
+
+``clkhash`` is automatically built and tested using Azure Pipeline
+ for Windows environment, in the project `Anonlink <https://dev.azure.com/data61/Anonlink>`
+
+Two pipelines are available:
+  - `Build pipeline <https://dev.azure.com/data61/Anonlink/_build?definitionId=2>`,
+  - `Release pipeline <https://dev.azure.com/data61/Anonlink/_release?definitionId=1>`.
+
+The build pipeline is described by the script `azurePipeline.yml`
+which is using resources from the folder `.azurePipeline`.
+Mainly, a number of builds and tests are started for different
+version of python and system architecture. 
+Only the packages created with ``Python 3.7`` and the ``x86``
+architecture are then published (in Azure).
+
+The build pipeline is triggered for every pushes on the master branch,
+for every tagged commit with the tag starting with a `v`, and for
+every pushes part of a pull request. We are not building on every push and
+on pull requests not to build twice the same code. For every tagged commit,
+the build pipeline will also add the Azure tag `Automated` which may trigger
+automatically the release pipeline.
+
+The build pipeline does:
+  - install the requirements,
+  - package ``clkhash``,
+  - run `pytest`,
+  - publish the test results,
+  - publish the code coverage (on Azure and codecov),
+  - publish the artifacts from the build using ``Python 3.7``
+  with a ``x86`` architecture (i.e. a whl, a tar.gz and an exe).
+
+The build pipeline requires one environment variable provided by Azure environment:
+ - `CODECOV_TOKEN` which is used to publish the coverage to codecov.
+
+
+The release pipeline can either be triggered manually, or automatically from
+a successful build on master where the build is tagged `Automated`
+(i.e. if the commit is tagged, cf previous paragraph). 
+
+The release pipeline consists of two steps: 
+  - asking for a manual confirmation that the artifacts from the
+  triggering build should be released,
+  - uses ``twine`` to publish the artifacts.
+
+The release pipeline requires two environment variables provided by Azure environment:
+ - `PYPI_LOGIN`: login to push an artifact to ``clkhash`` ``Pypi`` repository,
+ - `PYPI_PASSWORD`: password to push an artifact to ``clkhash`` ``Pypi`` repository
+ for the user `PYPI_LOGIN`.
+

docs/index.rst

@@ -33,6 +33,7 @@ Table of Contents
    cli
    schema
    development
+   devops
    rest_client
    references