Skip to content

bpo-34582: Update syntax of Azure Pipelines builds #9521

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
zooba merged 1 commit into from
Sep 24, 2018
Merged

bpo-34582: Update syntax of Azure Pipelines builds #9521

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
136 changes: 136 additions & 0 deletions .azure-pipelines/ci.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,136 @@
variables:
manylinux: false
coverage: false

jobs:
- job: Prebuild
displayName: Pre-build checks

pool:
vmImage: ubuntu-16.04

steps:
- template: ./prebuild-checks.yml


- job: Docs_PR
displayName: Docs PR
dependsOn: Prebuild
condition: and(succeeded(), eq(dependencies.Prebuild.outputs['docs.run'], 'true'))

pool:
vmImage: ubuntu-16.04

steps:
- template: ./docs-steps.yml
parameters:
upload: true


- job: macOS_CI_Tests
displayName: macOS CI Tests
dependsOn: Prebuild
condition: and(succeeded(), eq(dependencies.Prebuild.outputs['tests.run'], 'true'))

variables:
testRunTitle: '$(build.sourceBranchName)-macos'
testRunPlatform: macos

pool:
vmImage: xcode9-macos10.13

steps:
- template: ./macos-steps.yml


- job: Ubuntu_CI_Tests
displayName: Ubuntu CI Tests
dependsOn: Prebuild
condition: and(succeeded(), eq(dependencies.Prebuild.outputs['tests.run'], 'true'))

pool:
vmImage: ubuntu-16.04

variables:
testRunTitle: '$(build.sourceBranchName)-linux'
testRunPlatform: linux
openssl_version: 1.1.0g

steps:
- template: ./posix-steps.yml


- job: ManyLinux1_CI_Tests
displayName: ManyLinux1 CI Tests
dependsOn: Prebuild
condition: |
and(
and(
succeeded(),
eq(variables['manylinux'], 'true')
),
eq(dependencies.Prebuild.outputs['tests.run'], 'true')
)

pool:
vmImage: ubuntu-16.04

variables:
testRunTitle: '$(build.sourceBranchName)-manylinux1'
testRunPlatform: manylinux1
imageName: 'dockcross/manylinux-x64'

steps:
- template: ./docker-steps.yml


- job: Ubuntu_Coverage_CI_Tests
displayName: Ubuntu CI Tests (coverage)
dependsOn: Prebuild
condition: |
and(
and(
succeeded(),
eq(variables['coverage'], 'true')
),
eq(dependencies.Prebuild.outputs['tests.run'], 'true')
)

pool:
vmImage: ubuntu-16.04

variables:
testRunTitle: '$(Build.SourceBranchName)-linux-coverage'
testRunPlatform: linux-coverage
openssl_version: 1.1.0g

steps:
- template: ./posix-steps.yml
parameters:
coverage: true


- job: Windows_CI_Tests
displayName: Windows CI Tests
dependsOn: Prebuild
condition: and(succeeded(), eq(dependencies.Prebuild.outputs['tests.run'], 'true'))

pool:
vmImage: vs2017-win2017

strategy:
matrix:
win32:
arch: win32
buildOpt:
testRunTitle: '$(Build.SourceBranchName)-win32'
testRunPlatform: win32
win64:
arch: amd64
buildOpt: '-p x64'
testRunTitle: '$(Build.SourceBranchName)-win64'
testRunPlatform: win64
maxParallel: 2

steps:
- template: ./windows-steps.yml
76 changes: 76 additions & 0 deletions .azure-pipelines/docker-steps.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,76 @@
steps:
- checkout: self
clean: true
fetchDepth: 5

- ${{ if ne(parameters.targetBranch, '') }}:
- script: |
git fetch -q origin ${{ parameters.targetbranch }}
if ! git diff --name-only HEAD $(git merge-base HEAD FETCH_HEAD) | grep -qvE '(\.rst$|^Doc|^Misc)'
then
echo "Only docs were updated, stopping build process."
echo "##vso[task.setvariable variable=DocOnly]true"
exit
fi
displayName: Detect doc-only changes

- task: docker@0
displayName: 'Configure CPython (debug)'
inputs:
action: 'Run an image'
imageName: $(imageName)
volumes: |
$(build.sourcesDirectory):/src
$(build.binariesDirectory):/build
workDir: '/src'
containerCommand: './configure --with-pydebug'
detached: false
condition: and(succeeded(), ne(variables['DocOnly'], 'true'))

- task: docker@0
displayName: 'Build CPython'
inputs:
action: 'Run an image'
imageName: $(imageName)
volumes: |
$(build.sourcesDirectory):/src
$(build.binariesDirectory):/build
workDir: '/src'
containerCommand: 'make -s -j4'
detached: false
condition: and(succeeded(), ne(variables['DocOnly'], 'true'))

- task: docker@0
displayName: 'Display build info'
inputs:
action: 'Run an image'
imageName: $(imageName)
volumes: |
$(build.sourcesDirectory):/src
$(build.binariesDirectory):/build
workDir: '/src'
containerCommand: 'make pythoninfo'
detached: false
condition: and(succeeded(), ne(variables['DocOnly'], 'true'))

- task: docker@0
displayName: 'Tests'
inputs:
action: 'Run an image'
imageName: $(imageName)
volumes: |
$(build.sourcesDirectory):/src
$(build.binariesDirectory):/build
workDir: '/src'
containerCommand: 'make buildbottest TESTOPTS="-j4 -uall,-cpu --junit-xml=/build/test-results.xml"'
detached: false
condition: and(succeeded(), ne(variables['DocOnly'], 'true'))

- task: PublishTestResults@2
displayName: 'Publish Test Results'
inputs:
testResultsFiles: '$(build.binariesDirectory)/test-results.xml'
mergeTestResults: true
testRunTitle: $(testRunTitle)
platform: $(testRunPlatform)
condition: and(succeededOrFailed(), ne(variables['DocOnly'], 'true'))
46 changes: 46 additions & 0 deletions .azure-pipelines/docs-steps.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,46 @@
parameters:
latex: false
upload: false

steps:
- checkout: self
clean: true
fetchDepth: 5

- task: UsePythonVersion@0
displayName: 'Use Python 3.6 or later'
inputs:
versionSpec: '>=3.6'

- script: python -m pip install sphinx~=1.6.1 blurb python-docs-theme
displayName: 'Install build dependencies'

- ${{ if ne(parameters.latex, 'true') }}:
- script: make check suspicious html PYTHON=python
workingDirectory: '$(build.sourcesDirectory)/Doc'
displayName: 'Build documentation'

- ${{ if eq(parameters.latex, 'true') }}:
- script: sudo apt-get update && sudo apt-get install -qy --force-yes texlive-full
displayName: 'Install LaTeX'

- script: make dist PYTHON=python SPHINXBUILD='python -m sphinx' BLURB='python -m blurb'
workingDirectory: '$(build.sourcesDirectory)/Doc'
displayName: 'Build documentation'

- ${{ if eq(parameters.upload, 'true') }}:
- task: PublishBuildArtifacts@1
displayName: 'Publish docs'

inputs:
PathToPublish: '$(build.sourcesDirectory)/Doc/build'
ArtifactName: docs
publishLocation: Container

- ${{ if eq(parameters.latex, 'true') }}:
- task: PublishBuildArtifacts@1
displayName: 'Publish dist'
inputs:
PathToPublish: '$(build.sourcesDirectory)/Doc/dist'
ArtifactName: docs_dist
publishLocation: Container
27 changes: 3 additions & 24 deletions .vsts/macos-buildbot.yml → .azure-pipelines/macos-steps.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,30 +1,9 @@
# Current docs for the syntax of this file are at:
# https://github.com/Microsoft/vsts-agent/blob/master/docs/preview/yamlgettingstarted.md

name: $(BuildDefinitionName)_$(Date:yyyyMMdd)$(Rev:.rr)

queue:
name: Hosted macOS

trigger:
branches:
include:
- master
- 3.7
- 3.6
paths:
exclude:
- Doc/*
- Tools/*

#variables:

steps:
- checkout: self
clean: true
fetchDepth: 5

- script: ./configure --with-pydebug --with-openssl=/usr/local/opt/openssl --prefix=/opt/python-vsts
- script: ./configure --with-pydebug --with-openssl=/usr/local/opt/openssl --prefix=/opt/python-azdev
displayName: 'Configure CPython (debug)'

- script: make -s -j4
Expand All @@ -41,6 +20,6 @@ steps:
inputs:
testResultsFiles: '$(build.binariesDirectory)/test-results.xml'
mergeTestResults: true
testRunTitle: '$(build.sourceBranchName)-macOS'
platform: macOS
testRunTitle: $(testRunTitle)
platform: $(testRunPlatform)
condition: succeededOrFailed()
9 changes: 8 additions & 1 deletion .vsts/install_deps.sh → .azure-pipelines/posix-deps.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
sudo apt-get update || true
sudo apt-get update

sudo apt-get -yq install \
build-essential \
Expand All @@ -17,3 +17,10 @@ sudo apt-get -yq install \
libffi-dev \
uuid-dev \
xvfb

if [ ! -z "$1" ]
then
echo ##vso[task.prependpath]$PWD/multissl/openssl/$1
echo ##vso[task.setvariable variable=OPENSSL_DIR]$PWD/multissl/openssl/$1
python3 Tools/ssl/multissltests.py --steps=library --base-directory $PWD/multissl --openssl $1 --system Linux
fi
63 changes: 63 additions & 0 deletions .azure-pipelines/posix-steps.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,63 @@
parameters:
coverage: false

steps:
- checkout: self
clean: true
fetchDepth: 5

- script: ./.azure-pipelines/posix-deps.sh $(openssl_version)
displayName: 'Install dependencies'

- script: ./configure --with-pydebug
displayName: 'Configure CPython (debug)'

- script: make -s -j4
displayName: 'Build CPython'

- ${{ if eq(parameters.coverage, 'true') }}:
- script: ./python -m venv venv && ./venv/bin/python -m pip install -U coverage
displayName: 'Set up virtual environment'

- script: ./venv/bin/python -m test.pythoninfo
displayName: 'Display build info'

- script: |
xvfb-run ./venv/bin/python -m coverage run --pylib -m test \
--fail-env-changed \
-uall,-cpu \
--junit-xml=$(build.binariesDirectory)/test-results.xml" \
-x test_multiprocessing_fork \
-x test_multiprocessing_forkserver \
-x test_multiprocessing_spawn \
-x test_concurrent_futures
displayName: 'Tests with coverage'

- script: ./venv/bin/python -m coverage xml
displayName: 'Generate coverage.xml'

- script: source ./venv/bin/activate && bash <(curl -s https://codecov.io/bash)
displayName: 'Publish code coverage results'


- ${{ if ne(parameters.coverage, 'true') }}:
- script: make pythoninfo
displayName: 'Display build info'

- script: xvfb-run make buildbottest TESTOPTS="-j4 -uall,-cpu --junit-xml=$(build.binariesDirectory)/test-results.xml"
displayName: 'Tests'


- script: python Tools/scripts/patchcheck.py --travis true
displayName: 'Run patchcheck.py'
condition: and(succeeded(), eq(variables['Build.Reason'], 'PullRequest'))


- task: PublishTestResults@2
displayName: 'Publish Test Results'
inputs:
testResultsFiles: '$(build.binariesDirectory)/test-results.xml'
mergeTestResults: true
testRunTitle: $(testRunTitle)
platform: $(testRunPlatform)
condition: succeededOrFailed()
Loading