Skip to content

Commit 219a04e

Browse files
pieternfacebook-github-bot
pietern
authored and
facebook-github-bot
committed
Use CircleCI commands for brew update/install (#26159)
Summary: Pull Request resolved: #26159 The snippets for working with Homebrew were duplicated across binary builds, macOS builds, and iOS builds. In #25336, the CircleCI configuration version was updated to version 2.1, which supports parameterized commands. This means we no longer have to use YAML tricks to duplicate stanzas and instead can natively define a series of reusable steps. Motivation for doing this is that the macOS binary builds were still using the slow `brew update` instead of `git fetch` (see #25988). [test macos] [test wheel] Test Plan: Imported from OSS Differential Revision: D17366538 Pulled By: pietern fbshipit-source-id: 194c0f37c1dc999705f3ba97fdabf4ff18728d93
1 parent 0963e17 commit 219a04e

File tree

8 files changed

+132
-181
lines changed

8 files changed

+132
-181
lines changed

‎.circleci/config.yml‎

Lines changed: 65 additions & 90 deletions
Original file line numberDiff line numberDiff line change
@@ -40,61 +40,64 @@ setup_ci_environment: &setup_ci_environment
4040
name: Set Up CI Environment After attach_workspace
4141
no_output_timeout: "1h"
4242
command: ~/workspace/.circleci/scripts/setup_ci_environment.sh
43+
commands:
44+
brew_update:
45+
description: "Update Homebrew and install base formulae"
46+
steps:
47+
- run:
48+
name: Update Homebrew
49+
no_output_timeout: "10m"
50+
command: |
51+
set -ex
4352
44-
# Installs expect and moreutils so that we can call `unbuffer` and `ts`.
45-
# Also installs OpenMP
46-
# !!!!NOTE!!!! this is copied into a binary_macos_brew_update job which is the
47-
# same but does not install libomp. If you are changing this, consider if you
48-
# need to change that step as well.
49-
macos_brew_update: &macos_brew_update
50-
name: Brew update and install moreutils, expect and libomp
51-
no_output_timeout: "1h"
52-
command: |
53-
set -ex
54-
55-
# Update repositories manually
56-
for path in $(find /usr/local/Homebrew -type d -name .git)
57-
do
58-
cd $path/..
59-
git fetch --depth=1 origin
60-
git reset --hard origin/master
61-
done
62-
63-
export HOMEBREW_NO_AUTO_UPDATE=1
64-
65-
# moreutils installs a `parallel` executable by default, which conflicts
66-
# with the executable from the GNU `parallel`, so we must unlink GNU
67-
# `parallel` first, and relink it afterwards
68-
brew unlink parallel
69-
brew install moreutils
70-
brew link parallel --overwrite
71-
brew install expect
72-
brew install libomp
73-
74-
ios_brew_update: &ios_brew_update
75-
name: Brew update and install iOS toolchains
76-
no_output_timeout: "1h"
77-
command: |
78-
set -ex
79-
80-
# Update repositories manually
81-
for path in $(find /usr/local/Homebrew -type d -name .git)
82-
do
83-
cd $path/..
84-
git fetch --depth=1 origin
85-
git reset --hard origin/master
86-
done
87-
88-
export HOMEBREW_NO_AUTO_UPDATE=1
89-
90-
# moreutils installs a `parallel` executable by default, which conflicts
91-
# with the executable from the GNU `parallel`, so we must unlink GNU
92-
# `parallel` first, and relink it afterwards
93-
brew unlink parallel
94-
brew install moreutils
95-
brew link parallel --overwrite
96-
brew install expect
97-
brew install libtool
53+
# Update repositories manually.
54+
# Running `brew update` produces a comparison between the
55+
# current checkout and the updated checkout, which takes a
56+
# very long time because the existing checkout is 2y old.
57+
for path in $(find /usr/local/Homebrew -type d -name .git)
58+
do
59+
cd $path/..
60+
git fetch --depth=1 origin
61+
git reset --hard origin/master
62+
done
63+
64+
export HOMEBREW_NO_AUTO_UPDATE=1
65+
66+
# Install expect and moreutils so that we can call `unbuffer` and `ts`.
67+
# moreutils installs a `parallel` executable by default, which conflicts
68+
# with the executable from the GNU `parallel`, so we must unlink GNU
69+
# `parallel` first, and relink it afterwards.
70+
brew unlink parallel
71+
brew install moreutils
72+
brew link parallel --overwrite
73+
brew install expect
74+
75+
brew_install:
76+
description: "Install Homebrew formulae"
77+
parameters:
78+
formulae:
79+
type: string
80+
default: ""
81+
steps:
82+
- run:
83+
name: Install << parameters.formulae >>
84+
no_output_timeout: "10m"
85+
command: |
86+
set -ex
87+
export HOMEBREW_NO_AUTO_UPDATE=1
88+
brew install << parameters.formulae >>
89+
90+
run_brew_for_macos_build:
91+
steps:
92+
- brew_update
93+
- brew_install:
94+
formulae: libomp
95+
96+
run_brew_for_ios_build:
97+
steps:
98+
- brew_update
99+
- brew_install:
100+
formulae: libtool
98101

99102
##############################################################################
100103
# Binary build (nightlies nightly build) defaults
@@ -145,26 +148,6 @@ binary_run_in_docker: &binary_run_in_docker
145148
# This step only runs on circleci linux machine executors that themselves
146149
# need to start docker images
147150
command: ~/workspace/.circleci/scripts/binary_run_in_docker.sh
148-
149-
# This is copied almost verbatim from the macos_brew_update job
150-
# In version 2.1 and above we could make this a command and pass a parameter to
151-
# it, but in this version there is no way to pass a parameter to a step
152-
binary_macos_brew_update: &binary_macos_brew_update
153-
name: Brew update and install moreutils and expect
154-
no_output_timeout: "1h"
155-
command: |
156-
set -eux -o pipefail
157-
# See https://discourse.brew.sh/t/fetching-homebrew-repos-is-slow/5374/3
158-
brew untap caskroom/homebrew-cask
159-
# moreutils installs a `parallel` executable by default, which conflicts
160-
# with the executable from the GNU `parallel`, so we must unlink GNU
161-
# `parallel` first, and relink it afterwards
162-
brew update
163-
brew unlink parallel
164-
brew install moreutils
165-
brew link parallel --overwrite
166-
brew install expect
167-
168151
##############################################################################
169152
# Build parameters
170153
##############################################################################
@@ -541,8 +524,7 @@ jobs:
541524
- run:
542525
<<: *should_run_job
543526
- checkout
544-
- run:
545-
<<: *macos_brew_update
527+
- run_brew_for_macos_build
546528
- run:
547529
name: Build
548530
no_output_timeout: "1h"
@@ -772,8 +754,7 @@ jobs:
772754
<<: *binary_checkout
773755
- run:
774756
<<: *binary_populate_env
775-
- run:
776-
<<: *binary_macos_brew_update
757+
- brew_update
777758
- run:
778759
<<: *binary_install_miniconda
779760
- run:
@@ -802,8 +783,7 @@ jobs:
802783
<<: *binary_checkout
803784
- run:
804785
<<: *binary_populate_env
805-
- run:
806-
<<: *binary_macos_brew_update
786+
- brew_update
807787
- run:
808788
<<: *binary_install_miniconda
809789

@@ -843,8 +823,7 @@ jobs:
843823
<<: *binary_checkout
844824
- run:
845825
<<: *binary_populate_env
846-
- run:
847-
<<: *binary_macos_brew_update
826+
- brew_update
848827
- run:
849828
<<: *binary_install_miniconda
850829
- attach_workspace: # TODO - we can `cp` from ~/workspace
@@ -1044,8 +1023,7 @@ jobs:
10441023
- run:
10451024
<<: *should_run_job
10461025
- checkout
1047-
- run:
1048-
<<: *macos_brew_update
1026+
- run_brew_for_macos_build
10491027
- run:
10501028
name: Build
10511029
no_output_timeout: "1h"
@@ -1088,8 +1066,7 @@ jobs:
10881066
at: ~/workspace
10891067
- run:
10901068
<<: *should_run_job
1091-
- run:
1092-
<<: *macos_brew_update
1069+
- run_brew_for_macos_build
10931070
- run:
10941071
name: Test
10951072
no_output_timeout: "1h"
@@ -1115,8 +1092,7 @@ jobs:
11151092
- run:
11161093
<<: *should_run_job
11171094
- checkout
1118-
- run:
1119-
<<: *macos_brew_update
1095+
- run_brew_for_macos_build
11201096
- run:
11211097
name: Build
11221098
no_output_timeout: "1h"
@@ -1308,8 +1284,7 @@ jobs:
13081284
- run:
13091285
<<: *should_run_job
13101286
- checkout
1311-
- run:
1312-
<<: *ios_brew_update
1287+
- run_brew_for_ios_build
13131288
- run:
13141289
name: Build
13151290
no_output_timeout: "1h"

‎.circleci/generate_config_yml.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -74,6 +74,7 @@ def write(self, output_filehandle):
7474
# Order of this list matters to the generated config.yml.
7575
YAML_SOURCES = [
7676
File("header-section.yml"),
77+
File("commands.yml"),
7778
File("nightly-binary-build-defaults.yml"),
7879
Header("Build parameters"),
7980
File("pytorch-build-params.yml"),

‎.circleci/verbatim-sources/binary-job-specs.yml‎

Lines changed: 3 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -155,8 +155,7 @@
155155
<<: *binary_checkout
156156
- run:
157157
<<: *binary_populate_env
158-
- run:
159-
<<: *binary_macos_brew_update
158+
- brew_update
160159
- run:
161160
<<: *binary_install_miniconda
162161
- run:
@@ -185,8 +184,7 @@
185184
<<: *binary_checkout
186185
- run:
187186
<<: *binary_populate_env
188-
- run:
189-
<<: *binary_macos_brew_update
187+
- brew_update
190188
- run:
191189
<<: *binary_install_miniconda
192190

@@ -226,8 +224,7 @@
226224
<<: *binary_checkout
227225
- run:
228226
<<: *binary_populate_env
229-
- run:
230-
<<: *binary_macos_brew_update
227+
- brew_update
231228
- run:
232229
<<: *binary_install_miniconda
233230
- attach_workspace: # TODO - we can `cp` from ~/workspace

‎.circleci/verbatim-sources/caffe2-job-specs.yml‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -142,8 +142,7 @@
142142
- run:
143143
<<: *should_run_job
144144
- checkout
145-
- run:
146-
<<: *macos_brew_update
145+
- run_brew_for_macos_build
147146
- run:
148147
name: Build
149148
no_output_timeout: "1h"

‎.circleci/verbatim-sources/commands.yml‎

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,58 @@
1+
commands:
2+
brew_update:
3+
description: "Update Homebrew and install base formulae"
4+
steps:
5+
- run:
6+
name: Update Homebrew
7+
no_output_timeout: "10m"
8+
command: |
9+
set -ex
10+
11+
# Update repositories manually.
12+
# Running `brew update` produces a comparison between the
13+
# current checkout and the updated checkout, which takes a
14+
# very long time because the existing checkout is 2y old.
15+
for path in $(find /usr/local/Homebrew -type d -name .git)
16+
do
17+
cd $path/..
18+
git fetch --depth=1 origin
19+
git reset --hard origin/master
20+
done
21+
22+
export HOMEBREW_NO_AUTO_UPDATE=1
23+
24+
# Install expect and moreutils so that we can call `unbuffer` and `ts`.
25+
# moreutils installs a `parallel` executable by default, which conflicts
26+
# with the executable from the GNU `parallel`, so we must unlink GNU
27+
# `parallel` first, and relink it afterwards.
28+
brew unlink parallel
29+
brew install moreutils
30+
brew link parallel --overwrite
31+
brew install expect
32+
33+
brew_install:
34+
description: "Install Homebrew formulae"
35+
parameters:
36+
formulae:
37+
type: string
38+
default: ""
39+
steps:
40+
- run:
41+
name: Install << parameters.formulae >>
42+
no_output_timeout: "10m"
43+
command: |
44+
set -ex
45+
export HOMEBREW_NO_AUTO_UPDATE=1
46+
brew install << parameters.formulae >>
47+
48+
run_brew_for_macos_build:
49+
steps:
50+
- brew_update
51+
- brew_install:
52+
formulae: libomp
53+
54+
run_brew_for_ios_build:
55+
steps:
56+
- brew_update
57+
- brew_install:
58+
formulae: libtool

‎.circleci/verbatim-sources/header-section.yml‎

Lines changed: 0 additions & 55 deletions
Original file line numberDiff line numberDiff line change
@@ -40,58 +40,3 @@ setup_ci_environment: &setup_ci_environment
4040
name: Set Up CI Environment After attach_workspace
4141
no_output_timeout: "1h"
4242
command: ~/workspace/.circleci/scripts/setup_ci_environment.sh
43-
44-
# Installs expect and moreutils so that we can call `unbuffer` and `ts`.
45-
# Also installs OpenMP
46-
# !!!!NOTE!!!! this is copied into a binary_macos_brew_update job which is the
47-
# same but does not install libomp. If you are changing this, consider if you
48-
# need to change that step as well.
49-
macos_brew_update: &macos_brew_update
50-
name: Brew update and install moreutils, expect and libomp
51-
no_output_timeout: "1h"
52-
command: |
53-
set -ex
54-
55-
# Update repositories manually
56-
for path in $(find /usr/local/Homebrew -type d -name .git)
57-
do
58-
cd $path/..
59-
git fetch --depth=1 origin
60-
git reset --hard origin/master
61-
done
62-
63-
export HOMEBREW_NO_AUTO_UPDATE=1
64-
65-
# moreutils installs a `parallel` executable by default, which conflicts
66-
# with the executable from the GNU `parallel`, so we must unlink GNU
67-
# `parallel` first, and relink it afterwards
68-
brew unlink parallel
69-
brew install moreutils
70-
brew link parallel --overwrite
71-
brew install expect
72-
brew install libomp
73-
74-
ios_brew_update: &ios_brew_update
75-
name: Brew update and install iOS toolchains
76-
no_output_timeout: "1h"
77-
command: |
78-
set -ex
79-
80-
# Update repositories manually
81-
for path in $(find /usr/local/Homebrew -type d -name .git)
82-
do
83-
cd $path/..
84-
git fetch --depth=1 origin
85-
git reset --hard origin/master
86-
done
87-
88-
export HOMEBREW_NO_AUTO_UPDATE=1
89-
90-
# moreutils installs a `parallel` executable by default, which conflicts
91-
# with the executable from the GNU `parallel`, so we must unlink GNU
92-
# `parallel` first, and relink it afterwards
93-
brew unlink parallel
94-
brew install moreutils
95-
brew link parallel --overwrite
96-
brew install expect
97-
brew install libtool

0 commit comments

Comments
 (0)