Skip to content

Commit 8e48dd1

Browse files
gorhillgorhill
committed
add script to automate signing/publishing of slef-hosted Firefox version
1 parent b4ce23f commit 8e48dd1

File tree

1 file changed

+274
-0
lines changed

1 file changed

+274
-0
lines changed

‎dist/firefox/publish-signed-beta.py‎

Lines changed: 274 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,274 @@
1+
#!/usr/bin/env python3
2+
3+
import datetime
4+
import json
5+
import jwt
6+
import os
7+
import re
8+
import requests
9+
import shutil
10+
import subprocess
11+
import sys
12+
import tempfile
13+
import time
14+
import zipfile
15+
16+
from distutils.version import StrictVersion
17+
from string import Template
18+
19+
# - Download target (raw) uMatrix.webext.xpi from GitHub
20+
# - This is referred to as "raw" package
21+
# - This will fail if not a dev build
22+
# - Modify raw package to make it self-hosted
23+
# - This is referred to as "unsigned" package
24+
# - Ask AMO to sign uMatrix.webext.xpi
25+
# - Generate JWT to be used for communication with server
26+
# - Upload unsigned package to AMO
27+
# - Wait for a valid download URL for signed package
28+
# - Download signed package as uMatrix.webext.signed.xpi
29+
# - This is referred to as "signed" package
30+
# - Upload uMatrix.webext.signed.xpi to GitHub
31+
# - Remove uMatrix.webext.xpi from GitHub
32+
# - Modify updates.json to point to new version
33+
# - Commit changes to repo
34+
35+
# Find path to project root
36+
projdir = os.path.split(os.path.abspath(__file__))[0]
37+
while not os.path.isdir(os.path.join(projdir, '.git')):
38+
projdir = os.path.normpath(os.path.join(projdir, '..'))
39+
# Check that found project root is valid
40+
version_filepath = os.path.join(projdir, 'dist', 'version')
41+
if not os.path.isfile(version_filepath):
42+
print('Version file not found.')
43+
exit(1)
44+
45+
extension_id = 'uMatrix@raymondhill.net'
46+
tmpdir = tempfile.TemporaryDirectory()
47+
raw_xpi_filename = 'uMatrix.webext.xpi'
48+
raw_xpi_filepath = os.path.join(tmpdir.name, raw_xpi_filename)
49+
unsigned_xpi_filepath = os.path.join(tmpdir.name, 'uMatrix.webext.unsigned.xpi')
50+
signed_xpi_filename = 'uMatrix.webext.signed.xpi'
51+
signed_xpi_filepath = os.path.join(tmpdir.name, signed_xpi_filename)
52+
github_owner = 'gorhill'
53+
github_repo = 'uMatrix'
54+
55+
# We need a version string to work with
56+
if len(sys.argv) >= 2 and sys.argv[1]:
57+
version = sys.argv[1]
58+
else:
59+
version = input('Github release version: ')
60+
version.strip()
61+
if not re.search('^\d+\.\d+\.\d+(b|rc)\d+$', version):
62+
print('Error: Invalid version string.')
63+
exit(1)
64+
65+
# GitHub API token
66+
# TODO: support as environment variable? (see os.environ)
67+
github_token = input("Github token: ").strip()
68+
if len(github_token) == 0:
69+
print('Error: invalid GitHub token')
70+
exit(1)
71+
github_auth = 'token ' + github_token
72+
73+
#
74+
# Get metadata from GitHub about the release
75+
#
76+
77+
# https://developer.github.com/v3/repos/releases/#get-a-single-release
78+
print('Downloading release info from GitHub...')
79+
release_info_url = 'https://api.github.com/repos/{0}/{1}/releases/tags/{2}'.format(github_owner, github_repo, version)
80+
headers = { 'Authorization': github_auth, }
81+
response = requests.get(release_info_url, headers=headers)
82+
if response.status_code != 200:
83+
print('Error: Release not found: {0}'.format(response.status_code))
84+
exit(1)
85+
release_info = response.json()
86+
87+
#
88+
# Extract URL to raw package from metadata
89+
#
90+
91+
# Find url for uMatrix.webext.xpi
92+
raw_xpi_url = ''
93+
for asset in release_info['assets']:
94+
if asset['name'] == signed_xpi_filename:
95+
print('Error: Found existing signed self-hosted package.')
96+
exit(1)
97+
if asset['name'] == raw_xpi_filename:
98+
raw_xpi_url = asset['url']
99+
if len(raw_xpi_url) == 0:
100+
print('Error: Release asset URL not found')
101+
exit(1)
102+
103+
#
104+
# Download raw package from GitHub
105+
#
106+
107+
# https://developer.github.com/v3/repos/releases/#get-a-single-release-asset
108+
print('Downloading raw xpi package from GitHub...')
109+
headers = {
110+
'Authorization': github_auth,
111+
'Accept': 'application/octet-stream',
112+
}
113+
response = requests.get(raw_xpi_url, headers=headers)
114+
# Redirections are transparently handled:
115+
# http://docs.python-requests.org/en/master/user/quickstart/#redirection-and-history
116+
if response.status_code != 200:
117+
print('Error: Downloading raw package failed -- server error {0}'.format(response.status_code))
118+
exit(1)
119+
with open(raw_xpi_filepath, 'wb') as f:
120+
f.write(response.content)
121+
print('Downloaded raw package saved as {0}'.format(raw_xpi_filepath))
122+
123+
#
124+
# Convert the package to a self-hosted one: add `update_url` to the manifest
125+
#
126+
127+
print('Converting raw xpi package into self-hosted xpi package...')
128+
with zipfile.ZipFile(raw_xpi_filepath, 'r') as zipin:
129+
with zipfile.ZipFile(unsigned_xpi_filepath, 'w') as zipout:
130+
for item in zipin.infolist():
131+
data = zipin.read(item.filename)
132+
if item.filename == 'manifest.json':
133+
manifest = json.loads(bytes.decode(data))
134+
manifest['applications']['gecko']['update_url'] = 'https://raw.githubusercontent.com/{0}/{1}/master/dist/firefox/updates.json'.format(github_owner, github_repo)
135+
data = json.dumps(manifest, indent=2, separators=(',', ': '), sort_keys=True).encode()
136+
zipout.writestr(item, data)
137+
138+
#
139+
# Ask AMO to sign the self-hosted package
140+
# - https://developer.mozilla.org/en-US/Add-ons/Distribution#Distributing_your_add-on
141+
# - https://pyjwt.readthedocs.io/en/latest/usage.html
142+
# - https://addons-server.readthedocs.io/en/latest/topics/api/auth.html
143+
# - https://addons-server.readthedocs.io/en/latest/topics/api/signing.html
144+
#
145+
146+
print('Ask AMO to sign self-hosted xpi package...')
147+
with open(unsigned_xpi_filepath, 'rb') as f:
148+
# TODO: support use of env variables for key/secret?
149+
amo_api_key = input("AMO API key: ").strip()
150+
amo_secret = input("AMO API secret: ").strip()
151+
amo_nonce = os.urandom(8).hex()
152+
jwt_payload = {
153+
'iss': amo_api_key,
154+
'jti': amo_nonce,
155+
'iat': datetime.datetime.utcnow(),
156+
'exp': datetime.datetime.utcnow() + datetime.timedelta(seconds=180),
157+
}
158+
jwt_auth = 'JWT ' + jwt.encode(jwt_payload, amo_secret).decode()
159+
headers = { 'Authorization': jwt_auth, }
160+
data = { 'channel': 'unlisted' }
161+
files = { 'upload': f, }
162+
signing_url = 'https://addons.mozilla.org/api/v3/addons/{0}/versions/{1}/'.format(extension_id, version)
163+
print('Submitting package to be signed...')
164+
response = requests.put(signing_url, headers=headers, data=data, files=files)
165+
if response.status_code != 202:
166+
print('Error: Creating new version failed -- server error {0}'.format(response.status_code))
167+
print(response.text)
168+
exit(1)
169+
print('Request for signing self-hosted xpi package succeeded.')
170+
signing_request_response = response.json();
171+
f.close()
172+
print('Waiting for AMO to process the request to sign the self-hosted xpi package...')
173+
# Wait for signed package to be ready
174+
signing_check_url = signing_request_response['url']
175+
# TODO: use real time instead
176+
countdown = 180 / 5
177+
while True:
178+
sys.stdout.write('.')
179+
sys.stdout.flush()
180+
time.sleep(5)
181+
countdown -= 1
182+
if countdown <= 0:
183+
print('Error: AMO signing timed out')
184+
exit(1)
185+
response = requests.get(signing_check_url, headers=headers)
186+
if response.status_code != 200:
187+
print('Error: AMO signing failed -- server error {0}'.format(response.status_code))
188+
exit(1)
189+
signing_check_response = response.json()
190+
if not signing_check_response['processed']:
191+
continue
192+
if not signing_check_response['valid']:
193+
print('Error: AMO validation failed')
194+
exit(1)
195+
if not signing_check_response['files'] or len(signing_check_response['files']) == 0:
196+
continue
197+
if not signing_check_response['files'][0]['signed']:
198+
print('Error: AMO signing failed')
199+
exit(1)
200+
print('\r')
201+
print('Self-hosted xpi package successfully signed.')
202+
download_url = signing_check_response['files'][0]['download_url']
203+
print('Downloading signed self-hosted xpi package from {0}...'.format(download_url))
204+
response = requests.get(download_url, headers=headers)
205+
if response.status_code != 200:
206+
print('Error: Download signed package failed -- server error {0}'.format(response.status_code))
207+
exit(1)
208+
with open(signed_xpi_filepath, 'wb') as f:
209+
f.write(response.content)
210+
f.close()
211+
print('Signed self-hosted xpi package downloaded.')
212+
break
213+
214+
#
215+
# Upload signed package to GitHub
216+
#
217+
218+
# https://developer.github.com/v3/repos/releases/#upload-a-release-asset
219+
print('Uploading signed self-hosted xpi package to GitHub...')
220+
with open(signed_xpi_filepath, 'rb') as f:
221+
url = release_info['upload_url'].replace('{?name,label}', '?name=' + signed_xpi_filename)
222+
headers = {
223+
'Authorization': github_auth,
224+
'Content-Type': 'application/zip',
225+
}
226+
response = requests.post(url, headers=headers, data=f.read())
227+
if response.status_code != 201:
228+
print('Error: Upload signed package failed -- server error: {0}'.format(response.status_code))
229+
exit(1)
230+
231+
#
232+
# Remove raw package from GitHub
233+
#
234+
235+
# https://developer.github.com/v3/repos/releases/#delete-a-release-asset
236+
print('Remove raw xpi package from GitHub...')
237+
headers = { 'Authorization': github_auth, }
238+
response = requests.delete(raw_xpi_url, headers=headers)
239+
if response.status_code != 204:
240+
print('Error: Deletion of raw package failed -- server error: {0}'.format(response.status_code))
241+
242+
#
243+
# Update updates.json to point to new package -- but only if just-signed
244+
# package is higher version than current one.
245+
#
246+
247+
print('Update GitHub to point to newly signed self-hosted xpi package...')
248+
updates_json_filepath = os.path.join(projdir, 'dist', 'firefox', 'updates.json')
249+
with open(updates_json_filepath) as f:
250+
updates_json = json.load(f)
251+
f.close()
252+
previous_version = updates_json['addons'][extension_id]['updates'][0]['version']
253+
if StrictVersion(version) > StrictVersion(previous_version):
254+
with open(os.path.join(projdir, 'platform', 'webext', 'updates.template.json')) as f:
255+
template_json = Template(f.read())
256+
f.close()
257+
updates_json = template_json.substitute(version=version)
258+
with open(updates_json_filepath, 'w') as f:
259+
f.write(updates_json)
260+
f.close()
261+
# Automatically git add/commit if needed.
262+
# - Stage the changed file
263+
r = subprocess.run(['git', 'status', '-s', updates_json_filepath], stdout=subprocess.PIPE)
264+
rout = bytes.decode(r.stdout).strip()
265+
if len(rout) >= 2 and rout[1] == 'M':
266+
subprocess.run(['git', 'add', updates_json_filepath])
267+
# - Commit the staged file
268+
r = subprocess.run(['git', 'status', '-s', updates_json_filepath], stdout=subprocess.PIPE)
269+
rout = bytes.decode(r.stdout).strip()
270+
if len(rout) >= 2 and rout[0] == 'M':
271+
subprocess.run(['git', 'commit', '-m', 'make Firefox dev build auto-update', updates_json_filepath])
272+
subprocess.run(['git', 'push', 'origin', 'master'])
273+
274+
print('All done.')

0 commit comments

Comments
 (0)