ci: use a custom android sdk manager with pinning and mirroring
This commit is contained in:
Pietro Albini
parent
ee1474acc4
commit
4e920f2b04
|
|
@ -7,23 +7,21 @@ COPY scripts/android-ndk.sh /scripts/
|
|||
RUN . /scripts/android-ndk.sh && \
|
||||
download_and_make_toolchain android-ndk-r15c-linux-x86_64.zip arm 14
|
||||
|
||||
# Note:
|
||||
# Do not upgrade to `openjdk-9-jre-headless`, as it will cause certificate error
|
||||
# when installing the Android SDK (see PR #45193). This is unfortunate, but
|
||||
# every search result suggested either disabling HTTPS or replacing JDK 9 by
|
||||
# JDK 8 as the solution (e.g. https://stackoverflow.com/q/41421340). :|
|
||||
RUN dpkg --add-architecture i386 && \
|
||||
apt-get update && \
|
||||
apt-get install -y --no-install-recommends \
|
||||
libgl1-mesa-glx \
|
||||
libpulse0 \
|
||||
libstdc++6:i386 \
|
||||
openjdk-8-jre-headless \
|
||||
tzdata
|
||||
openjdk-9-jre-headless \
|
||||
tzdata \
|
||||
wget \
|
||||
python3
|
||||
|
||||
COPY scripts/android-sdk.sh /scripts/
|
||||
RUN . /scripts/android-sdk.sh && \
|
||||
download_and_create_avd 4333796 armeabi-v7a 18 5264690
|
||||
COPY scripts/android-sdk-manager.py /scripts/
|
||||
COPY arm-android/android-sdk.lock /android/sdk/android-sdk.lock
|
||||
RUN /scripts/android-sdk.sh
|
||||
|
||||
ENV PATH=$PATH:/android/sdk/emulator
|
||||
ENV PATH=$PATH:/android/sdk/tools
|
||||
|
|
|
|||
|
|
@ -0,0 +1,6 @@
|
|||
emulator emulator-linux-5264690.zip 48c1cda2bdf3095d9d9d5c010fbfb3d6d673e3ea
|
||||
patcher;v4 3534162-studio.sdk-patcher.zip 046699c5e2716ae11d77e0bad814f7f33fab261e
|
||||
platform-tools platform-tools_r28.0.2-linux.zip 46a4c02a9b8e4e2121eddf6025da3c979bf02e28
|
||||
platforms;android-18 android-18_r03.zip e6b09b3505754cbbeb4a5622008b907262ee91cb
|
||||
system-images;android-18;default;armeabi-v7a sys-img/android/armeabi-v7a-18_r05.zip 580b583720f7de671040d5917c8c9db0c7aa03fd
|
||||
tools sdk-tools-linux-4333796.zip 8c7c28554a32318461802c1291d76fccfafde054
|
||||
|
|
@ -0,0 +1,190 @@
|
|||
#!/usr/bin/env python3
|
||||
# Simpler reimplementation of Android's sdkmanager
|
||||
# Extra features of this implementation are pinning and mirroring
|
||||
|
||||
# These URLs are the Google repositories containing the list of available
|
||||
# packages and their versions. The list has been generated by listing the URLs
|
||||
# fetched while executing `tools/bin/sdkmanager --list`
|
||||
BASE_REPOSITORY = "https://dl.google.com/android/repository/"
|
||||
REPOSITORIES = [
|
||||
"sys-img/android/sys-img2-1.xml",
|
||||
"sys-img/android-wear/sys-img2-1.xml",
|
||||
"sys-img/android-wear-cn/sys-img2-1.xml",
|
||||
"sys-img/android-tv/sys-img2-1.xml",
|
||||
"sys-img/google_apis/sys-img2-1.xml",
|
||||
"sys-img/google_apis_playstore/sys-img2-1.xml",
|
||||
"addon2-1.xml",
|
||||
"glass/addon2-1.xml",
|
||||
"extras/intel/addon2-1.xml",
|
||||
"repository2-1.xml",
|
||||
]
|
||||
|
||||
# Available hosts: linux, macosx and windows
|
||||
HOST_OS = "linux"
|
||||
|
||||
# Mirroring options
|
||||
MIRROR_BUCKET = "rust-lang-ci2"
|
||||
MIRROR_BASE_DIR = "rust-ci-mirror/android/"
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import urllib.request
|
||||
import xml.etree.ElementTree as ET
|
||||
|
||||
class Package:
|
||||
def __init__(self, path, url, sha1, deps=None):
|
||||
if deps is None:
|
||||
deps = []
|
||||
self.path = path.strip()
|
||||
self.url = url.strip()
|
||||
self.sha1 = sha1.strip()
|
||||
self.deps = deps
|
||||
|
||||
def download(self, base_url):
|
||||
_, file = tempfile.mkstemp()
|
||||
url = base_url + self.url
|
||||
subprocess.run(["curl", "-o", file, url], check=True)
|
||||
# Ensure there are no hash mismatches
|
||||
with open(file, "rb") as f:
|
||||
sha1 = hashlib.sha1(f.read()).hexdigest()
|
||||
if sha1 != self.sha1:
|
||||
raise RuntimeError(
|
||||
"hash mismatch for package " + self.path + ": " +
|
||||
sha1 + " vs " + self.sha1 + " (known good)"
|
||||
)
|
||||
return file
|
||||
|
||||
def __repr__(self):
|
||||
return "<Package "+self.path+" at "+self.url+" (sha1="+self.sha1+")"
|
||||
|
||||
def fetch_url(url):
|
||||
page = urllib.request.urlopen(url)
|
||||
return page.read()
|
||||
|
||||
def fetch_repository(base, repo_url):
|
||||
packages = {}
|
||||
root = ET.fromstring(fetch_url(base + repo_url))
|
||||
for package in root:
|
||||
if package.tag != "remotePackage":
|
||||
continue
|
||||
path = package.attrib["path"]
|
||||
|
||||
for archive in package.find("archives"):
|
||||
host_os = archive.find("host-os")
|
||||
if host_os is not None and host_os.text != HOST_OS:
|
||||
continue
|
||||
complete = archive.find("complete")
|
||||
url = os.path.join(os.path.dirname(repo_url), complete.find("url").text)
|
||||
sha1 = complete.find("checksum").text
|
||||
|
||||
deps = []
|
||||
dependencies = package.find("dependencies")
|
||||
if dependencies is not None:
|
||||
for dep in dependencies:
|
||||
deps.append(dep.attrib["path"])
|
||||
|
||||
packages[path] = Package(path, url, sha1, deps)
|
||||
break
|
||||
|
||||
return packages
|
||||
|
||||
def fetch_repositories():
|
||||
packages = {}
|
||||
for repo in REPOSITORIES:
|
||||
packages.update(fetch_repository(BASE_REPOSITORY, repo))
|
||||
return packages
|
||||
|
||||
class Lockfile:
|
||||
def __init__(self, path):
|
||||
self.path = path
|
||||
self.packages = {}
|
||||
if os.path.exists(path):
|
||||
with open(path) as f:
|
||||
for line in f:
|
||||
path, url, sha1 = line.split(" ")
|
||||
self.packages[path] = Package(path, url, sha1)
|
||||
|
||||
def add(self, packages, name, *, update=True):
|
||||
if name not in packages:
|
||||
raise NameError("package not found: " + name)
|
||||
if not update and name in self.packages:
|
||||
return
|
||||
self.packages[name] = packages[name]
|
||||
for dep in packages[name].deps:
|
||||
self.add(packages, dep, update=False)
|
||||
|
||||
def save(self):
|
||||
packages = list(sorted(self.packages.values(), key=lambda p: p.path))
|
||||
with open(self.path, "w") as f:
|
||||
for package in packages:
|
||||
f.write(package.path + " " + package.url + " " + package.sha1 + "\n")
|
||||
|
||||
def cli_add_to_lockfile(args):
|
||||
lockfile = Lockfile(args.lockfile)
|
||||
packages = fetch_repositories()
|
||||
for package in args.packages:
|
||||
lockfile.add(packages, package)
|
||||
lockfile.save()
|
||||
|
||||
def cli_update_mirror(args):
|
||||
lockfile = Lockfile(args.lockfile)
|
||||
for package in lockfile.packages.values():
|
||||
path = package.download(BASE_REPOSITORY)
|
||||
subprocess.run([
|
||||
"aws", "s3", "mv", path,
|
||||
"s3://" + MIRROR_BUCKET + "/" + MIRROR_BASE_DIR + package.url,
|
||||
"--profile=" + args.awscli_profile,
|
||||
], check=True)
|
||||
|
||||
def cli_install(args):
|
||||
lockfile = Lockfile(args.lockfile)
|
||||
for package in lockfile.packages.values():
|
||||
# Download the file from the mirror into a temp file
|
||||
url = "https://" + MIRROR_BUCKET + ".s3.amazonaws.com/" + MIRROR_BASE_DIR
|
||||
downloaded = package.download(url)
|
||||
# Extract the file in a temporary directory
|
||||
extract_dir = tempfile.mkdtemp()
|
||||
subprocess.run([
|
||||
"unzip", "-q", downloaded, "-d", extract_dir,
|
||||
], check=True)
|
||||
# Figure out the prefix used in the zip
|
||||
subdirs = [d for d in os.listdir(extract_dir) if not d.startswith(".")]
|
||||
if len(subdirs) != 1:
|
||||
raise RuntimeError("extracted directory contains more than one dir")
|
||||
# Move the extracted files in the proper directory
|
||||
dest = os.path.join(args.dest, package.path.replace(";", "/"))
|
||||
os.makedirs("/".join(dest.split("/")[:-1]), exist_ok=True)
|
||||
os.rename(os.path.join(extract_dir, subdirs[0]), dest)
|
||||
os.unlink(downloaded)
|
||||
|
||||
def cli():
|
||||
parser = argparse.ArgumentParser()
|
||||
subparsers = parser.add_subparsers()
|
||||
|
||||
add_to_lockfile = subparsers.add_parser("add-to-lockfile")
|
||||
add_to_lockfile.add_argument("lockfile")
|
||||
add_to_lockfile.add_argument("packages", nargs="+")
|
||||
add_to_lockfile.set_defaults(func=cli_add_to_lockfile)
|
||||
|
||||
update_mirror = subparsers.add_parser("update-mirror")
|
||||
update_mirror.add_argument("lockfile")
|
||||
update_mirror.add_argument("--awscli-profile", default="default")
|
||||
update_mirror.set_defaults(func=cli_update_mirror)
|
||||
|
||||
install = subparsers.add_parser("install")
|
||||
install.add_argument("lockfile")
|
||||
install.add_argument("dest")
|
||||
install.set_defaults(func=cli_install)
|
||||
|
||||
args = parser.parse_args()
|
||||
if not hasattr(args, "func"):
|
||||
print("error: a subcommand is required (see --help)")
|
||||
exit(1)
|
||||
args.func(args)
|
||||
|
||||
if __name__ == "__main__":
|
||||
cli()
|
||||
|
|
@ -2,66 +2,28 @@ set -ex
|
|||
|
||||
export ANDROID_HOME=/android/sdk
|
||||
PATH=$PATH:"${ANDROID_HOME}/tools/bin"
|
||||
LOCKFILE="${ANDROID_HOME}/android-sdk.lock"
|
||||
|
||||
download_sdk() {
|
||||
mkdir -p /android
|
||||
curl -fo sdk.zip "https://dl.google.com/android/repository/sdk-tools-linux-$1.zip"
|
||||
unzip -q sdk.zip -d "$ANDROID_HOME"
|
||||
rm -f sdk.zip
|
||||
}
|
||||
|
||||
download_sysimage() {
|
||||
abi=$1
|
||||
api=$2
|
||||
|
||||
# See https://developer.android.com/studio/command-line/sdkmanager.html for
|
||||
# usage of `sdkmanager`.
|
||||
#
|
||||
# The output from sdkmanager is so noisy that it will occupy all of the 4 MB
|
||||
# log extremely quickly. Thus we must silence all output.
|
||||
yes | sdkmanager --licenses > /dev/null
|
||||
yes | sdkmanager platform-tools \
|
||||
"platforms;android-$api" \
|
||||
"system-images;android-$api;default;$abi" > /dev/null
|
||||
}
|
||||
|
||||
download_emulator() {
|
||||
# Download a pinned version of the emulator since upgrades can cause issues
|
||||
curl -fo emulator.zip "https://dl.google.com/android/repository/emulator-linux-$1.zip"
|
||||
rm -rf "${ANDROID_HOME}/emulator"
|
||||
unzip -q emulator.zip -d "${ANDROID_HOME}"
|
||||
rm -f emulator.zip
|
||||
}
|
||||
|
||||
create_avd() {
|
||||
abi=$1
|
||||
api=$2
|
||||
|
||||
# See https://developer.android.com/studio/command-line/avdmanager.html for
|
||||
# usage of `avdmanager`.
|
||||
echo no | avdmanager create avd \
|
||||
-n "$abi-$api" \
|
||||
-k "system-images;android-$api;default;$abi"
|
||||
}
|
||||
|
||||
download_and_create_avd() {
|
||||
download_sdk $1
|
||||
download_sysimage $2 $3
|
||||
create_avd $2 $3
|
||||
download_emulator $4
|
||||
}
|
||||
|
||||
# Usage:
|
||||
# To add a new packages to the SDK or to update an existing one you need to
|
||||
# run the command:
|
||||
#
|
||||
# download_and_create_avd 4333796 armeabi-v7a 18 5264690
|
||||
# android-sdk-manager.py add-to-lockfile $LOCKFILE <package-name>
|
||||
#
|
||||
# 4333796 =>
|
||||
# SDK tool version.
|
||||
# Copy from https://developer.android.com/studio/index.html#command-tools
|
||||
# armeabi-v7a =>
|
||||
# System image ABI
|
||||
# 18 =>
|
||||
# Android API Level (18 = Android 4.3 = Jelly Bean MR2)
|
||||
# 5264690 =>
|
||||
# Android Emulator version.
|
||||
# Copy from the "build_id" in the `/android/sdk/emulator/emulator -version` output
|
||||
# Then, after every lockfile update the mirror has to be synchronized as well:
|
||||
#
|
||||
# android-sdk-manager.py update-mirror $LOCKFILE
|
||||
#
|
||||
/scripts/android-sdk-manager.py install "${LOCKFILE}" "${ANDROID_HOME}"
|
||||
|
||||
details=$(cat "${LOCKFILE}" \
|
||||
| grep system-images \
|
||||
| sed 's/^system-images;android-\([0-9]\+\);default;\([a-z0-9-]\+\) /\1 \2 /g')
|
||||
api="$(echo "${details}" | awk '{print($1)}')"
|
||||
abi="$(echo "${details}" | awk '{print($2)}')"
|
||||
|
||||
# See https://developer.android.com/studio/command-line/avdmanager.html for
|
||||
# usage of `avdmanager`.
|
||||
echo no | avdmanager create avd \
|
||||
-n "$abi-$api" \
|
||||
-k "system-images;android-$api;default;$abi"
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue