bromite-buildtools/.github/workflows/build_bromite.yaml
2021-12-08 14:06:50 +01:00

409 lines
17 KiB
YAML

name: Build x86
permissions:
actions: none
checks: none
contents: none
deployments: none
issues: none
packages: none
pull-requests: none
repository-projects: none
security-events: none
statuses: none
on:
workflow_dispatch:
inputs:
sha:
description: 'uazo/bromite SHA'
required: true
default: '76745fde6ed75542f005ea5528f486df41c56da4'
usegoma:
description: 'Use goma?'
required: true
default: 'true'
env:
BROMITE_SHA: ${{ github.event.inputs.sha }}
USEGOMA: ${{ github.event.inputs.usegoma }}
REMOVEDOCKERSUPPORT: true
USELOCALIMAGE: true
GOMAJOBS: 60
jobs:
check_images:
runs-on: self-hosted
steps:
- name: Checkout repo
uses: actions/checkout@v2
with:
path: bromite-buildtools
fetch-depth: 1
- name: Enable proxy on container
shell: bash
run: |
if ! [[ -z "${HTTP_PROXY}" ]]; then
PROXY_ADDR=http://$(hostname -I | cut -d' ' -f1 | xargs):8118
echo "PROXY_ADDR=$PROXY_ADDR" >> $GITHUB_ENV
sudo iptables -D INPUT -p tcp -s localhost --dport 8118 -j ACCEPT
sudo iptables -D INPUT -p tcp --dport 8118 -j DROP
fi
- name: Get current chromium version
shell: bash
run: |
mkdir bromite
cd bromite
git init
git remote add origin https://github.com/uazo/bromite
git fetch origin $BROMITE_SHA
git reset --hard FETCH_HEAD
cd ..
export VERSION=$( cat ./bromite/build/RELEASE )
rm -rf bromite
echo Current version is $VERSION
echo "VERSION=$VERSION" >> $GITHUB_ENV
cd bromite-buildtools
- name: Checking build-deps for ${{ env.VERSION }}
shell: bash
run: |
IS_PRESENT=$(docker inspect --type=image uazo/build-deps:$VERSION > /dev/null ; echo $?)
if [ $IS_PRESENT -ne "0" ]; then
IS_PRESENT=$(docker manifest inspect uazo/build-deps:$VERSION > /dev/null ; echo $?)
if [ $IS_PRESENT -ne "0" ]; then
DOCKER_BUILDKIT=1 docker build -t uazo/build-deps:$VERSION \
--progress plain \
--build-arg VERSION=$VERSION \
--build-arg HTTP_PROXY="$PROXY_ADDR" \
--no-cache \
bromite-buildtools/images/build-deps/.
fi
fi
- name: Checking chromium for ${{ env.VERSION }}
shell: bash
run: |
IS_PRESENT=$(docker inspect --type=image uazo/chromium:$VERSION > /dev/null ; echo $?)
if [ $IS_PRESENT -ne "0" ]; then
IS_PRESENT=$(docker manifest inspect uazo/chromium:$VERSION > /dev/null ; echo $?)
if [ $IS_PRESENT -ne "0" ]; then
DOCKER_BUILDKIT=1 docker build -t uazo/chromium:$VERSION \
--progress plain \
--build-arg VERSION=$VERSION \
--build-arg HTTP_PROXY="$PROXY_ADDR" \
bromite-buildtools/images/chr-source/.
fi
fi
- name: Checking bromite for ${{ env.BROMITE_SHA }}
shell: bash
run: |
IS_PRESENT=$(docker inspect --type=image uazo/bromite:$BROMITE_SHA > /dev/null ; echo $?)
if [ $IS_PRESENT -ne "0" ]; then
IS_PRESENT=$(docker manifest inspect uazo/bromite:$BROMITE_SHA > /dev/null ; echo $?)
if [ $IS_PRESENT -ne "0" ]; then
DOCKER_BUILDKIT=1 docker build -t uazo/bromite:$BROMITE_SHA --progress plain \
--build-arg BROMITE_SHA=$BROMITE_SHA \
--build-arg VERSION=$VERSION \
--build-arg HTTP_PROXY="$PROXY_ADDR" \
bromite-buildtools/images/bromite-source/.
fi
fi
- name: Checking bromite-build for ${{ env.BROMITE_SHA }}
shell: bash
run: |
IS_PRESENT=$(docker inspect --type=image uazo/bromite-build:$BROMITE_SHA > /dev/null ; echo $?)
if [ $IS_PRESENT -ne "0" ]; then
IS_PRESENT=$(docker manifest inspect uazo/bromite-build:$BROMITE_SHA > /dev/null ; echo $?)
if [ $IS_PRESENT -ne "0" ]; then
DOCKER_BUILDKIT=1 docker build -t uazo/bromite-build:$BROMITE_SHA --progress plain \
--build-arg BROMITE_SHA=$BROMITE_SHA \
--build-arg HTTP_PROXY="$PROXY_ADDR" \
--no-cache \
bromite-buildtools/images/bromite-build/.
fi
fi
- name: Mark image to build
shell: bash
run: |
IS_PRESENT=$(docker inspect --type=image uazo/bromite-build:build > /dev/null ; echo $?)
if [ $IS_PRESENT -eq "0" ]; then
docker rmi uazo/bromite-build:build
fi
docker tag uazo/bromite-build:$BROMITE_SHA uazo/bromite-build:build
build:
runs-on: self-hosted
needs: check_images
if: success()
timeout-minutes: 720
services:
gomaserver:
image: uazo/goma-server
volumes:
- /tmp/proxy:/tmp/proxy
- /redis:/var/lib/redis
options: >-
--health-cmd "hostname -I >/tmp/proxy/gomaserverip"
--health-interval 30s
--health-timeout 5s
--health-retries 5
env:
REMOVEDOCKERSUPPORT: true
USELOCALIMAGE: true
container:
image: uazo/bromite-build:build
env:
SERVER_HOST_GOMA: gomaserver
REMOVEDOCKERSUPPORT: true # CUSTOM RUNNER: remove sharing of docker socket
USELOCALIMAGE: true # CUSTOM RUNNER: permit use of local images
USEINTERNALNETWORK: true # CUSTOM RUNNER: create the docker network as internal
WORKSPACE: /home/lg/working_dir
KYTHE_CORPUS: chromium.googlesource.com/chromium/src
KYTHE_ROOT_DIRECTORY: /home/lg/working_dir/chromium/src
KYTHE_OUTPUT_DIRECTORY: /home/lg/working_dir/chromium/src/out/bromite/kythe
# compile in debug mode
TARGET_ISDEBUG: true
volumes:
- /storage/images/${{ github.event.inputs.sha }}:/home/lg/working_dir/chromium/src/out/bromite
- /tmp/proxy:/tmp/proxy
steps:
- name: Prepare Build Container
shell: bash
run: |
# set workspace paths
PATH=$WORKSPACE/chromium/src/third_party/llvm-build/Release+Asserts/bin:$WORKSPACE/depot_tools/:/usr/local/go/bin:$WORKSPACE/mtool/bin:$PATH
cd $WORKSPACE
# reset proxy env
HTTP_PROXY=
HTTPS_PROXY=
http_proxy=
https_proxy=
# update hosts
#sudo echo "$(cat /tmp/proxy/gomaserverip | xargs) gomaserver" >/etc/hosts
#sudo echo "$(cat /tmp/proxy/redisip | xargs) redis" >/etc/hosts
# set goma options
export SERVER_HOST_GOMA=$(cat /tmp/proxy/gomaserverip | xargs)
export GOMA_SERVER_HOST=$SERVER_HOST_GOMA
export GOMA_SERVER_PORT=5050
export GOMA_USE_SSL=false
export GOMA_HTTP_AUTHORIZATION_FILE=$WORKSPACE/.debug_auth_file
export GOMA_HERMETIC=error
export GOMA_USE_LOCAL=false
export GOMA_FALLBACK=true
export GOMA_ARBITRARY_TOOLCHAIN_SUPPORT=true
export GOMA_MAX_SUBPROCS_LOW=0
# set out folder permissions
test -d chromium/src/out/bromite || sudo mkdir -p chromium/src/out/bromite && \
sudo chown lg chromium/src/out &&
sudo chown lg chromium/src/out/bromite &&
# make kythe output directory
test -d $KYTHE_OUTPUT_DIRECTORY || mkdir -p $KYTHE_OUTPUT_DIRECTORY
sudo mkdir -p /run/user/1000/
sudo chown lg /run/user/1000/
sudo chmod g-rxw /run/user/1000/
sudo chmod o-rxw /run/user/1000/
# start goma client
[[ "$USEGOMA" = "true" ]] && \
echo "::group::-------- start goma client" && \
$WORKSPACE/goma/goma_ctl.py ensure_stop && \
$WORKSPACE/goma/goma_ctl.py ensure_start && \
echo "::endgroup::" \
|| true
cd chromium/src
OUT_PRESENT=0
test -f out/bromite/prepare_gn && OUT_PRESENT=1
if [[ OUT_PRESENT -eq 0 ]]; then
echo "::group::-------- gn gen"
[[ "$USEGOMA" = "true" ]] && \
gn gen --args="import(\"/home/lg/working_dir/bromite/build/GN_ARGS\") use_goma=true goma_dir=\"$WORKSPACE/goma\" $(cat ../../build_args.gni) " out/bromite \
|| \
gn gen --args="import(\"/home/lg/working_dir/bromite/build/GN_ARGS\") $(cat ../../build_args.gni) " out/bromite
echo "::endgroup::"
echo "::group::-------- gn args"
gn args out/bromite/ --list --short
gn args out/bromite/ --list >out/bromite/gn_list
echo "::endgroup::"
echo "::group::-------- apply .mtool"
test -f out/bromite/.mtool && \
cp out/bromite/.mtool .mtool && \
$WORKSPACE/mtool/chromium/mtime.sh --restore
echo "::endgroup::"
echo "OK" >out/bromite/prepare_gn
fi
if [[ -z "${GOMAJOBS}" ]]; then
GOMAJOBS=40
fi
echo "::group::-------- pre-cache toolchain"
[[ "$USEGOMA" = "true" ]] && \
sudo ../../casupload --cas-server=unix:/tmp/proxy/bots.sock --instance=default_instance \
third_party/android_ndk/toolchains/llvm/prebuilt/linux-x86_64/sysroot/usr/include \
third_party/android_ndk/toolchains/llvm/prebuilt/linux-x86_64/include \
third_party/llvm-build/Release+Asserts/lib \
third_party/llvm-build/Release+Asserts/bin \
buildtools/third_party/libc++ \
chrome/android/profiles/afdo.prof \
|| true
echo "::endgroup::"
- name: Build Bromite
shell: bash
run: |
PATH=$WORKSPACE/chromium/src/third_party/llvm-build/Release+Asserts/bin:$WORKSPACE/depot_tools/:/usr/local/go/bin:$WORKSPACE/mtool/bin:$PATH
cd $WORKSPACE/chromium/src
[[ "$USEGOMA" = "true" ]] && \
autoninja -j $GOMAJOBS -C out/bromite chrome_public_apk \
|| \
autoninja -C out/bromite chrome_public_apk
- name: Get ninja logs
shell: bash
run: |
cd $WORKSPACE
$WORKSPACE/ninjatracing/ninjatracing $WORKSPACE/chromium/src/out/bromite/.ninja_log >$WORKSPACE/chromium/src/out/bromite/ninja_log_trace.json
$WORKSPACE/chromium/src/third_party/catapult/tracing/bin/trace2html $WORKSPACE/chromium/src/out/bromite/ninja_log_trace.json
- name: Get goma logs
shell: bash
run: |
# reset proxy env
HTTP_PROXY=
HTTPS_PROXY=
http_proxy=
https_proxy=
[[ "$USEGOMA" = "true" ]] && \
wget http://127.0.0.1:8088/logz?INFO -O out/bromite/goma-client.log \
|| \
echo Skipped.
- name: Generate breakpad symbols
shell: bash
run: |
PATH=$WORKSPACE/chromium/src/third_party/llvm-build/Release+Asserts/bin:$WORKSPACE/depot_tools/:/usr/local/go/bin:$WORKSPACE/mtool/bin:$PATH
cd $WORKSPACE/chromium/src
echo "::group::-------- generating breakpad symbols"
autoninja -j $GOMAJOBS -C out/bromite minidump_stackwalk dump_syms
components/crash/content/tools/generate_breakpad_symbols.py --build-dir=out/bromite \
--symbols-dir=out/bromite/symbols/ --binary=out/bromite/lib.unstripped/libchrome.so \
--platform=android --clear --verbose
cp out/bromite/lib.unstripped/libchrome.so out/bromite/symbols/libchrome.lib.so
cp out/bromite/minidump_stackwalk out/bromite/symbols
cp out/bromite/dump_syms out/bromite/symbols
echo "::endgroup::"
- name: Generate kythe kzip
shell: bash
run: |
PATH=$WORKSPACE/chromium/src/third_party/llvm-build/Release+Asserts/bin:$WORKSPACE/depot_tools/:/usr/local/go/bin:$WORKSPACE/mtool/bin:$PATH
cd $WORKSPACE/chromium/src
#python tools/clang/scripts/generate_compdb.py -p out/bromite/ -o out/bromite/compile_commands.json chrome_public_apk
#ninja -C out/bromite chrome_public_apk -t compdb cc cxx objc objcxx >out/bromite/compile_commands.json
$WORKSPACE/ninja/ninja -C $WORKSPACE/chromium/src/out/bromite -a chrome_public_apk \
-t compdb cc cxx objc objcxx >$WORKSPACE/chromium/src/out/bromite/compile_commands.json
gn desc out/bromite 'chrome_public_apk' --format=json >out/bromite/gn_targets.json
#python tools/clang/scripts/run_tool.py --tool translation_unit -p out/bromite/ --all || true
tools/clang/scripts/run_tool.py --tool translation_unit -p out/bromite || true
python /home/lg/add_kythe_metadata.py out/bromite/ --corpus $KYTHE_CORPUS || true
/home/lg/package_index/latest/package_index \
--checkout_dir /home/lg/working_dir/chromium/src \
--path_to_compdb /home/lg/working_dir/chromium/src/out/bromite/compile_commands.json \
--path_to_gn_targets /home/lg/working_dir/chromium/src/out/bromite/gn_targets.json \
--path_to_java_kzips $KYTHE_OUTPUT_DIRECTORY \
--path_to_archive_output /home/lg/working_dir/chromium/src/out/bromite/chromium_linux.kzip \
--corpus $KYTHE_CORPUS \
--out_dir src/out/bromite || true
- name: Generate clangd index
shell: bash
run: |
PATH=$WORKSPACE/chromium/src/third_party/llvm-build/Release+Asserts/bin:$WORKSPACE/depot_tools/:/usr/local/go/bin:$WORKSPACE/mtool/bin:$PATH
cd $WORKSPACE/chromium/src
/home/lg/working_dir/clangd_snapshot_20211205/bin/clangd-indexer --executor=all-TUs out/bromite/compile_commands.json >out/bromite/bromite.idx
- name: Build junit tests
shell: bash
run: |
PATH=$WORKSPACE/chromium/src/third_party/llvm-build/Release+Asserts/bin:$WORKSPACE/depot_tools/:/usr/local/go/bin:$WORKSPACE/mtool/bin:$PATH
cd $WORKSPACE/chromium/src
git log | grep FILE:Fix-build-test-suite.patch && TEST=1
if [[ TEST -eq 1 ]]; then
autoninja -j $GOMAJOBS -C out/bromite chrome_junit_tests
autoninja -j $GOMAJOBS -C out/bromite components_junit_tests
autoninja -j $GOMAJOBS -C out/bromite content_junit_tests
autoninja -j $GOMAJOBS -C out/bromite base_junit_tests
autoninja -j $GOMAJOBS -C out/bromite ui_junit_tests
fi
- name: Build c++ tests
shell: bash
run: |
PATH=$WORKSPACE/chromium/src/third_party/llvm-build/Release+Asserts/bin:$WORKSPACE/depot_tools/:/usr/local/go/bin:$WORKSPACE/mtool/bin:$PATH
cd $WORKSPACE/chromium/src
git log | grep FILE:Fix-build-test-suite.patch && TEST=1
if [[ TEST -eq 1 ]]; then
autoninja -j $GOMAJOBS -C out/bromite network_service
autoninja -j $GOMAJOBS -C out/bromite unit_tests
autoninja -j $GOMAJOBS -C out/bromite content_browsertests
autoninja -j $GOMAJOBS -C out/bromite components_unittests
autoninja -j $GOMAJOBS -C out/bromite components_browsertests
fi
- name: Build instrumentation tests
shell: bash
run: |
PATH=$WORKSPACE/chromium/src/third_party/llvm-build/Release+Asserts/bin:$WORKSPACE/depot_tools/:/usr/local/go/bin:$WORKSPACE/mtool/bin:$PATH
cd $WORKSPACE/chromium/src
git log | grep FILE:Fix-build-test-suite.patch && TEST=1
if [[ TEST -eq 1 ]]; then
autoninja -j $GOMAJOBS -C out/bromite content_shell_test_apk
autoninja -j $GOMAJOBS -C out/bromite chrome_public_test_apk
fi
- name: Stop goma
shell: bash
run: |
PATH=$WORKSPACE/chromium/src/third_party/llvm-build/Release+Asserts/bin:$WORKSPACE/depot_tools/:/usr/local/go/bin:$WORKSPACE/mtool/bin:$PATH
cd $WORKSPACE/chromium/src
echo "::group::-------- stop goma"
[[ "$USEGOMA" = "true" ]] && $WORKSPACE/goma/goma_ctl.py ensure_stop || true
[[ "$USEGOMA" = "true" ]] && find /tmp/ -maxdepth 1 -name "gomacc*" -print0 | xargs -0 rm || true
[[ "$USEGOMA" = "true" ]] && rm -rf /tmp/goma_lg/ || true
echo "::endgroup::"