name: Build Bromite permissions: actions: none checks: none contents: none deployments: none issues: none packages: none pull-requests: none repository-projects: none security-events: none statuses: none on: workflow_dispatch: inputs: sha: description: 'uazo/bromite SHA' required: true default: '76745fde6ed75542f005ea5528f486df41c56da4' build: description: 'arch [arm64/x64]' required: true default: 'x64' type: description: 'runner? [dev/ci]' required: true default: 'dev' debug: description: 'debug? [true/false]' required: true default: 'true' usegoma: description: 'Use goma? [true/false]' required: true default: 'false' env: BROMITE_SHA: ${{ github.event.inputs.sha }} USEGOMA: ${{ github.event.inputs.usegoma }} REMOVEDOCKERSUPPORT: true USELOCALIMAGE: true GOMAJOBS: 60 jobs: check_images: runs-on: ${{ github.event.inputs.type }} steps: - name: Checkout repo uses: actions/checkout@v2 with: path: bromite-buildtools fetch-depth: 1 - name: Enable proxy on container shell: bash run: | if ! [[ -z "${HTTP_PROXY}" ]]; then PROXY_ADDR=http://$(hostname -I | cut -d' ' -f1 | xargs):8118 echo "PROXY_ADDR=$PROXY_ADDR" >> $GITHUB_ENV sudo iptables -D INPUT -p tcp -s localhost --dport 8118 -j ACCEPT sudo iptables -D INPUT -p tcp --dport 8118 -j DROP fi - name: Get current chromium version shell: bash run: | mkdir bromite cd bromite git init git remote add origin https://github.com/uazo/bromite git fetch origin $BROMITE_SHA git reset --hard FETCH_HEAD cd .. export VERSION=$( cat ./bromite/build/RELEASE ) rm -rf bromite echo Current version is $VERSION echo "VERSION=$VERSION" >> $GITHUB_ENV cd bromite-buildtools - name: Building build-deps container ${{ env.VERSION }} shell: bash run: | IS_PRESENT=$(docker inspect --type=image uazo/build-deps:$VERSION > /dev/null ; echo $?) if [ $IS_PRESENT -ne "0" ]; then IS_PRESENT=$(docker manifest inspect uazo/build-deps:$VERSION > /dev/null ; echo $?) if [ $IS_PRESENT -ne "0" ]; then DOCKER_BUILDKIT=1 docker build -t uazo/build-deps:$VERSION \ --progress plain \ --build-arg VERSION=$VERSION \ --build-arg HTTP_PROXY="$PROXY_ADDR" \ --no-cache \ bromite-buildtools/images/build-deps/. fi fi - name: Building chromium container ${{ env.VERSION }} shell: bash run: | IS_PRESENT=$(docker inspect --type=image uazo/chromium:$VERSION > /dev/null ; echo $?) if [ $IS_PRESENT -ne "0" ]; then IS_PRESENT=$(docker manifest inspect uazo/chromium:$VERSION > /dev/null ; echo $?) if [ $IS_PRESENT -ne "0" ]; then DOCKER_BUILDKIT=1 docker build -t uazo/chromium:$VERSION \ --progress plain \ --build-arg VERSION=$VERSION \ --build-arg HTTP_PROXY="$PROXY_ADDR" \ bromite-buildtools/images/chr-source/. fi fi - name: Building bromite container ${{ env.BROMITE_SHA }} shell: bash run: | IS_PRESENT=$(docker inspect --type=image uazo/bromite:$BROMITE_SHA > /dev/null ; echo $?) if [ $IS_PRESENT -ne "0" ]; then IS_PRESENT=$(docker manifest inspect uazo/bromite:$BROMITE_SHA > /dev/null ; echo $?) if [ $IS_PRESENT -ne "0" ]; then DOCKER_BUILDKIT=1 docker build -t uazo/bromite:$BROMITE_SHA --progress plain \ --build-arg BROMITE_SHA=$BROMITE_SHA \ --build-arg VERSION=$VERSION \ --build-arg HTTP_PROXY="$PROXY_ADDR" \ bromite-buildtools/images/bromite-source/. fi fi - name: Building bromite-build container ${{ env.BROMITE_SHA }} shell: bash run: | IS_PRESENT=$(docker inspect --type=image uazo/bromite-build:$BROMITE_SHA > /dev/null ; echo $?) if [ $IS_PRESENT -ne "0" ]; then IS_PRESENT=$(docker manifest inspect uazo/bromite-build:$BROMITE_SHA > /dev/null ; echo $?) if [ $IS_PRESENT -ne "0" ]; then DOCKER_BUILDKIT=1 docker build -t uazo/bromite-build:$BROMITE_SHA --progress plain \ --build-arg BROMITE_SHA=$BROMITE_SHA \ --build-arg HTTP_PROXY="$PROXY_ADDR" \ --no-cache \ bromite-buildtools/images/bromite-build/. fi fi - name: Mark image to build shell: bash run: | IS_PRESENT=$(docker inspect --type=image uazo/bromite-build:build > /dev/null ; echo $?) if [ $IS_PRESENT -eq "0" ]; then docker rmi uazo/bromite-build:build fi docker tag uazo/bromite-build:$BROMITE_SHA uazo/bromite-build:build build: runs-on: ${{ github.event.inputs.type }} needs: check_images if: success() timeout-minutes: 720 services: gomaserver: image: uazo/goma-server volumes: - /tmp/proxy:/tmp/proxy - /redis:/var/lib/redis options: >- --health-cmd "hostname -I >/tmp/proxy/gomaserverip" --health-interval 30s --health-timeout 5s --health-retries 5 env: REMOVEDOCKERSUPPORT: true USELOCALIMAGE: true container: image: uazo/bromite-build:build env: SERVER_HOST_GOMA: gomaserver REMOVEDOCKERSUPPORT: true # CUSTOM RUNNER: remove sharing of docker socket USELOCALIMAGE: true # CUSTOM RUNNER: permit use of local images USEINTERNALNETWORK: true # CUSTOM RUNNER: create the docker network as internal WORKSPACE: /home/lg/working_dir KYTHE_CORPUS: chromium.googlesource.com/chromium/src KYTHE_ROOT_DIRECTORY: /home/lg/working_dir/chromium/src KYTHE_OUTPUT_DIRECTORY: /home/lg/working_dir/chromium/src/out/bromite/kythe # compile in debug mode TARGET_ISDEBUG: ${{ github.event.inputs.debug }} TARGET_CPU: ${{ github.event.inputs.build }} volumes: - /storage/images/${{ github.event.inputs.build }}/${{ github.event.inputs.sha }}:/home/lg/working_dir/chromium/src/out/bromite - /tmp/proxy:/tmp/proxy steps: - name: Prepare Build Container shell: bash run: | # set workspace paths PATH=$WORKSPACE/chromium/src/third_party/llvm-build/Release+Asserts/bin:$WORKSPACE/depot_tools/:/usr/local/go/bin:$WORKSPACE/mtool/bin:$PATH cd $WORKSPACE # reset proxy env HTTP_PROXY= HTTPS_PROXY= http_proxy= https_proxy= # update hosts #sudo echo "$(cat /tmp/proxy/gomaserverip | xargs) gomaserver" >/etc/hosts #sudo echo "$(cat /tmp/proxy/redisip | xargs) redis" >/etc/hosts # set goma options export SERVER_HOST_GOMA=$(cat /tmp/proxy/gomaserverip | xargs) export GOMA_SERVER_HOST=$SERVER_HOST_GOMA export GOMA_SERVER_PORT=5050 export GOMA_USE_SSL=false export GOMA_HTTP_AUTHORIZATION_FILE=$WORKSPACE/.debug_auth_file export GOMA_HERMETIC=error export GOMA_USE_LOCAL=false export GOMA_FALLBACK=true export GOMA_ARBITRARY_TOOLCHAIN_SUPPORT=true export GOMA_MAX_SUBPROCS_LOW=0 # set out folder permissions test -d chromium/src/out/bromite || sudo mkdir -p chromium/src/out/bromite && \ sudo chown lg chromium/src/out && sudo chown lg chromium/src/out/bromite && # make kythe output directory test -d $KYTHE_OUTPUT_DIRECTORY || mkdir -p $KYTHE_OUTPUT_DIRECTORY sudo mkdir -p /run/user/1000/ sudo chown lg /run/user/1000/ sudo chmod g-rxw /run/user/1000/ sudo chmod o-rxw /run/user/1000/ # start goma client [[ "$USEGOMA" = "true" ]] && \ echo "::group::-------- start goma client" && \ $WORKSPACE/goma/goma_ctl.py ensure_stop && \ $WORKSPACE/goma/goma_ctl.py ensure_start && \ echo "::endgroup::" \ || true cd chromium/src OUT_PRESENT=0 test -f out/bromite/prepare_gn && OUT_PRESENT=1 if [[ OUT_PRESENT -eq 0 ]]; then echo "::group::-------- gn gen" [[ "$USEGOMA" = "true" ]] && \ gn gen --args="import(\"/home/lg/working_dir/bromite/build/bromite.gn_args\") use_goma=true goma_dir=\"$WORKSPACE/goma\" $(cat ../../build_args.gni) " out/bromite \ || \ gn gen --args="import(\"/home/lg/working_dir/bromite/build/bromite.gn_args\") $(cat ../../build_args.gni) " out/bromite echo "::endgroup::" echo "::group::-------- gn args" gn args out/bromite/ --list --short gn args out/bromite/ --list >out/bromite/gn_list echo "::endgroup::" echo "::group::-------- apply .mtool" test -f out/bromite/.mtool && \ cp out/bromite/.mtool .mtool && \ $WORKSPACE/mtool/chromium/mtime.sh --restore echo "::endgroup::" echo "OK" >out/bromite/prepare_gn fi cp ../../bromite/build/RELEASE out/bromite if [[ -z "${GOMAJOBS}" ]]; then GOMAJOBS=40 fi echo "::group::-------- pre-cache toolchain" [[ "$USEGOMA" = "true" ]] && \ sudo ../../casupload --cas-server=unix:/tmp/proxy/bots.sock --instance=default_instance \ third_party/android_ndk/toolchains/llvm/prebuilt/linux-x86_64/sysroot/usr/include \ third_party/android_ndk/toolchains/llvm/prebuilt/linux-x86_64/include \ third_party/llvm-build/Release+Asserts/lib \ third_party/llvm-build/Release+Asserts/bin \ buildtools/third_party/libc++ \ chrome/android/profiles/afdo.prof \ || true echo "::endgroup::" - name: Build Bromite shell: bash run: | PATH=$WORKSPACE/chromium/src/third_party/llvm-build/Release+Asserts/bin:$WORKSPACE/depot_tools/:/usr/local/go/bin:$WORKSPACE/mtool/bin:$PATH cd $WORKSPACE/chromium/src [[ "$USEGOMA" = "true" ]] && \ autoninja -j $GOMAJOBS -C out/bromite chrome_public_apk \ || \ autoninja -C out/bromite chrome_public_apk - name: Get ninja logs shell: bash run: | cd $WORKSPACE $WORKSPACE/ninjatracing/ninjatracing $WORKSPACE/chromium/src/out/bromite/.ninja_log >$WORKSPACE/chromium/src/out/bromite/ninja_log_trace.json $WORKSPACE/chromium/src/third_party/catapult/tracing/bin/trace2html $WORKSPACE/chromium/src/out/bromite/ninja_log_trace.json - name: Get goma logs shell: bash run: | # reset proxy env HTTP_PROXY= HTTPS_PROXY= http_proxy= https_proxy= [[ "$USEGOMA" = "true" ]] && \ wget http://127.0.0.1:8088/logz?INFO -O out/bromite/goma-client.log \ || \ echo Skipped. - name: Generate breakpad symbols shell: bash run: | PATH=$WORKSPACE/chromium/src/third_party/llvm-build/Release+Asserts/bin:$WORKSPACE/depot_tools/:/usr/local/go/bin:$WORKSPACE/mtool/bin:$PATH cd $WORKSPACE/chromium/src echo "::group::-------- generating breakpad symbols" autoninja -j $GOMAJOBS -C out/bromite minidump_stackwalk dump_syms components/crash/content/tools/generate_breakpad_symbols.py --build-dir=out/bromite \ --symbols-dir=out/bromite/symbols/ --binary=out/bromite/lib.unstripped/libchrome.so \ --platform=android --clear --verbose cp out/bromite/lib.unstripped/libchrome.so out/bromite/symbols/libchrome.lib.so cp out/bromite/minidump_stackwalk out/bromite/symbols cp out/bromite/dump_syms out/bromite/symbols echo "::endgroup::" # - name: Generate kythe kzip # shell: bash # run: | # PATH=$WORKSPACE/chromium/src/third_party/llvm-build/Release+Asserts/bin:$WORKSPACE/depot_tools/:/usr/local/go/bin:$WORKSPACE/mtool/bin:$PATH # cd $WORKSPACE/chromium/src # # gn desc out/bromite 'chrome_public_apk' --format=json >out/bromite/gn_targets.json # # #python tools/clang/scripts/run_tool.py --tool translation_unit -p out/bromite/ --all || true # tools/clang/scripts/run_tool.py --tool translation_unit -p out/bromite || true # python /home/lg/add_kythe_metadata.py out/bromite/ --corpus $KYTHE_CORPUS || true # # /home/lg/package_index/latest/package_index \ # --checkout_dir /home/lg/working_dir/chromium/src \ # --path_to_compdb /home/lg/working_dir/chromium/src/out/bromite/compile_commands.json \ # --path_to_gn_targets /home/lg/working_dir/chromium/src/out/bromite/gn_targets.json \ # --path_to_java_kzips $KYTHE_OUTPUT_DIRECTORY \ # --path_to_archive_output /home/lg/working_dir/chromium/src/out/bromite/chromium_linux.kzip \ # --corpus $KYTHE_CORPUS \ # --out_dir src/out/bromite || true - name: Generate clangd index if: ${{ github.event.inputs.debug == 'true' }} shell: bash run: | PATH=$WORKSPACE/chromium/src/third_party/llvm-build/Release+Asserts/bin:$WORKSPACE/depot_tools/:/usr/local/go/bin:$WORKSPACE/mtool/bin:$PATH cd $WORKSPACE/chromium/src $WORKSPACE/ninja/ninja -C $WORKSPACE/chromium/src/out/bromite -a chrome_public_apk \ -t compdb cc cxx objc objcxx >$WORKSPACE/chromium/src/out/bromite/compile_commands.json test -f out/bromite/bromite.idx || \ /home/lg/working_dir/clangd_snapshot_20211205/bin/clangd-indexer --executor=all-TUs out/bromite/compile_commands.json >out/bromite/bromite.idx - name: Generate Supersize data if: ${{ github.event.inputs.debug == 'false' }} shell: bash run: | PATH=$WORKSPACE/chromium/src/third_party/llvm-build/Release+Asserts/bin:$WORKSPACE/depot_tools/:/usr/local/go/bin:$WORKSPACE/mtool/bin:$PATH cd $WORKSPACE/chromium/src tools/binary_size/supersize archive out/bromite/chrome.size --apk-file out/bromite/apks/ChromePublic.apk -v - name: Build junit tests shell: bash run: | PATH=$WORKSPACE/chromium/src/third_party/llvm-build/Release+Asserts/bin:$WORKSPACE/depot_tools/:/usr/local/go/bin:$WORKSPACE/mtool/bin:$PATH cd $WORKSPACE/chromium/src git log | grep FILE:Fix-build-test-suite.patch && TEST=1 if [[ TEST -eq 1 ]]; then autoninja -j $GOMAJOBS -C out/bromite chrome_junit_tests autoninja -j $GOMAJOBS -C out/bromite components_junit_tests autoninja -j $GOMAJOBS -C out/bromite content_junit_tests autoninja -j $GOMAJOBS -C out/bromite base_junit_tests autoninja -j $GOMAJOBS -C out/bromite ui_junit_tests fi - name: Build c++ tests shell: bash run: | PATH=$WORKSPACE/chromium/src/third_party/llvm-build/Release+Asserts/bin:$WORKSPACE/depot_tools/:/usr/local/go/bin:$WORKSPACE/mtool/bin:$PATH cd $WORKSPACE/chromium/src git log | grep FILE:Fix-build-test-suite.patch && TEST=1 if [[ TEST -eq 1 ]]; then autoninja -j $GOMAJOBS -C out/bromite network_service autoninja -j $GOMAJOBS -C out/bromite unit_tests autoninja -j $GOMAJOBS -C out/bromite content_browsertests autoninja -j $GOMAJOBS -C out/bromite components_unittests autoninja -j $GOMAJOBS -C out/bromite components_browsertests fi - name: Build instrumentation tests shell: bash run: | PATH=$WORKSPACE/chromium/src/third_party/llvm-build/Release+Asserts/bin:$WORKSPACE/depot_tools/:/usr/local/go/bin:$WORKSPACE/mtool/bin:$PATH cd $WORKSPACE/chromium/src git log | grep FILE:Fix-build-test-suite.patch && TEST=1 if [[ TEST -eq 1 ]]; then autoninja -j $GOMAJOBS -C out/bromite content_shell_test_apk autoninja -j $GOMAJOBS -C out/bromite chrome_public_test_apk fi - name: Stop goma shell: bash run: | PATH=$WORKSPACE/chromium/src/third_party/llvm-build/Release+Asserts/bin:$WORKSPACE/depot_tools/:/usr/local/go/bin:$WORKSPACE/mtool/bin:$PATH cd $WORKSPACE/chromium/src echo "::group::-------- stop goma" [[ "$USEGOMA" = "true" ]] && $WORKSPACE/goma/goma_ctl.py ensure_stop || true [[ "$USEGOMA" = "true" ]] && find /tmp/ -maxdepth 1 -name "gomacc*" -print0 | xargs -0 rm || true [[ "$USEGOMA" = "true" ]] && rm -rf /tmp/goma_lg/ || true echo "::endgroup::"