diff --git a/casupload b/casupload new file mode 100755 index 0000000..a181f10 Binary files /dev/null and b/casupload differ diff --git a/images/bromite-build/Dockerfile b/images/bromite-build/Dockerfile new file mode 100644 index 0000000..f6f877a --- /dev/null +++ b/images/bromite-build/Dockerfile @@ -0,0 +1,23 @@ +ARG VERSION + +FROM localhost:5000/uazo/bromite:$VERSION +ARG VERSION +ENV VERSION=$VERSION + +USER lg + +COPY pre-start.sh . +COPY start-build.sh . +COPY goma_auth.py . +COPY casupload . + +RUN sudo chmod +x ./start-build.sh \ + && \ + sudo chmod +x ./pre-start.sh \ + && \ + sudo chmod 775 ./goma_auth.py \ + && \ + ./pre-start.sh + +CMD ./start-build.sh + diff --git a/images/bromite-build/casupload b/images/bromite-build/casupload new file mode 100755 index 0000000..a181f10 Binary files /dev/null and b/images/bromite-build/casupload differ diff --git a/images/bromite-build/goma_auth.py b/images/bromite-build/goma_auth.py new file mode 100644 index 0000000..05f417d --- /dev/null +++ b/images/bromite-build/goma_auth.py @@ -0,0 +1,3 @@ +#!/usr/bin/env python +def main(): + return 0 diff --git a/images/bromite-build/pre-start.sh b/images/bromite-build/pre-start.sh new file mode 100644 index 0000000..4293cdc --- /dev/null +++ b/images/bromite-build/pre-start.sh @@ -0,0 +1,19 @@ +#!/bin/bash + +RED='\033[0;31m' +NC='\033[0m' # No Color + +WORKSPACE=/home/lg/working_dir + +PATH=$WORKSPACE/chromium/src/third_party/llvm-build/Release+Asserts/bin:$WORKSPACE/depot_tools/:/usr/local/go/bin:$PATH + +sudo apt-get install -y lsof libgoogle-glog-dev libprotobuf17 libgrpc++1 +cipd install infra/goma/client/linux-amd64 -root $WORKSPACE/goma + +echo "nomatter" >$WORKSPACE/.debug_auth_file +sudo cp $WORKSPACE/goma_auth.py $WORKSPACE/goma/ + +cd chromium/src + +echo -e ${RED} -------- gn gen ${NC} +gn gen --args="$(cat ../../bromite/build/GN_ARGS) target_cpu=\"x86\" use_goma=true goma_dir=\"$WORKSPACE/goma\" " out/bromite diff --git a/images/bromite-build/start-build.sh b/images/bromite-build/start-build.sh new file mode 100644 index 0000000..5c8461e --- /dev/null +++ b/images/bromite-build/start-build.sh @@ -0,0 +1,42 @@ +#!/bin/bash + +RED='\033[0;31m' +NC='\033[0m' # No Color + +WORKSPACE=/home/lg/working_dir + +PATH=$WORKSPACE/chromium/src/third_party/llvm-build/Release+Asserts/bin:$WORKSPACE/depot_tools/:/usr/local/go/bin:$PATH + +export GOMA_SERVER_HOST=127.0.0.1 +export GOMA_SERVER_PORT=5050 +export GOMA_USE_SSL=false +export GOMA_HTTP_AUTHORIZATION_FILE=$WORKSPACE/.debug_auth_file +export GOMA_HERMETIC=error +export GOMA_USE_LOCAL=false +export GOMA_FALLBACK=true +export GOMA_ARBITRARY_TOOLCHAIN_SUPPORT=true + +$WORKSPACE/goma/goma_ctl.py ensure_stop +$WORKSPACE/goma/goma_ctl.py ensure_start + +cd chromium/src + +if [[ -z "${GOMAJOBS}" ]]; then + GOMAJOBS=40 +fi + +echo -e ${RED} -------- pre-cache toolchain ${NC} +../../casupload --cas-server=http://$REMOTEEXEC_ADDR --instance=default_instance \ + third_party/android_ndk/toolchains/llvm/prebuilt/linux-x86_64/sysroot/usr/include \ + third_party/android_ndk/toolchains/llvm/prebuilt/linux-x86_64/include \ + third_party/llvm-build/Release+Asserts/lib \ + third_party/llvm-build/Release+Asserts/bin \ + buildtools/third_party/libc++ \ + chrome/android/profiles/afdo.prof + + +echo -e ${RED} -------- start build ${NC} +autoninja -j $GOMAJOBS -C out/bromite chrome_public_apk + +bash + diff --git a/images/bromite-source/Dockerfile b/images/bromite-source/Dockerfile new file mode 100644 index 0000000..cd6c54e --- /dev/null +++ b/images/bromite-source/Dockerfile @@ -0,0 +1,16 @@ +ARG VERSION + +FROM localhost:5000/uazo/chromium:$VERSION +ARG VERSION +ENV VERSION=$VERSION + +USER lg + +COPY apply-bromite-patches.sh . + +RUN sudo chmod +x ./apply-bromite-patches.sh \ + && \ + git clone https://github.com/bromite/bromite \ + && \ + ./apply-bromite-patches.sh + diff --git a/images/bromite-source/apply-bromite-patches.sh b/images/bromite-source/apply-bromite-patches.sh new file mode 100644 index 0000000..bb753b7 --- /dev/null +++ b/images/bromite-source/apply-bromite-patches.sh @@ -0,0 +1,24 @@ +#!/bin/bash + +RED='\033[0;31m' +NC='\033[0m' # No Color + +sudo apt install sed + +cd chromium/src + +echo -e ${RED} ------- apply patchs ${NC} +for file in $(cat ../../bromite/build/bromite_patches_list.txt) ; do + echo -e ${RED} " -> Apply $file" ${NC} + + REPL="0,/^---/s//FILE:"$file"\n---/" + cat ../../bromite/build/patches/$file | sed $REPL | git am + + if [ $? -ne 0 ] + then + echo -e "Error on ../../bromite/build/patches/${file}" + exit 1 + fi + + echo " " +done diff --git a/images/build-deps/Dockerfile b/images/build-deps/Dockerfile new file mode 100644 index 0000000..ae39867 --- /dev/null +++ b/images/build-deps/Dockerfile @@ -0,0 +1,38 @@ +FROM ubuntu:20.04 +ARG VERSION + +RUN apt-get update &&\ + DEBIAN_FRONTEND=noninteractive \ + apt-get -f -y install sudo lsb-release cl-base64 bash wget apt-utils python sed tzdata g++-multilib + +ENV user lg + +RUN useradd -m -d /home/${user} ${user} && \ + chown -R ${user} /home/${user} && \ + adduser ${user} sudo && \ + echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers + +USER ${user} + +WORKDIR /home/${user} + +RUN wget https://chromium.googlesource.com/chromium/src/+/refs/tags/$VERSION/build/install-build-deps.sh?format=TEXT -O install-build-deps.sh.base64 \ + && \ + wget https://chromium.googlesource.com/chromium/src/+/refs/tags/$VERSION/build/install-build-deps-android.sh?format=TEXT -O install-build-deps-android.sh.base64 \ + && \ + sudo base64 -d install-build-deps.sh.base64 >install-build-deps.sh \ + && \ + sudo base64 -d install-build-deps-android.sh.base64 >install-build-deps-android.sh \ + && \ + sed -i 's/snapcraft/wget/' install-build-deps.sh + +RUN chmod +x ./install-build-deps.sh \ + && \ + chmod +x ./install-build-deps-android.sh \ + && ls -la \ + && \ + sudo bash ./install-build-deps-android.sh --no-prompt + + + + diff --git a/images/buildboxcasd/Dockerfile b/images/buildboxcasd/Dockerfile new file mode 100644 index 0000000..09497ab --- /dev/null +++ b/images/buildboxcasd/Dockerfile @@ -0,0 +1,23 @@ +ARG VERSION +FROM localhost:5000/uazo/build-deps:$VERSION + +ARG REMOTEEXEC_ADDR + +ENV REMOTEEXEC_ADDR=$REMOTEEXEC_ADDR +ENV DEBIAN_FRONTEND=noninteractive + +USER lg +COPY buildbox-casd . + +RUN sudo apt-get -f -y install libgoogle-glog-dev libprotobuf17 libgrpc++1 &&\ + sudo chmod +x buildbox-casd + +CMD ./buildbox-casd \ + --instance=default_instance \ + --cas-instance=default_instance \ + --cas-remote=http://$REMOTEEXEC_ADDR \ + --ra-remote=http://$REMOTEEXEC_ADDR \ + --verbose \ + /wrk-cache + + diff --git a/images/buildboxcasd/buildbox-casd b/images/buildboxcasd/buildbox-casd new file mode 100644 index 0000000..1c06469 Binary files /dev/null and b/images/buildboxcasd/buildbox-casd differ diff --git a/images/buildboxrunner/Dockerfile b/images/buildboxrunner/Dockerfile new file mode 100644 index 0000000..9aab154 --- /dev/null +++ b/images/buildboxrunner/Dockerfile @@ -0,0 +1,26 @@ +ARG VERSION +FROM localhost:5000/uazo/build-deps:$VERSION + +ARG REMOTEEXEC_ADDR + +ENV REMOTEEXEC_ADDR=$REMOTEEXEC_ADDR +ENV DEBIAN_FRONTEND=noninteractive + +USER lg +COPY buildbox-worker . +COPY buildbox-run-hosttools . + +RUN sudo apt-get -f -y install libgoogle-glog-dev libprotobuf17 libgrpc++1 &&\ + sudo chmod +x buildbox-worker &&\ + sudo chmod +x buildbox-run-hosttools + +CMD PATH=.:$PATH &&\ + ./buildbox-worker \ + --instance=default_instance \ + --bots-remote=http://$REMOTEEXEC_ADDR \ + --bots-retry-limit=10 \ + --buildbox-run=buildbox-run-hosttools \ + --cas-remote=unix:/wrk-cache/casd.sock \ + --logstream-remote=unix:/wrk-cache/casd.sock \ + --cas-instance=default_instance \ + --cas-retry-limit=10 diff --git a/images/buildboxrunner/buildbox-run-hosttools b/images/buildboxrunner/buildbox-run-hosttools new file mode 100755 index 0000000..5a8ae74 Binary files /dev/null and b/images/buildboxrunner/buildbox-run-hosttools differ diff --git a/images/buildboxrunner/buildbox-worker b/images/buildboxrunner/buildbox-worker new file mode 100755 index 0000000..3f85e8d Binary files /dev/null and b/images/buildboxrunner/buildbox-worker differ diff --git a/images/chr-source/Dockerfile b/images/chr-source/Dockerfile new file mode 100644 index 0000000..a5f5759 --- /dev/null +++ b/images/chr-source/Dockerfile @@ -0,0 +1,16 @@ +ARG VERSION + +FROM localhost:5000/uazo/build-deps:$VERSION +ARG VERSION +ENV VERSION=$VERSION + +USER lg +WORKDIR working_dir + +COPY prepare-build.sh . +COPY depot_tools.diff . + +RUN sudo chmod +x ./prepare-build.sh \ + && \ + ./prepare-build.sh + diff --git a/images/chr-source/depot_tools.diff b/images/chr-source/depot_tools.diff new file mode 100644 index 0000000..0422a72 --- /dev/null +++ b/images/chr-source/depot_tools.diff @@ -0,0 +1,60 @@ +diff --git a/gclient_scm.py b/gclient_scm.py +--- a/gclient_scm.py ++++ b/gclient_scm.py +@@ -1012,28 +1012,35 @@ class GitWrapper(SCMWrapper): + gclient_utils.safe_makedirs(parent_dir) + + template_dir = None ++ use_fetch = False ++ ++ tmp_dir = tempfile.mkdtemp( ++ prefix='_gclient_%s_' % os.path.basename(self.checkout_path), ++ dir=parent_dir) ++ + if hasattr(options, 'no_history') and options.no_history: + if gclient_utils.IsGitSha(revision): + # In the case of a subproject, the pinned sha is not necessarily the + # head of the remote branch (so we can't just use --depth=N). Instead, + # we tell git to fetch all the remote objects from SHA..HEAD by means of + # a template git dir which has a 'shallow' file pointing to the sha. +- template_dir = tempfile.mkdtemp( +- prefix='_gclient_gittmp_%s' % os.path.basename(self.checkout_path), +- dir=parent_dir) +- self._Run(['init', '--bare', template_dir], options, cwd=self._root_dir) +- with open(os.path.join(template_dir, 'shallow'), 'w') as template_file: +- template_file.write(revision) +- clone_cmd.append('--template=' + template_dir) ++ self._Run(['init', tmp_dir], options, cwd=self._root_dir) ++ ++ self._Run(['-C', tmp_dir, 'remote', 'add', 'origin', url], options, cwd=self._root_dir) ++ ++ clone_cmd = cfg + ['-C', tmp_dir, 'fetch', '--progress'] ++ clone_cmd.append('--depth=1') ++ clone_cmd.append(url) ++ clone_cmd.append(revision) ++ use_fetch = True + else: + # Otherwise, we're just interested in the HEAD. Just use --depth. + clone_cmd.append('--depth=1') + +- tmp_dir = tempfile.mkdtemp( +- prefix='_gclient_%s_' % os.path.basename(self.checkout_path), +- dir=parent_dir) + try: +- clone_cmd.append(tmp_dir) ++ if use_fetch == False: ++ clone_cmd.append(tmp_dir) ++ + if self.print_outbuf: + print_stdout = True + filter_fn = None +@@ -1328,6 +1335,9 @@ class GitWrapper(SCMWrapper): + if refspec: + fetch_cmd.append(refspec) + ++ if hasattr(options, 'no_history') and options.no_history: ++ fetch_cmd.append('--depth=1') ++ + if prune: + fetch_cmd.append('--prune') + if options.verbose: diff --git a/images/chr-source/prepare-build.sh b/images/chr-source/prepare-build.sh new file mode 100644 index 0000000..e9e36dd --- /dev/null +++ b/images/chr-source/prepare-build.sh @@ -0,0 +1,69 @@ +#!/bin/bash + +RED='\033[0;31m' +NC='\033[0m' # No Color + +sudo chown lg . +sudo chmod -R 775 . + +mkdir aa + +echo -e ${RED} -------- chromium version is: $VERSION ${NC} + +echo -e ${RED} -------- cloning depot_tools ${NC} +git clone https://chromium.googlesource.com/chromium/tools/depot_tools.git + +echo -e ${RED} -------- apply depot_tools patch ${NC} +cd depot_tools/ +git apply ../depot_tools.diff +cd .. + +echo -e ${RED} -------- set envs ${NC} +PATH=/home/lg/working_dir/depot_tools:$PATH +echo $PATH + +echo -e ${RED} -------- download chromium repo ${NC} +mkdir ./chromium +cd ./chromium + +gclient root + +mkdir ./src +cd ./src + +git init +git remote add origin https://chromium.googlesource.com/chromium/src.git + +git fetch --depth 2 https://chromium.googlesource.com/chromium/src.git +refs/tags/$VERSION:chromium_$VERSION +git checkout $VERSION +VERSION_SHA=$( git show-ref -s $VERSION | head -n1 ) + +echo >../.gclient "solutions = [" +echo >>../.gclient " { \"name\" : 'src'," +echo >>../.gclient " \"url\" : 'https://chromium.googlesource.com/chromium/src.git@$VERSION_SHA'," +echo >>../.gclient " \"deps_file\" : 'DEPS'," +echo >>../.gclient " \"managed\" : True," +echo >>../.gclient " \"custom_deps\" : {" +echo >>../.gclient " }," +echo >>../.gclient " \"custom_vars\": {}," +echo >>../.gclient " }," +echo >>../.gclient "]" +echo >>../.gclient "target_os=['android']" + +git submodule foreach git config -f ./.git/config submodule.$name.ignore all +git config --add remote.origin.fetch '+refs/tags/*:refs/tags/*' +#git config diff.ignoreSubmodules all + +echo -e ${RED} -------- sync third_party repos ${NC} +gclient sync -D --no-history --nohooks + +git config user.email "you@example.com" +git config user.name "Your Name" + +# remove origin for chromium +# git remote remove origin + +echo -e ${RED} -------- running hooks ${NC} +gclient runhooks + + diff --git a/images/goma-server/Dockerfile b/images/goma-server/Dockerfile new file mode 100644 index 0000000..647b8ab --- /dev/null +++ b/images/goma-server/Dockerfile @@ -0,0 +1,19 @@ +FROM ubuntu:20.04 + +COPY install-goma-server.sh . +COPY start-goma-server.sh . +COPY config-file . + +RUN apt-get update \ + && \ + apt-get -y install sudo wget git \ + && \ + sudo chmod +x ./install-goma-server.sh \ + && \ + sudo chmod +x ./start-goma-server.sh \ + && \ + ./install-goma-server.sh + +CMD ./start-goma-server.sh + + diff --git a/images/goma-server/config-file b/images/goma-server/config-file new file mode 100644 index 0000000..a78e6e4 --- /dev/null +++ b/images/goma-server/config-file @@ -0,0 +1,17 @@ +version_id:"1", +configs: [{ + target: { + addr:"localhost", + }, + build_info: { + creator:"uazo", + hostname:"localhost", + toolchain:"xx" + }, + remoteexec_platform: { + rbe_instance_basename: "default_instance", + has_nsjail: false + }, + dimensions: "os:linux" +}] + diff --git a/images/goma-server/install-goma-server.sh b/images/goma-server/install-goma-server.sh new file mode 100755 index 0000000..61db901 --- /dev/null +++ b/images/goma-server/install-goma-server.sh @@ -0,0 +1,18 @@ +#!/bin/bash + +RED='\033[0;31m' +NC='\033[0m' # No Color + +#export GOPATH=~/go +#export GOCACHE=~/go +export REDISHOST=localhost + +echo -e ${RED} -------- install go 16.2 ${NC} + +wget https://golang.org/dl/go1.16.2.linux-amd64.tar.gz +sudo rm -rf /usr/local/go +sudo tar -C /usr/local -xzf go1.16.2.linux-amd64.tar.gz + +echo -e ${RED} -------- cloning goma-server ${NC} + +git clone https://github.com/uazo/goma-server diff --git a/images/goma-server/start-goma-server.sh b/images/goma-server/start-goma-server.sh new file mode 100755 index 0000000..41f0f6f --- /dev/null +++ b/images/goma-server/start-goma-server.sh @@ -0,0 +1,24 @@ +#!/bin/bash + +RED='\033[0;31m' +NC='\033[0m' # No Color + +#export REDISHOST=localhost + +echo -e ${RED} -------- start goma-server ${NC} + +cd ./goma-server/ +/usr/local/go/bin/go run ./cmd/remoteexec_proxy/main.go \ + --port 5050 \ + --remoteexec-addr $REMOTEEXEC_ADDR \ + --remote-instance-name default_instance \ + --insecure-remoteexec \ + --allowed-users ppp \ + --exec-config-file "./config-file" \ + --exec-check-cache-timeout 180s \ + --exec-max-retry-count 1 \ + --exec-execute-timeout 180s \ + --exec-upload-blobs-timeout 600s + +cd .. + diff --git a/images/registry/docker-compose.yml b/images/registry/docker-compose.yml new file mode 100644 index 0000000..07d55b7 --- /dev/null +++ b/images/registry/docker-compose.yml @@ -0,0 +1,13 @@ +version: '3' + +services: + registry: + image: registry:2 + restart: always + ports: + - "5000:5000" + environment: + REGISTRY_STORAGE_FILESYSTEM_ROOTDIRECTORY: /data + volumes: + - /storage/data:/data +