Include liblsquic build in dockerfile + add support for ARM64 on Docker (#2147)

Co-authored-by: Emilien Devos <contact@emiliendevos.be>
2021-07-15 23:57:20 +02:00 · 2021-07-15 23:57:20 +02:00 · aa3608f0af
commit aa3608f0af
parent 56ebef4352
5 changed files with 226 additions and 4 deletions
--- a/.github/workflows/container-release.yml
+++ b/.github/workflows/container-release.yml
@ -17,6 +17,8 @@ jobs:

      - name: Set up QEMU
        uses: docker/setup-qemu-action@v1
+        with:
+          platforms: arm64

      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v1
@ -28,12 +30,43 @@ jobs:
          username: ${{ secrets.QUAY_USERNAME }}
          password: ${{ secrets.QUAY_PASSWORD }}

-      - name: Build and push for Push Event
+      - name: Cache Docker layers
+        if: github.ref == 'refs/heads/master'
+        uses: actions/cache@v2
+        with:
+          path: /tmp/.buildx-cache
+          key: ${{ runner.os }}-multi-buildx-${{ github.sha }}
+          restore-keys: |
+            ${{ runner.os }}-multi-buildx
+
+      - name: Build and push Docker AMD64 image for Push Event
        if: github.ref == 'refs/heads/master'
        uses: docker/build-push-action@v2
        with:
          context: .
          file: docker/Dockerfile
+          platforms: linux/amd64
          labels: quay.expires-after=12w
          push: true
          tags: quay.io/invidious/invidious:${{ github.sha }},quay.io/invidious/invidious:latest
+          cache-from: type=local,src=/tmp/.buildx-cache
+          cache-to: type=local,mode=max,dest=/tmp/.buildx-cache-new
+
+      - name: Build and push Docker ARM64 image for Push Event
+        if: github.ref == 'refs/heads/master'
+        uses: docker/build-push-action@v2
+        with:
+          context: .
+          file: docker/Dockerfile.arm64
+          platforms: linux/arm64/v8
+          labels: quay.expires-after=12w
+          push: true
+          tags: quay.io/invidious/invidious:${{ github.sha }}-arm64,quay.io/invidious/invidious:latest-arm64
+          cache-from: type=local,src=/tmp/.buildx-cache
+          cache-to: type=local,mode=max,dest=/tmp/.buildx-cache-new
+
+      - name: Override old Docker cache
+        if: github.ref == 'refs/heads/master'
+        run: |
+          rm -rf /tmp/.buildx-cache
+          mv /tmp/.buildx-cache-new /tmp/.buildx-cache
--- a/docker/APKBUILD-boringssl
+++ b/docker/APKBUILD-boringssl
@ -0,0 +1,46 @@
+# Based on https://aur.archlinux.org/packages/boringssl-git/
+# Maintainer: Omar Roth <omarroth@protonmail.com>
+pkgname=boringssl
+pkgver=1.1.0
+pkgrel=0
+pkgdesc="BoringSSL is a fork of OpenSSL that is designed to meet Google's needs"
+url="https://boringssl.googlesource.com/boringssl"
+arch="all"
+license="MIT"
+replaces="openssl libressl"
+depends="!openssl-libs-static"
+makedepends_host="linux-headers"
+makedepends="cmake git go perl"
+subpackages="$pkgname-static $pkgname-dev $pkgname-doc"
+source="251b516.tar.gz::https://github.com/google/boringssl/tarball/251b516"
+builddir="$srcdir/google-boringssl-251b516"
+
+prepare() {
+	:
+}
+
+build() {
+	cmake -DCMAKE_BUILD_TYPE=Release .
+	make ssl crypto
+}
+
+check() {
+	make all_tests
+}
+
+package() {
+	for i in *.md ; do
+		install -Dm644 $i "$pkgdir/usr/share/doc/$pkgname/$i"
+	done
+	install -d "$pkgdir/usr/lib"
+	install -d "$pkgdir/usr/include"
+	cp -R include/openssl "$pkgdir/usr/include"
+
+	install -Dm755 crypto/libcrypto.a "$pkgdir/usr/lib/libcrypto.a"
+	install -Dm755 ssl/libssl.a "$pkgdir/usr/lib/libssl.a"
+#	install -Dm755 decrepit/libdecrepit.a "$pkgdir/usr/lib/libdecrepit.a"
+#	install -Dm755 libboringssl_gtest.a "$pkgdir/usr/lib/libboringssl_gtest.a"
+}
+sha512sums="
+b1d42ed188cf0cce89d40061fa05de85b387ee4244f1236ea488a431536a2c6b657b4f03daed0ac9328c7f5c4c9330499283b8a67f1444dcf9ba5e97e1199c4e  251b516.tar.gz
+"
--- a/docker/APKBUILD-lsquic
+++ b/docker/APKBUILD-lsquic
@ -0,0 +1,43 @@
+# Maintainer: Omar Roth <omarroth@protonmail.com>
+pkgname=lsquic
+pkgver=2.18.1
+pkgrel=0
+pkgdesc="LiteSpeed QUIC and HTTP/3 Library"
+url="https://github.com/litespeedtech/lsquic"
+arch="all"
+license="MIT"
+depends="boringssl-dev boringssl-static zlib-static libevent-static"
+makedepends="cmake git go perl bsd-compat-headers linux-headers"
+subpackages="$pkgname-static"
+source="v$pkgver.tar.gz::https://github.com/litespeedtech/lsquic/tarball/v2.18.1
+ls-qpack-$pkgver.tar.gz::https://github.com/litespeedtech/ls-qpack/tarball/a8ae6ef
+ls-hpack-$pkgver.tar.gz::https://github.com/litespeedtech/ls-hpack/tarball/bd5d589"
+builddir="$srcdir/litespeedtech-$pkgname-692a910"
+
+prepare() {
+	cp -r -T "$srcdir/litespeedtech-ls-qpack-a8ae6ef" "$builddir/src/liblsquic/ls-qpack"
+	cp -r -T "$srcdir/litespeedtech-ls-hpack-bd5d589" "$builddir/src/lshpack"
+}
+
+build() {
+	cmake \
+		-DCMAKE_BUILD_TYPE=None \
+		-DBORINGSSL_INCLUDE=/usr/include/openssl \
+		-DBORINGSSL_LIB_crypto=/usr/lib \
+		-DBORINGSSL_LIB_ssl=/usr/lib .
+	make lsquic
+}
+
+check() {
+	make tests
+}
+
+package() {
+	install -d "$pkgdir/usr/lib"
+	install -Dm755 src/liblsquic/liblsquic.a "$pkgdir/usr/lib/liblsquic.a"
+}
+sha512sums="
+d015a72f1e88750ecb364768a40f532678f11ded09c6447a2e698b20f43fa499ef143a53f4c92a5938dfece0e39e687dc9df4aea97c618faee0c63da771561c3  v2.18.1.tar.gz
+c5629085a3881815fb0b72a321eeba8de093eff9417b8ac7bde1ee1264971be0dca6d61d74799b02ae03a4c629b2a9cf21387deeb814935339a8a2503ea33fee  ls-qpack-2.18.1.tar.gz
+1b9f7ce4c82dadfca8154229a415b0335a61761eba698f814d4b94195c708003deb5cb89318a1ab78ac8fa88b141bc9df283fb1c6e40b3ba399660feaae353a0  ls-hpack-2.18.1.tar.gz
+"
--- a/docker/Dockerfile
+++ b/docker/Dockerfile
@ -1,10 +1,44 @@
+FROM alpine:edge AS liblsquic-builder
+WORKDIR /src
+
+RUN apk add --no-cache build-base git apk-tools abuild cmake go perl linux-headers
+
+RUN abuild-keygen -a -n && \
+    cp /root/.abuild/-*.rsa.pub /etc/apk/keys/
+
+COPY docker/APKBUILD-boringssl boringssl/APKBUILD
+RUN cd boringssl && abuild -F -r && cd ..
+
+RUN apk add --repository /root/packages/src boringssl boringssl-dev boringssl-static
+
+RUN apk add --no-cache zlib-dev zlib-static libevent-dev libevent-static
+
+COPY docker/APKBUILD-lsquic lsquic/APKBUILD
+RUN cd lsquic && abuild -F -r && cd ..
+
+RUN apk add --repository /root/packages/src lsquic-static
+
+RUN mkdir tmp && cd tmp && \
+    ar -x /usr/lib/libssl.a && \
+    ar -x /usr/lib/libcrypto.a && \
+    ar -x /usr/lib/liblsquic.a && \
+    ar rc liblsquic.a *.o && \
+    strip --strip-unneeded liblsquic.a && \
+    ranlib liblsquic.a && \
+    cp liblsquic.a /root/liblsquic.a && \
+    cd .. && rm -rf tmp
+
+
 FROM crystallang/crystal:1.0.0-alpine AS builder
-RUN apk add --no-cache curl sqlite-static yaml-static
+RUN apk add --no-cache sqlite-static yaml-static
+
 WORKDIR /invidious
 COPY ./shard.yml ./shard.yml
 COPY ./shard.lock ./shard.lock
-RUN shards install && \
-    curl -Lo ./lib/lsquic/src/lsquic/ext/liblsquic.a https://github.com/iv-org/lsquic-static-alpine/releases/download/v2.18.1/liblsquic.a
+RUN shards install
+
+COPY --from=liblsquic-builder /root/liblsquic.a ./lib/lsquic/src/lsquic/ext/liblsquic.a
+
 COPY ./src/ ./src/
 # TODO: .git folder is required for building – this is destructive.
 # See definition of CURRENT_BRANCH, CURRENT_COMMIT and CURRENT_VERSION.
--- a/docker/Dockerfile.arm64
+++ b/docker/Dockerfile.arm64
@ -0,0 +1,66 @@
+FROM alpine:3.14 AS liblsquic-builder
+WORKDIR /src
+
+RUN apk add --no-cache build-base git apk-tools abuild cmake go perl linux-headers
+
+RUN abuild-keygen -a -n && \
+    cp /root/.abuild/-*.rsa.pub /etc/apk/keys/
+
+COPY docker/APKBUILD-boringssl boringssl/APKBUILD
+RUN cd boringssl && abuild -F -r && cd ..
+
+RUN apk add --repository /root/packages/src boringssl boringssl-dev boringssl-static
+
+RUN apk add --no-cache zlib-dev zlib-static libevent-dev libevent-static
+
+COPY docker/APKBUILD-lsquic lsquic/APKBUILD
+RUN cd lsquic && abuild -F -r && cd ..
+
+RUN apk add --repository /root/packages/src lsquic-static
+
+RUN mkdir tmp && cd tmp && \
+    ar -x /usr/lib/libssl.a && \
+    ar -x /usr/lib/libcrypto.a && \
+    ar -x /usr/lib/liblsquic.a && \
+    ar rc liblsquic.a *.o && \
+    strip --strip-unneeded liblsquic.a && \
+    ranlib liblsquic.a && \
+    cp liblsquic.a /root/liblsquic.a && \
+    cd .. && rm -rf tmp
+
+
+FROM alpine:3.14 AS builder
+RUN apk add --no-cache 'crystal<2' shards sqlite-static yaml-static yaml-dev libxml2-dev zlib-static openssl-libs-static openssl-dev musl-dev
+
+WORKDIR /invidious
+COPY ./shard.yml ./shard.yml
+COPY ./shard.lock ./shard.lock
+RUN shards install
+
+COPY --from=liblsquic-builder /root/liblsquic.a ./lib/lsquic/src/lsquic/ext/liblsquic.a
+
+COPY ./src/ ./src/
+# TODO: .git folder is required for building – this is destructive.
+# See definition of CURRENT_BRANCH, CURRENT_COMMIT and CURRENT_VERSION.
+COPY ./.git/ ./.git/
+RUN crystal build ./src/invidious.cr \
+    --static --warnings all \
+    --link-flags "-lxml2 -llzma"
+
+FROM alpine:latest
+RUN apk add --no-cache librsvg ttf-opensans
+WORKDIR /invidious
+RUN addgroup -g 1000 -S invidious && \
+    adduser -u 1000 -S invidious -G invidious
+COPY ./assets/ ./assets/
+COPY --chown=invidious ./config/config.* ./config/
+RUN mv -n config/config.example.yml config/config.yml
+RUN sed -i 's/host: \(127.0.0.1\|localhost\)/host: postgres/' config/config.yml
+COPY ./config/sql/ ./config/sql/
+COPY ./locales/ ./locales/
+COPY --from=builder /invidious/invidious .
+RUN chmod o+rX -R ./assets ./config ./locales
+
+EXPOSE 3000
+USER invidious
+CMD [ "/invidious/invidious" ]