diff --git a/.editorconfig b/.editorconfig index 1598f5e..b399155 100644 --- a/.editorconfig +++ b/.editorconfig @@ -5,12 +5,15 @@ charset = utf-8 end_of_line = lf trim_trailing_whitespace = true insert_final_newline = true +indent_style = tab +indent_size = 4 -[{.,}*.{js{,*},y{a,}ml,sh}] +[*.ps1] +indent_style = space + +[{{.,}*.{js{,on},y{a,}ml,sh,md,txt},.dockerfilelintrc}] indent_style = space indent_size = 2 [*.{md,txt}] -indent_style = space -indent_size = 2 trim_trailing_whitespace = true diff --git a/.github/workflows/build-ubuntu.yml b/.github/workflows/build-ubuntu.yml index 8c0b139..5ec3ae5 100644 --- a/.github/workflows/build-ubuntu.yml +++ b/.github/workflows/build-ubuntu.yml @@ -16,9 +16,11 @@ on: workflow_dispatch: env: + IMAGE_LABEL_OWNER: ${{ github.repository_owner }} + IMAGE_LABEL_REPO: ${{ github.repository }} SLUG: ${{ github.repository_owner }}/ubuntu DISTRO: ubuntu - NODE: '14' + NODE: '12 16' BUILD_REF: ${{ github.sha }} SKIP_TEST: false @@ -28,20 +30,27 @@ defaults: jobs: build-base: + name: Build base ${{ matrix.TAG }} runs-on: ubuntu-latest + env: + PLATFORMS: ${{ matrix.PLATFORMS }} strategy: fail-fast: true max-parallel: 4 matrix: + PLATFORMS: + - linux/amd64,linux/arm64,linux/armhf + TAG: + - latest + - 20.04 + - 18.04 include: - - TAG: latest - PLATFORMS: linux/amd64,linux/arm64 - - TAG: 20.04 - PLATFORMS: linux/amd64,linux/arm64 - TAG: 18.04 PLATFORMS: linux/amd64 - env: - PLATFORMS: ${{ matrix.PLATFORMS }} + exclude: + # no arm bins for docker CLIs + - TAG: 18.04 + PLATFORMS: linux/amd64,linux/arm64,linux/armhf steps: - name: Login to GitHub Container Registry if: ${{ github.repository_owner == github.actor }} @@ -71,25 +80,24 @@ jobs: - name: Print tag run: | - if ${{ github.event_name == 'pull_request' }} && ${{ !env.ACT }} ; then - echo "PART_TAG=PR-${{ github.event.number }}" >> $GITHUB_ENV - elif ${{ !env.ACT }} ; then - echo "PART_TAG=$(date +%Y%m%d)" >> $GITHUB_ENV + if ${{ ( github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' ) }} ; then + echo "PART_TAG=$(date +%Y%m%d)" | tee -a "$GITHUB_ENV" else - echo "PART_TAG=dev" >> $GITHUB_ENV + echo "PART_TAG=dev" | tee -a "$GITHUB_ENV" fi - name: Set Ubuntu version to RELEASE run: | if [ "latest" = "${{ matrix.TAG }}" ]; then - echo "RELEASE_TAG=$(lsb_release -rs)" >> $GITHUB_ENV + echo "RELEASE_TAG=$(lsb_release -rs)" | tee -a "$GITHUB_ENV" else - echo "RELEASE_TAG=${{ matrix.TAG }}" >> $GITHUB_ENV + echo "RELEASE_TAG=${{ matrix.TAG }}" | tee -a "$GITHUB_ENV" fi - name: Set up QEMU uses: docker/setup-qemu-action@v1 + # setup buildkit ourselves so it doesn't create a new one each time act is executed - run: | docker buildx create \ --use \ @@ -101,8 +109,8 @@ jobs: - uses: actions/checkout@v2 - - name: Build and push ${{ env.SLUG }}:${{ env.TAG }} - run: ./build.sh + - name: Build and push ${{ env.DISTRO }}:${{ env.TAG }} + shell: pwsh env: RUNNER: root TAG: act-${{ matrix.TAG }}-${{ env.PART_TAG }} @@ -111,9 +119,17 @@ jobs: FROM_TAG: ${{ env.RELEASE_TAG }} BUILD_TAG_VERSION: ${{ env.PART_TAG }} BUILD_TAG: act-${{ matrix.TAG }} + run: | + ./build.ps1 ` + -push ` + -tags @( + 'ghcr.io/${{ env.SLUG }}:${{ env.TAG }}', + 'quay.io/${{ env.SLUG }}:${{ env.TAG }}', + 'docker.io/${{ env.SLUG }}:${{ env.TAG }}' + ) - - name: Build and push ${{ env.SLUG }}:${{ env.TAG }} - run: ./build.sh + - name: Build and push ${{ env.DISTRO }}:${{ env.TAG }} + shell: pwsh env: RUNNER: runner TAG: runner-${{ matrix.TAG }}-${{ env.PART_TAG }} @@ -122,12 +138,22 @@ jobs: FROM_TAG: act-${{ matrix.TAG }}-${{ env.PART_TAG }} BUILD_TAG_VERSION: ${{ env.PART_TAG }} BUILD_TAG: runner-${{ matrix.TAG }} + run: | + ./build.ps1 ` + -push ` + -tags @( + 'ghcr.io/${{ env.SLUG }}:${{ env.TAG }}', + 'quay.io/${{ env.SLUG }}:${{ env.TAG }}', + 'docker.io/${{ env.SLUG }}:${{ env.TAG }}' + ) - - uses: actions/setup-go@v2 + - if: ${{ !env.SKIP_TEST }} + uses: actions/setup-go@v2 with: - go-version: 1.16 + go-version: '^1.16' - - uses: actions/checkout@v2 + - if: ${{ !env.SKIP_TEST }} + uses: actions/checkout@v2 with: repository: nektos/act path: act @@ -139,21 +165,28 @@ jobs: cd act/ go test ./... - - name: Build and push ${{ env.SLUG }}:${{ env.TAG }} - if: ${{ ( github.event_name != 'pull_request' && !env.ACT ) }} - run: ./build.sh + - name: Build and push ${{ env.DISTRO }}:${{ env.TAG }} + if: ${{ ( github.event_name != 'pull_request' && github.event_name != 'push' && !env.ACT ) }} + shell: pwsh env: - RUNNER: root TAG: act-${{ matrix.TAG }} TYPE: act FROM_IMAGE: buildpack-deps FROM_TAG: ${{ env.RELEASE_TAG }} BUILD_TAG_VERSION: ${{ env.PART_TAG }} BUILD_TAG: act-${{ matrix.TAG }} + run: | + ./build.ps1 ` + -push ` + -tags @( + 'ghcr.io/${{ env.SLUG }}:${{ env.TAG }}', + 'quay.io/${{ env.SLUG }}:${{ env.TAG }}', + 'docker.io/${{ env.SLUG }}:${{ env.TAG }}' + ) - - name: Build and push ${{ env.SLUG }}:${{ env.TAG }} - if: ${{ ( github.event_name != 'pull_request' && !env.ACT ) }} - run: ./build.sh + - name: Build and push ${{ env.DISTRO }}:${{ env.TAG }} + if: ${{ ( github.event_name != 'pull_request' && github.event_name != 'push' && !env.ACT ) }} + shell: pwsh env: RUNNER: runner TAG: runner-${{ matrix.TAG }} @@ -162,40 +195,42 @@ jobs: FROM_TAG: act-${{ matrix.TAG }} BUILD_TAG_VERSION: ${{ env.PART_TAG }} BUILD_TAG: runner-${{ matrix.TAG }} + run: | + ./build.ps1 ` + -push ` + -tags @( + 'ghcr.io/${{ env.SLUG }}:${{ env.TAG }}', + 'quay.io/${{ env.SLUG }}:${{ env.TAG }}', + 'docker.io/${{ env.SLUG }}:${{ env.TAG }}' + ) + build-flavours: + name: Build ${{ matrix.TYPE }}:${{ matrix.TAG }} runs-on: ubuntu-latest needs: [build-base] - strategy: - fail-fast: false - max-parallel: 2 - matrix: - include: - - TAG: latest - TYPE: js - PLATFORMS: linux/amd64,linux/arm64 - - TAG: latest - TYPE: pwsh - PLATFORMS: linux/amd64,linux/arm64 - - TAG: 20.04 - TYPE: js - PLATFORMS: linux/amd64,linux/arm64 - - TAG: 20.04 - TYPE: pwsh - PLATFORMS: linux/amd64,linux/arm64 - - TAG: 18.04 - TYPE: js - PLATFORMS: linux/amd64 - - TAG: 18.04 - TYPE: pwsh - PLATFORMS: linux/amd64 env: PLATFORMS: ${{ matrix.PLATFORMS }} BUILD_TAG: ${{ matrix.TYPE }}-${{ matrix.TAG }} - TAG: ${{ matrix.TYPE }}-${{ matrix.TAG }} TYPE: ${{ matrix.TYPE }} - RUNNER: root + strategy: + fail-fast: false + max-parallel: 8 + matrix: + TAG: + - latest + - 20.04 + - 18.04 + PLATFORMS: + - 'linux/amd64' + TYPE: + - go + - js + - pwsh + - rust + - custom steps: - name: Login to GitHub Container Registry + if: ${{ github.repository_owner == github.actor }} id: ghcr uses: docker/login-action@v1 with: @@ -204,6 +239,7 @@ jobs: password: ${{ secrets.GITHUB_TOKEN }} - name: Login to Quay + if: ${{ github.repository_owner == github.actor }} id: quay uses: docker/login-action@v1 with: @@ -212,6 +248,7 @@ jobs: password: ${{ secrets.QUAY_TOKEN }} - name: Login to Docker Hub + if: ${{ github.repository_owner == github.actor }} id: dckr uses: docker/login-action@v1 with: @@ -220,20 +257,18 @@ jobs: - name: Print tag run: | - if ${{ github.event_name == 'pull_request' }} && ${{ !env.ACT }} ; then - echo "PART_TAG=PR-${{ github.event.number }}" >> $GITHUB_ENV - elif ${{ !env.ACT }} ; then - echo "PART_TAG=$(date +%Y%m%d)" >> $GITHUB_ENV + if ${{ ( github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' ) }} ; then + echo "PART_TAG=$(date +%Y%m%d)" | tee -a "$GITHUB_ENV" else - echo "PART_TAG=dev" >> $GITHUB_ENV + echo "PART_TAG=dev" | tee -a "$GITHUB_ENV" fi - name: Set Ubuntu version to RELEASE run: | if [ "latest" = "${{ matrix.TAG }}" ]; then - echo "RELEASE_TAG=$(lsb_release -rs)" >> $GITHUB_ENV + echo "RELEASE_TAG=$(lsb_release -rs)" | tee -a "$GITHUB_ENV" else - echo "RELEASE_TAG=${{ matrix.TAG }}" >> $GITHUB_ENV + echo "RELEASE_TAG=${{ matrix.TAG }}" | tee -a "$GITHUB_ENV" fi - name: Set up QEMU @@ -250,18 +285,35 @@ jobs: - uses: actions/checkout@v2 - - name: Build and push ${{ env.SLUG }}:${{ env.TAG }} - run: ./build.sh + - name: Build and push ${{ env.DISTRO }}:${{ env.TAG }} + shell: pwsh env: - TAG: ${{ env.TAG }}-${{ env.PART_TAG }} + TAG: ${{ matrix.TYPE }}-${{ matrix.TAG }}-${{ env.PART_TAG }} FROM_IMAGE: ghcr.io/${{ env.SLUG }} FROM_TAG: act-${{ matrix.TAG }}-${{ env.PART_TAG }} BUILD_TAG_VERSION: ${{ env.PART_TAG }} + run: | + ./build.ps1 ` + -push ` + -tags @( + 'ghcr.io/${{ env.SLUG }}:${{ env.TAG }}', + 'quay.io/${{ env.SLUG }}:${{ env.TAG }}', + 'docker.io/${{ env.SLUG }}:${{ env.TAG }}' + ) - - name: Build and push ${{ env.SLUG }}:${{ env.TAG }} - run: ./build.sh - if: ${{ ( github.event_name != 'pull_request' && !env.ACT ) }} + - name: Build and push ${{ env.DISTRO }}:${{ env.TAG }} + shell: pwsh + if: ${{ ( github.event_name != 'pull_request' && github.event_name != 'push' && !env.ACT ) }} env: + TAG: ${{ matrix.TYPE }}-${{ matrix.TAG }}-${{ env.PART_TAG }} FROM_IMAGE: ghcr.io/${{ env.SLUG }} FROM_TAG: act-${{ matrix.TAG }}-${{ env.PART_TAG }} BUILD_TAG_VERSION: ${{ env.PART_TAG }} + run: | + ./build.ps1 ` + -push ` + -tags @( + 'ghcr.io/${{ env.SLUG }}:${{ env.TAG }}', + 'quay.io/${{ env.SLUG }}:${{ env.TAG }}', + 'docker.io/${{ env.SLUG }}:${{ env.TAG }}' + ) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index f4b5af7..893780a 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -19,6 +19,8 @@ jobs: - name: Lint uses: megalinter/megalinter/flavors/documentation@v5.2.0 env: - VALIDATE_ALL_CODEBASE: ${{ github.event_name != 'pull_request' }} DEFAULT_BRANCH: master GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + VALIDATE_ALL_CODEBASE: false + GITHUB_STATUS_REPORTER: ${{ !env.ACT }} + GITHUB_COMMENT_REPORTER: ${{ !env.ACT }} diff --git a/.gitignore b/.gitignore index 90fcf32..c82384b 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,9 @@ .secrets .env +# wip +*.hcl + # mega-linter report diff --git a/.mega-linter.yml b/.mega-linter.yml index 2ee8471..43d3843 100644 --- a/.mega-linter.yml +++ b/.mega-linter.yml @@ -3,4 +3,4 @@ DISABLE: - COPYPASTE DISABLE_LINTERS: - SPELL_CSPELL - - DOCKERFILE_DOCKERFILELINT +PRINT_ALPACA: false diff --git a/README.md b/README.md index 81ceeff..280cffd 100644 --- a/README.md +++ b/README.md @@ -16,7 +16,9 @@ ## Images available - [catthehacker/virtual-environments-fork][catthehacker/virtual-environments-fork] - GitHub Actions runner image containing all possible tools (image is extremely big, 20GB compressed, ~60GB extracted) + - this image is updated manually due to amount of changes in [actions/virtual-environments][actions/virtual-environments] + - `ghcr.io/catthehacker/ubuntu:full-latest` - `ghcr.io/catthehacker/ubuntu:full-20.04` - `ghcr.io/catthehacker/ubuntu:full-18.04` diff --git a/build.ps1 b/build.ps1 index 5bdf731..22fb7d1 100644 --- a/build.ps1 +++ b/build.ps1 @@ -1,30 +1,64 @@ param( - $slug = 'catthehacker/ubuntu', - $tag, - $node = '12', - $distro = 'ubuntu', - $type, - $image = 'ubuntu', - $platforms = 'linux/amd64', - $build_version = "master", - $build_tag, - $build_tag_version = "dev", - $build_ref = 'master', - $from_image, - $from_tag, - $runner + [Parameter(ValueFromPipeline)] + [string]$progress, + [Parameter(ValueFromPipeline)] + [string]$owner = "${env:IMAGE_LABEL_OWNER}", + [Parameter(ValueFromPipeline)] + [string]$repository = "${env:IMAGE_LABEL_REPO}", + [Parameter(ValueFromPipeline)] + [string]$slug = 'catthehacker/ubuntu', + [Parameter(ValueFromPipeline)] + [string[]]$tags, + [Parameter(ValueFromPipeline)] + [string]$tag, + [Parameter(ValueFromPipeline)] + [string]$node = "${env:NODE}", + [Parameter(ValueFromPipeline)] + [string]$distro = 'ubuntu', + [Parameter(ValueFromPipeline)] + [string]$type = "${env:TYPE}", + [Parameter(ValueFromPipeline)] + [string]$runner = "${env:RUNNER}", + [Parameter(ValueFromPipeline)] + [string]$image = 'ubuntu', + [Parameter(ValueFromPipeline)] + [string]$platforms = "${env:PLATFORMS}", + [Parameter(ValueFromPipeline)] + [string]$build_tag = "${env:BUILD_TAG}", + [Parameter(ValueFromPipeline)] + [string]$build_tag_version = "${env:BUILD_TAG_VERSION}", + [Parameter(ValueFromPipeline)] + [string]$build_ref = "${env:BUILD_REF}", + [Parameter(ValueFromPipeline)] + [string]$from_image = "${env:FROM_IMAGE}", + [Parameter(ValueFromPipeline)] + [string]$from_tag = "${env:FROM_TAG}", + [Parameter(ValueFromPipeline)] + [switch]$push ) -& (Get-Command 'docker').source @( +$arguments = @( 'buildx', - 'build', - '--progress=plain', - "--tag=ghcr.io/${slug}:${tag}", - "--tag=quay.io/${slug}:${tag}", - "--tag=docker.io/${slug}:${tag}", + 'build' +) + +$arguments += $push -eq $True ? @("--push") : @() + +$arguments += $progress -ne 'plain' ? @("--progress=$progress") : @("--progress=plain") + +$tags.Count -ne 0 ? ($tags | ForEach-Object { $arguments += @("--tag=$_") }) : "" + +$arguments += $tag -ne '' ? @("--tag=$tag") : @() + +$arguments += @( "--build-arg=NODE_VERSION=${node}", "--build-arg=DISTRO=${distro}", "--build-arg=TYPE=${type}", + "--build-arg=RUNNER=${runner}", + "--build-arg=BUILD_DATE=$((Get-Date).ToString('u'))", + "--build-arg=BUILD_OWNER=${owner}", + "--build-arg=BUILD_OWNER_MAIL=${owner}", + "--build-arg=BUILD_REPO=${repository}", "--build-arg=BUILD_TAG=${build_tag}", "--build-arg=BUILD_TAG_VERSION=${build_tag_version}", "--build-arg=BUILD_REF=${build_ref}", @@ -34,3 +68,5 @@ param( "--platform=${platforms}", '.' ) + +& (Get-Command 'docker').source $arguments diff --git a/linux/ubuntu/Dockerfile b/linux/ubuntu/Dockerfile index aaa79b8..af785ed 100644 --- a/linux/ubuntu/Dockerfile +++ b/linux/ubuntu/Dockerfile @@ -10,29 +10,36 @@ ARG FROM_IMAGE ARG FROM_TAG # > Our custom ARGs -ARG NODE_VERSION=14 +ARG NODE_VERSION="12 16" ARG DISTRO=ubuntu ARG TYPE=act -ARG RUNNER=root +ARG RUNNER # > Force apt to not be interactive/not ask ENV DEBIAN_FRONTEND=noninteractive -SHELL [ "/bin/bash", "--noprofile", "--norc", "-e", "-o", "pipefail", "-c" ] +SHELL [ "/bin/bash", "--login", "-e", "-o", "pipefail", "-c" ] +WORKDIR /tmp COPY ./linux/${DISTRO}/scripts /imagegeneration/installers RUN /imagegeneration/installers/${TYPE}.sh -ARG BUILD_TAG -ARG BUILD_REF="master" -ARG BUILD_TAG_VERSION="dev" +ARG BUILD_DATE +ARG BUILD_TAG=${TYPE} +ARG BUILD_REF +ARG BUILD_TAG_VERSION +ARG BUILD_OWNER +ARG BUILD_REPO -LABEL org.opencontainers.image.vendor="catthehacker" -LABEL org.opencontainers.image.authors="me@hackerc.at" -LABEL org.opencontainers.image.url="https://github.com/catthehacker/docker_images/tree/${BUILD_REF}/linux/${DISTRO}/${TYPE}/" -LABEL org.opencontainers.image.source="https://github.com/catthehacker/docker_images" -LABEL org.opencontainers.image.version=${BUILD_TAG_VERSION} -LABEL org.opencontainers.image.title=${BUILD_TAG}-${TARGETARCH} -LABEL org.opencontainers.image.revision=${BUILD_REF} +LABEL org.opencontainers.image.created="${BUILD_DATE}" +LABEL org.opencontainers.image.vendor="${BUILD_OWNER}" +LABEL org.opencontainers.image.authors="https://github.com/${BUILD_OWNER}" +LABEL org.opencontainers.image.url="https://github.com/${BUILD_OWNER}/${BUILD_REPO}/tree/${BUILD_REF}/linux/${DISTRO}/${TYPE}/" +LABEL org.opencontainers.image.source="https://github.com/${BUILD_OWNER}/${BUILD_REPO}" +LABEL org.opencontainers.image.documentation="https://github.com/${BUILD_OWNER}/${BUILD_REPO}" +LABEL org.opencontainers.image.version="${BUILD_TAG_VERSION}" +LABEL org.opencontainers.image.title="${BUILD_TAG}-${TARGETARCH}" +LABEL org.opencontainers.image.description="Special image built for using with https://github.com/nektos/act" +LABEL org.opencontainers.image.revision="${BUILD_REF}" USER ${RUNNER} diff --git a/linux/ubuntu/scripts/act.sh b/linux/ubuntu/scripts/act.sh index b296f7e..2fb7327 100755 --- a/linux/ubuntu/scripts/act.sh +++ b/linux/ubuntu/scripts/act.sh @@ -7,21 +7,21 @@ set -Eeuxo pipefail printf "\n\t๐Ÿ‹ Build started ๐Ÿ‹\t\n" +# Remove '"' so it can be sourced by sh/bash sed 's|"||g' -i "/etc/environment" -echo "USER=$(whoami)" | tee -a "/etc/environment" -echo "RUNNER_USER=$(whoami)" | tee -a "/etc/environment" - ImageOS=ubuntu$(echo "${FROM_TAG}" | cut -d'.' -f 1) -echo "IMAGE_OS=$ImageOS" | tee -a "/etc/environment" -echo "ImageOS=$ImageOS" | tee -a "/etc/environment" -echo "LSB_RELEASE=${FROM_TAG}" | tee -a "/etc/environment" - AGENT_TOOLSDIRECTORY=/opt/hostedtoolcache -echo "AGENT_TOOLSDIRECTORY=${AGENT_TOOLSDIRECTORY}" | tee -a "/etc/environment" -echo "RUN_TOOL_CACHE=${AGENT_TOOLSDIRECTORY}" | tee -a "/etc/environment" -echo "DEPLOYMENT_BASEPATH=/opt/runner" | tee -a "/etc/environment" -echo ". /etc/environment" | tee -a /etc/profile +{ + echo "IMAGE_OS=$ImageOS" + echo "ImageOS=$ImageOS" + echo "LSB_RELEASE=${FROM_TAG}" + echo "AGENT_TOOLSDIRECTORY=${AGENT_TOOLSDIRECTORY}" + echo "RUN_TOOL_CACHE=${AGENT_TOOLSDIRECTORY}" + echo "DEPLOYMENT_BASEPATH=/opt/runner" + echo "USER=$(whoami)" + echo "RUNNER_USER=$(whoami)" +} | tee -a "/etc/environment" mkdir -m 0777 -p "${AGENT_TOOLSDIRECTORY}" chown -R 1001:1000 "${AGENT_TOOLSDIRECTORY}" @@ -36,6 +36,7 @@ packages=( gawk curl git + jq wget sudo gnupg-agent @@ -50,7 +51,7 @@ packages=( ) apt-get -yq update -apt-get -yq install --no-install-recommends "${packages[@]}" +apt-get -yq install --no-install-recommends --no-install-suggests "${packages[@]}" ln -s "$(which python3)" "/usr/local/bin/python" @@ -58,53 +59,65 @@ LSB_OS_VERSION=$(lsb_release -rs | sed 's|\.||g') echo "LSB_OS_VERSION=${LSB_OS_VERSION}" | tee -a "/etc/environment" wget -qO "/imagegeneration/toolset.json" "https://raw.githubusercontent.com/actions/virtual-environments/main/images/linux/toolsets/toolset-${LSB_OS_VERSION}.json" +wget -qO "/imagegeneration/LICENSE" "https://raw.githubusercontent.com/actions/virtual-environments/main/LICENSE" -wget -qO "/usr/bin/jq" "https://github.com/stedolan/jq/releases/download/jq-1.6/jq-linux64" -chmod +x "/usr/bin/jq" +ARCH=$(uname -m) +if [ "$ARCH" = x86_64 ]; then + ARCH=x64 + wget -qO "/usr/bin/jq" "https://github.com/stedolan/jq/releases/download/jq-1.6/jq-linux64" + chmod +x "/usr/bin/jq" +fi +if [ "$ARCH" = aarch64 ]; then ARCH=arm64; fi if [[ "${FROM_TAG}" == "16.04" ]]; then printf 'git-lfs not available for Xenial' else - apt-get -yq install --no-install-recommends git-lfs + apt-get -yq install --no-install-recommends --no-install-suggests git-lfs fi printf "\n\t๐Ÿ‹ Updated apt lists and upgraded packages ๐Ÿ‹\t\n" printf "\n\t๐Ÿ‹ Creating ~/.ssh and adding 'github.com' ๐Ÿ‹\t\n" mkdir -m 0700 -p ~/.ssh -ssh-keyscan -t rsa github.com >>/etc/ssh/ssh_known_hosts -ssh-keyscan -t rsa ssh.dev.azure.com >>/etc/ssh/ssh_known_hosts +{ + ssh-keyscan -t rsa github.com + ssh-keyscan -t rsa ssh.dev.azure.com +} >>/etc/ssh/ssh_known_hosts printf "\n\t๐Ÿ‹ Installed base utils ๐Ÿ‹\t\n" printf "\n\t๐Ÿ‹ Installing docker cli ๐Ÿ‹\t\n" -curl -sSL https://packages.microsoft.com/keys/microsoft.asc | sudo apt-key add - -apt-add-repository "https://packages.microsoft.com/ubuntu/${FROM_TAG}/prod" +curl "https://packages.microsoft.com/config/ubuntu/${FROM_TAG}/prod.list" | tee /etc/apt/sources.list.d/microsoft-prod.list +wget -q https://packages.microsoft.com/keys/microsoft.asc +gpg --dearmor /etc/apt/trusted.gpg.d/microsoft.gpg +apt-key add - /etc/apt/apt.conf.d/80-retries + +# Configure apt to always assume Y +echo 'APT::Get::Assume-Yes "true";' >/etc/apt/apt.conf.d/90assumeyes + +apt-get update +apt-get install apt-utils + +# Install apt-fast using quick-install.sh +# https://github.com/ilikenwf/apt-fast +bash -c "$(curl -sL https://raw.githubusercontent.com/ilikenwf/apt-fast/master/quick-install.sh)" + +# echo 'session required pam_limits.so' >>/etc/pam.d/common-session +# echo 'session required pam_limits.so' >>/etc/pam.d/common-session-noninteractive +# echo 'DefaultLimitNOFILE=65536' >>/etc/systemd/system.conf +# echo 'DefaultLimitSTACK=16M:infinity' >>/etc/systemd/system.conf + +# { +# # Raise Number of File Descriptors +# echo '* soft nofile 65536' +# echo '* hard nofile 65536' + +# # Double stack size from default 8192KB +# echo '* soft stack 16384' +# echo '* hard stack 16384' +# } >>/etc/security/limits.conf + +scripts=( + basic + pwsh + java-tools + go + js + rust + vcpkg +) + +for SCRIPT in "${scripts[@]}"; do + printf "\n\t๐Ÿงจ Executing %s.sh ๐Ÿงจ\t\n" "${SCRIPT}" + "/imagegeneration/installers/${SCRIPT}.sh" +done + +printf "\n\t๐Ÿ‹ Cleaning image ๐Ÿ‹\t\n" +apt-get clean +rm -rf /var/cache/* /var/log/* /var/lib/apt/lists/* /tmp/* || echo 'Failed to delete directories' +printf "\n\t๐Ÿ‹ Cleaned up image ๐Ÿ‹\t\n" diff --git a/linux/ubuntu/scripts/gh.sh b/linux/ubuntu/scripts/gh.sh new file mode 100755 index 0000000..00069bf --- /dev/null +++ b/linux/ubuntu/scripts/gh.sh @@ -0,0 +1,12 @@ +#!/bin/bash -e +################################################################################ +## File: github-cli.sh +## Desc: Installs GitHub CLI +## Must be run as non-root user after homebrew +################################################################################ +# source: https://github.com/actions/virtual-environments/blob/be27ebfdb31aece2c90fbe1984c1749cbd1b464c/images/linux/scripts/installers/github-cli.sh + +# Install GitHub CLI +url=$(curl -s https://api.github.com/repos/cli/cli/releases/latest | jq -r ".assets[].browser_download_url|select(contains(\"linux\") and contains(\"$(arch)\") and contains(\".deb\"))") +wget -q "$url" -O "/tmp/gh.deb" +apt install /tmp/gh.deb diff --git a/linux/ubuntu/scripts/go.sh b/linux/ubuntu/scripts/go.sh index 7018f32..751ce52 100755 --- a/linux/ubuntu/scripts/go.sh +++ b/linux/ubuntu/scripts/go.sh @@ -1,15 +1,9 @@ #!/bin/bash -# disable warning about 'mkdir -m -p' -# shellcheck disable=SC2174 - -# source environment because Linux is beautiful and not really confusing like Windows, also you are apparently not supposed to source that file because it's not conforming to standard shell format but we already fix that in base image -# yes, this is sarcasm # shellcheck disable=SC1091 . /etc/environment -# no -x because big json -set -Eeuo pipefail +set -Eeuxo pipefail printf "\n\t๐Ÿ‹ Installing Go(lang) ๐Ÿ‹\t\n" @@ -20,6 +14,7 @@ for V in $(jq -r '.toolcache[] | select(.name == "go") | .versions[]' "/imagegen VER=$(echo "${JSON}" | jq "[.[] | select(.version|test(\"^${V}\"))][0].version" -r) GOPATH="$AGENT_TOOLSDIRECTORY/go/${VER}/x64" + # shellcheck disable=SC2174 mkdir -v -m 0777 -p "$GOPATH" ARCH=$(uname -m) if [ "$ARCH" = x86_64 ]; then ARCH=amd64; fi diff --git a/linux/ubuntu/scripts/helpers/install.sh b/linux/ubuntu/scripts/helpers/install.sh new file mode 100755 index 0000000..87d04ac --- /dev/null +++ b/linux/ubuntu/scripts/helpers/install.sh @@ -0,0 +1,69 @@ +#!/bin/bash -e +################################################################################ +## File: install.sh +## Desc: Helper functions for installing tools +################################################################################ + +download_with_retries() { + # Due to restrictions of bash functions, positional arguments are used here. + # In case if you using latest argument NAME, you should also set value to all previous parameters. + # Example: download_with_retries $ANDROID_SDK_URL "." "android_sdk.zip" + local URL="$1" + local DEST="${2:-.}" + local NAME="${3:-${URL##*/}}" + local COMPRESSED="${4:-}" + + if [[ $COMPRESSED == "compressed" ]]; then + local COMMAND="curl $URL -4 -sL --compressed -o '$DEST/$NAME' -w '%{http_code}'" + else + local COMMAND="curl $URL -4 -sL -o '$DEST/$NAME' -w '%{http_code}'" + fi + + echo "Downloading '$URL' to '${DEST}/${NAME}'..." + retries=20 + interval=30 + while [ $retries -gt 0 ]; do + ((retries--)) + # Temporary disable exit on error to retry on non-zero exit code + set +e + http_code=$(eval "$COMMAND") + exit_code=$? + if [ "$http_code" -eq 200 ] && [ $exit_code -eq 0 ]; then + echo "Download completed" + return 0 + else + echo "Error โ€” Either HTTP response code for '$URL' is wrong - '$http_code' or exit code is not 0 - '$exit_code'. Waiting $interval seconds before the next attempt, $retries attempts left" + sleep 30 + fi + # Enable exit on error back + set -e + done + + echo "Could not download $URL" + return 1 +} + +## Use dpkg to figure out if a package has already been installed +## Example use: +## if ! IsPackageInstalled packageName; then +## echo "packageName is not installed!" +## fi +IsPackageInstalled() { + dpkg -S "$1" &>/dev/null +} + +verlte() { + sortedVersion=$(echo -e "$1\n$2" | sort -V | head -n1) + [ "$1" = "$sortedVersion" ] +} + +get_toolset_path() { + echo "/imagegeneration/toolset.json" +} + +get_toolset_value() { + local toolset_path + toolset_path=$(get_toolset_path) + local query=$1 + jq -r "$query" "$toolset_path" +} diff --git a/linux/ubuntu/scripts/helpers/os.sh b/linux/ubuntu/scripts/helpers/os.sh new file mode 100755 index 0000000..f7be471 --- /dev/null +++ b/linux/ubuntu/scripts/helpers/os.sh @@ -0,0 +1,17 @@ +#!/bin/bash -e +################################################################################ +## File: install-helpers.sh +## Desc: Helper functions for installing tools +################################################################################ + +function isUbuntu18() { + lsb_release -d | grep -q 'Ubuntu 18' +} + +function isUbuntu20() { + lsb_release -d | grep -q 'Ubuntu 20' +} + +function getOSVersionLabel() { + lsb_release -cs +} diff --git a/linux/ubuntu/scripts/java-tools.sh b/linux/ubuntu/scripts/java-tools.sh new file mode 100755 index 0000000..82425b7 --- /dev/null +++ b/linux/ubuntu/scripts/java-tools.sh @@ -0,0 +1,83 @@ +#!/bin/bash -e +################################################################################ +## File: java-tools.sh +## Desc: Installs Java and related tooling (Ant, Gradle, Maven) +################################################################################ + +set -Eeuxo pipefail + +# shellcheck disable=SC1091 +. /etc/environment +# shellcheck disable=SC1091 +. /imagegeneration/installers/helpers/os.sh +# shellcheck disable=SC1091 +. /imagegeneration/installers/helpers/install.sh + +JAVA_VERSIONS_LIST=$(get_toolset_value '.java.versions | .[]') +DEFAULT_JDK_VERSION=$(get_toolset_value '.java.default') +JAVA_TOOLCACHE_PATH="$AGENT_TOOLSDIRECTORY/Java_Adopt_jdk" + +# Install GPG Key for Adopt Open JDK. See https://adoptopenjdk.net/installation.html +wget -qO - "https://adoptopenjdk.jfrog.io/adoptopenjdk/api/gpg/key/public" | apt-key add - +add-apt-repository --yes https://adoptopenjdk.jfrog.io/adoptopenjdk/deb/ + +if isUbuntu18; then + # Install GPG Key for Azul Open JDK. See https://www.azul.com/downloads/azure-only/zulu/ + apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 0xB1998361219BD9C9 + apt-add-repository "deb https://repos.azul.com/azure-only/zulu/apt stable main" +fi + +apt-get update + +# shellcheck disable=SC2068 +for JAVA_VERSION in ${JAVA_VERSIONS_LIST[@]}; do + apt-get -y install adoptopenjdk-"$JAVA_VERSION"-hotspot=\* + javaVersionPath="/usr/lib/jvm/adoptopenjdk-${JAVA_VERSION}-hotspot-amd64" + echo "JAVA_HOME_${JAVA_VERSION}_X64=$javaVersionPath" | tee -a /etc/environment + fullJavaVersion=$(grep "^SEMANTIC" <"$javaVersionPath/release" | cut -d "=" -f 2 | tr -d "\"" | tr "+" "-") + + # If there is no semver in java release, then extract java version from -fullversion + if [[ -z $fullJavaVersion ]]; then + fullJavaVersion=$(java -fullversion 2>&1 | tr -d "\"" | tr "+" "-" | awk '{print $4}') + fi + + javaToolcacheVersionPath="$JAVA_TOOLCACHE_PATH/$fullJavaVersion" + mkdir -p "$javaToolcacheVersionPath" + + # Create a complete file + touch "$javaToolcacheVersionPath/x64.complete" + + # Create symlink for Java + ln -s "$javaVersionPath" "$javaToolcacheVersionPath/x64" +done + +# Set Default Java version +update-java-alternatives -s /usr/lib/jvm/adoptopenjdk-"${DEFAULT_JDK_VERSION}"-hotspot-amd64 + +echo "JAVA_HOME=/usr/lib/jvm/adoptopenjdk-${DEFAULT_JDK_VERSION}-hotspot-amd64" | tee -a /etc/environment + +# add extra permissions to be able execute command without sudo +chmod -R 777 /usr/lib/jvm +# Install Ant +apt-fast install -y --no-install-recommends ant ant-optional +echo "ANT_HOME=/usr/share/ant" | tee -a /etc/environment + +# Install Maven +mavenVersion=$(get_toolset_value '.java.maven') +mavenDownloadUrl="https://www-eu.apache.org/dist/maven/maven-3/${mavenVersion}/binaries/apache-maven-${mavenVersion}-bin.zip" +download_with_retries "${mavenDownloadUrl}" "/tmp" "maven.zip" +unzip -qq -d /usr/share /tmp/maven.zip +ln -s /usr/share/apache-maven-"${mavenVersion}"/bin/mvn /usr/bin/mvn + +# Install Gradle +# This script founds the latest gradle release from https://services.gradle.org/versions/all +# The release is downloaded, extracted, a symlink is created that points to it, and GRADLE_HOME is set. +gradleJson=$(curl -s https://services.gradle.org/versions/all) +gradleLatestVersion=$(echo "$gradleJson" | jq -r '.[] | select(.version | contains("-") | not).version' | sort -V | tail -n1) +gradleDownloadUrl=$(echo "$gradleJson" | jq -r ".[] | select(.version==\"$gradleLatestVersion\") | .downloadUrl") +echo "gradleUrl=$gradleDownloadUrl" +echo "gradleVersion=$gradleLatestVersion" +download_with_retries "$gradleDownloadUrl" "/tmp" "gradleLatest.zip" +unzip -qq -d /usr/share /tmp/gradleLatest.zip +ln -s /usr/share/gradle-"${gradleLatestVersion}"/bin/gradle /usr/bin/gradle +echo "GRADLE_HOME=$(find /usr/share -depth -maxdepth 1 -name "gradle*")" | tee -a /etc/environment diff --git a/linux/ubuntu/scripts/js.sh b/linux/ubuntu/scripts/js.sh index 1b869ae..0605ad3 100755 --- a/linux/ubuntu/scripts/js.sh +++ b/linux/ubuntu/scripts/js.sh @@ -2,17 +2,28 @@ set -Eeuo pipefail -# source environment because Linux is beautiful and not really confusing like Windows -# also you are apparently not supposed to source that file because it's not conforming to standard shell envvar -# format but we already fix that in base image -# yes, this is sarcasm # shellcheck disable=SC1091 . /etc/environment -printf "\n\t๐Ÿ‹ Installed NPM ๐Ÿ‹\t\n" -npm -v +printf "\n\t๐Ÿ‹ Installing NVM tools ๐Ÿ‹\t\n" +VERSION=$(curl -s https://api.github.com/repos/nvm-sh/nvm/releases/latest | jq -r '.tag_name') +curl -o- "https://raw.githubusercontent.com/nvm-sh/nvm/$VERSION/install.sh" | bash +export NVM_DIR=$HOME/.nvm +echo "NVM_DIR=$HOME/.nvm" | tee -a /etc/environment -versions=("10" "12") +# Expressions don't expand in single quotes, use double quotes for that.shellcheck(SC2016) +# shellcheck disable=SC2016 +echo '[ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh" # This loads nvm' | tee -a /etc/skel/.bash_profile + +# Not following: ./nvm.sh was not specified as input (see shellcheck -x).shellcheck(SC1091) +# shellcheck disable=SC1091 +[ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh" + +printf "\n\t๐Ÿ‹ Installed NVM ๐Ÿ‹\t\n" +nvm --version + +# node 12 and 16 are installed already in act-* +versions=("14") JSON=$(wget -qO- https://nodejs.org/download/release/index.json | jq --compact-output) for V in "${versions[@]}"; do @@ -51,23 +62,6 @@ pnpm -v printf "\n\t๐Ÿ‹ Installed YARN ๐Ÿ‹\t\n" yarn -v -printf "\n\t๐Ÿ‹ Installing NVM tools ๐Ÿ‹\t\n" -VERSION=$(curl -s https://api.github.com/repos/nvm-sh/nvm/releases/latest | jq -r '.tag_name') -curl -o- "https://raw.githubusercontent.com/nvm-sh/nvm/$VERSION/install.sh" | bash -export NVM_DIR=$HOME/.nvm -echo "NVM_DIR=$HOME/.nvm" | tee -a /etc/environment - -# Expressions don't expand in single quotes, use double quotes for that.shellcheck(SC2016) -# shellcheck disable=SC2016 -echo '[ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh" # This loads nvm' | tee -a /etc/skel/.bash_profile - -# Not following: ./nvm.sh was not specified as input (see shellcheck -x).shellcheck(SC1091) -# shellcheck disable=SC1091 -[ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh" - -printf "\n\t๐Ÿ‹ Installed NVM ๐Ÿ‹\t\n" -nvm --version - printf "\n\t๐Ÿ‹ Cleaning image ๐Ÿ‹\t\n" apt-get clean rm -rf /var/cache/* /var/log/* /var/lib/apt/lists/* /tmp/* || echo 'Failed to delete directories' diff --git a/linux/ubuntu/scripts/pwsh.sh b/linux/ubuntu/scripts/pwsh.sh index d408108..ce23fd5 100755 --- a/linux/ubuntu/scripts/pwsh.sh +++ b/linux/ubuntu/scripts/pwsh.sh @@ -2,6 +2,9 @@ set -Eeuxo pipefail +# shellcheck disable=SC1091 +. /etc/environment + printf "\n\t๐Ÿ‹ Installing PowerShell ๐Ÿ‹\t\n" # While an linux/amd64 platform installation can use apt-get, the linux/arm64 @@ -12,10 +15,12 @@ printf "\n\t๐Ÿ‹ Installing PowerShell ๐Ÿ‹\t\n" # described here: # https://docs.microsoft.com/en-us/powershell/scripting/install/installing-powershell-core-on-linux?view=powershell-7.1#linux # +# TODO: think of a more robust installation for multiple architectures ARCH=$(uname -m) if [ "$ARCH" = x86_64 ]; then ARCH=x64; fi if [ "$ARCH" = aarch64 ]; then ARCH=arm64; fi +if [ "$ARCH" = armv7l ]; then ARCH=arm32; fi VER=$(curl --silent "https://api.github.com/repos/PowerShell/PowerShell/releases/latest" | jq -r .tag_name) curl -L -o /tmp/powershell.tar.gz "https://github.com/PowerShell/PowerShell/releases/download/$VER/powershell-${VER:1}-linux-$ARCH.tar.gz" sudo mkdir -p "/opt/microsoft/powershell/${VER:1:1}" @@ -33,6 +38,6 @@ modules=("MarkdownPS" "Pester" "PSScriptAnalyzer") pwsh -nol -nop -c "Set-PSRepository -Name PSGallery -InstallationPolicy Trusted" for mod in "${modules[@]}"; do - printf "\n\t๐Ÿ‹ Installing %s ๐Ÿ‹\t\n" "${mod}" - pwsh -nol -nop -c "Install-Module -Name ${mod} -Scope AllUsers -SkipPublisherCheck -Force" + printf "\n\t๐Ÿงจ Installing %s ๐Ÿงจ\t\n" "${mod}" + pwsh -nol -nop -c "\$ProgressPreference = \"SilentlyContinue\" ; Install-Module -Name ${mod} -Scope AllUsers -SkipPublisherCheck -Force" done diff --git a/linux/ubuntu/scripts/runner.sh b/linux/ubuntu/scripts/runner.sh index 5b823b4..a764be6 100755 --- a/linux/ubuntu/scripts/runner.sh +++ b/linux/ubuntu/scripts/runner.sh @@ -10,8 +10,12 @@ groupadd -g 1001 "${RUNNER}" groupadd -g 1000 "${RUNNER}admin" useradd -u 1001 -g "${RUNNER}" -G sudo -m -s /bin/bash "${RUNNER}" useradd -u 1000 -g "${RUNNER}admin" -G sudo -m -s /bin/bash "${RUNNER}admin" -echo "${RUNNER} ALL=(ALL) NOPASSWD: ALL" >>/etc/sudoers -echo "${RUNNER}admin ALL=(ALL) NOPASSWD: ALL" >>/etc/sudoers +usermod -aG docker "runner" +usermod -aG docker "runneradmin" +{ + echo "${RUNNER} ALL=(ALL) NOPASSWD: ALL" + echo "${RUNNER}admin ALL=(ALL) NOPASSWD: ALL" +} | tee -a /etc/sudoers printf "\n\t๐Ÿ‹ Runner user ๐Ÿ‹\t\n" su - "${RUNNER}" -c id @@ -31,8 +35,10 @@ mkdir -p "/home/${RUNNER}/work/_temp" chown -R "${RUNNER}":"${RUNNER}" "/home/${RUNNER}/work" mkdir -m 0700 -p "/home/${RUNNER}/.ssh" -ssh-keyscan -t rsa github.com >>"/home/${RUNNER}/.ssh/known_hosts" -ssh-keyscan -t rsa ssh.dev.azure.com >>"/home/${RUNNER}/.ssh/known_hosts" +{ + ssh-keyscan -t rsa github.com + ssh-keyscan -t rsa ssh.dev.azure.com +} | tee -a "/home/${RUNNER}/.ssh/known_hosts" chmod 644 "/home/${RUNNER}/.ssh/known_hosts" chown -R "${RUNNER}":"${RUNNER}" "/home/${RUNNER}/.ssh" diff --git a/linux/ubuntu/scripts/rust.sh b/linux/ubuntu/scripts/rust.sh index 00ea26e..e3b7ed8 100755 --- a/linux/ubuntu/scripts/rust.sh +++ b/linux/ubuntu/scripts/rust.sh @@ -2,8 +2,6 @@ set -Eeuxo pipefail -# source environment because Linux is beautiful and not really confusing like Windows, also you are apparently not supposed to source that file because it's not conforming to standard shell format but we already fix that in base image -# yes, this is sarcasm # shellcheck disable=SC1091 . /etc/environment @@ -32,8 +30,10 @@ sed "s|PATH=|PATH=${CARGO_HOME}/bin:|g" -i /etc/environment cd /root ln -sf "${CARGO_HOME}" .cargo ln -sf "${RUSTUP_HOME}" .rustup -echo "RUSTUP_HOME=${RUSTUP_HOME}" >>/etc/environment -echo "CARGO_HOME=${CARGO_HOME}" >>/etc/environment +{ + echo "RUSTUP_HOME=${RUSTUP_HOME}" + echo "CARGO_HOME=${CARGO_HOME}" +} | tee -a /etc/environment printf "\n\t๐Ÿ‹ Installed RUSTUP ๐Ÿ‹\t\n" rustup -V diff --git a/linux/ubuntu/scripts/vcpkg.sh b/linux/ubuntu/scripts/vcpkg.sh new file mode 100755 index 0000000..fae7888 --- /dev/null +++ b/linux/ubuntu/scripts/vcpkg.sh @@ -0,0 +1,18 @@ +#!/bin/bash -e +################################################################################ +## File: vcpkg.sh +## Desc: Installs vcpkg +################################################################################ +# source: https://github.com/actions/virtual-environments/blob/206a8183190e81d3266084457e619553551c1252/images/linux/scripts/installers/vcpkg.sh + +# Set env variable for vcpkg +VCPKG_INSTALLATION_ROOT=/usr/local/share/vcpkg +echo "VCPKG_INSTALLATION_ROOT=${VCPKG_INSTALLATION_ROOT}" | tee -a /etc/environment + +# Install vcpkg +git clone https://github.com/Microsoft/vcpkg $VCPKG_INSTALLATION_ROOT + +$VCPKG_INSTALLATION_ROOT/bootstrap-vcpkg.sh +$VCPKG_INSTALLATION_ROOT/vcpkg integrate install +chmod 0777 -R $VCPKG_INSTALLATION_ROOT +ln -sf $VCPKG_INSTALLATION_ROOT/vcpkg /usr/local/bin