From ec09e6e85fb6c5b8c3472612e924161d8612ec2e Mon Sep 17 00:00:00 2001 From: Robin van Boven <497556+Beanow@users.noreply.github.com> Date: Sun, 19 Nov 2023 19:59:49 +0100 Subject: [PATCH] feat: Add yq to act (#117) * feat: update install script * fix: revert toolset path Since there's a lot of hard-coded references to this path. * feat: Adds yq script Unlike upstream we detect the arch suffix here. * feat: include yq in act and custom flavors * fix: include armhf in supported arch for act.yml * fix: PR feedback, don't need to install twice for custom --- linux/ubuntu/scripts/act.sh | 24 +++ linux/ubuntu/scripts/custom.sh | 2 +- linux/ubuntu/scripts/helpers/install.sh | 223 +++++++++++++++++++----- linux/ubuntu/scripts/yq.sh | 29 +++ 4 files changed, 237 insertions(+), 41 deletions(-) create mode 100755 linux/ubuntu/scripts/yq.sh diff --git a/linux/ubuntu/scripts/act.sh b/linux/ubuntu/scripts/act.sh index d771585..5537228 100755 --- a/linux/ubuntu/scripts/act.sh +++ b/linux/ubuntu/scripts/act.sh @@ -138,6 +138,30 @@ for ver in "${NODE[@]}"; do "${NODEPATH}"/bin/npm -v done +case "$(uname -m)" in + 'aarch64') + scripts=( + yq + ) + ;; + 'x86_64') + scripts=( + yq + ) + ;; + 'armv7l') + scripts=( + yq + ) + ;; + *) exit 1 ;; +esac + +for SCRIPT in "${scripts[@]}"; do + printf "\n\t๐Ÿงจ Executing %s.sh ๐Ÿงจ\t\n" "${SCRIPT}" + "/imagegeneration/installers/${SCRIPT}.sh" +done + printf "\n\t๐Ÿ‹ Cleaning image ๐Ÿ‹\t\n" apt-get clean rm -rf /var/cache/* /var/log/* /var/lib/apt/lists/* /tmp/* || echo 'Failed to delete directories' diff --git a/linux/ubuntu/scripts/custom.sh b/linux/ubuntu/scripts/custom.sh index 9d4f988..61a95bc 100755 --- a/linux/ubuntu/scripts/custom.sh +++ b/linux/ubuntu/scripts/custom.sh @@ -43,7 +43,7 @@ case "$(uname -m)" in dotnet ) ;; - 'x86_64') + 'x86_64') scripts=( basic gh diff --git a/linux/ubuntu/scripts/helpers/install.sh b/linux/ubuntu/scripts/helpers/install.sh index 87d04ac..7c29918 100755 --- a/linux/ubuntu/scripts/helpers/install.sh +++ b/linux/ubuntu/scripts/helpers/install.sh @@ -3,44 +3,52 @@ ## File: install.sh ## Desc: Helper functions for installing tools ################################################################################ +# source: https://github.com/actions/runner-images/blob/5d6938f680075d63fa71f8aa70990866cd12884b/images/linux/scripts/helpers/install.sh download_with_retries() { - # Due to restrictions of bash functions, positional arguments are used here. - # In case if you using latest argument NAME, you should also set value to all previous parameters. - # Example: download_with_retries $ANDROID_SDK_URL "." "android_sdk.zip" - local URL="$1" - local DEST="${2:-.}" - local NAME="${3:-${URL##*/}}" - local COMPRESSED="${4:-}" +# Due to restrictions of bash functions, positional arguments are used here. +# In case if you using latest argument NAME, you should also set value to all previous parameters. +# Example: download_with_retries $ANDROID_SDK_URL "." "android_sdk.zip" + local URL="$1" + local DEST="${2:-.}" + local NAME="${3:-${URL##*/}}" + local COMPRESSED="$4" - if [[ $COMPRESSED == "compressed" ]]; then - local COMMAND="curl $URL -4 -sL --compressed -o '$DEST/$NAME' -w '%{http_code}'" - else - local COMMAND="curl $URL -4 -sL -o '$DEST/$NAME' -w '%{http_code}'" - fi - - echo "Downloading '$URL' to '${DEST}/${NAME}'..." - retries=20 - interval=30 - while [ $retries -gt 0 ]; do - ((retries--)) - # Temporary disable exit on error to retry on non-zero exit code - set +e - http_code=$(eval "$COMMAND") - exit_code=$? - if [ "$http_code" -eq 200 ] && [ $exit_code -eq 0 ]; then - echo "Download completed" - return 0 + if [[ $COMPRESSED == "compressed" ]]; then + local COMMAND="curl $URL -4 -sL --compressed -o '$DEST/$NAME' -w '%{http_code}'" else - echo "Error โ€” Either HTTP response code for '$URL' is wrong - '$http_code' or exit code is not 0 - '$exit_code'. Waiting $interval seconds before the next attempt, $retries attempts left" - sleep 30 + local COMMAND="curl $URL -4 -sL -o '$DEST/$NAME' -w '%{http_code}'" fi - # Enable exit on error back - set -e - done - echo "Could not download $URL" - return 1 + # Save current errexit state and disable it to prevent unexpected exit on error + if echo $SHELLOPTS | grep '\(^\|:\)errexit\(:\|$\)' > /dev/null; + then + local ERR_EXIT_ENABLED=true + else + local ERR_EXIT_ENABLED=false + fi + set +e + + echo "Downloading '$URL' to '${DEST}/${NAME}'..." + retries=20 + interval=30 + while [ $retries -gt 0 ]; do + ((retries--)) + test "$ERR_EXIT_ENABLED" = true && set +e + http_code=$(eval $COMMAND) + exit_code=$? + test "$ERR_EXIT_ENABLED" = true && set -e + if [ $http_code -eq 200 ] && [ $exit_code -eq 0 ]; then + echo "Download completed" + return 0 + else + echo "Error โ€” Either HTTP response code for '$URL' is wrong - '$http_code' or exit code is not 0 - '$exit_code'. Waiting $interval seconds before the next attempt, $retries attempts left" + sleep $interval + fi + done + + echo "Could not download $URL" + return 1 } ## Use dpkg to figure out if a package has already been installed @@ -49,21 +57,156 @@ download_with_retries() { ## echo "packageName is not installed!" ## fi IsPackageInstalled() { - dpkg -S "$1" &>/dev/null + dpkg -S $1 &> /dev/null } verlte() { - sortedVersion=$(echo -e "$1\n$2" | sort -V | head -n1) - [ "$1" = "$sortedVersion" ] + sortedVersion=$(echo -e "$1\n$2" | sort -V | head -n1) + [ "$1" = "$sortedVersion" ] } get_toolset_path() { - echo "/imagegeneration/toolset.json" + echo "/imagegeneration/toolset.json" } get_toolset_value() { - local toolset_path - toolset_path=$(get_toolset_path) - local query=$1 - jq -r "$query" "$toolset_path" + local toolset_path=$(get_toolset_path) + local query=$1 + echo "$(jq -r "$query" $toolset_path)" +} + +get_github_package_download_url() { + local REPO_ORG=$1 + local FILTER=$2 + local VERSION=$3 + local SEARCH_IN_COUNT="100" + + json=$(curl -fsSL "https://api.github.com/repos/${REPO_ORG}/releases?per_page=${SEARCH_IN_COUNT}") + + if [ -n "$VERSION" ]; then + tagName=$(echo $json | jq -r '.[] | select(.prerelease==false).tag_name' | sort --unique --version-sort | egrep -v ".*-[a-z]|beta" | egrep "\w*${VERSION}" | tail -1) + else + tagName=$(echo $json | jq -r '.[] | select((.prerelease==false) and (.assets | length > 0)).tag_name' | sort --unique --version-sort | egrep -v ".*-[a-z]|beta" | tail -1) + fi + + downloadUrl=$(echo $json | jq -r ".[] | select(.tag_name==\"${tagName}\").assets[].browser_download_url | select(${FILTER})" | head -n 1) + if [ -z "$downloadUrl" ]; then + echo "Failed to parse a download url for the '${tagName}' tag using '${FILTER}' filter" + exit 1 + fi + echo $downloadUrl +} + +get_github_package_hash() { + local repo_owner=$1 + local repo_name=$2 + local file_name=$3 + local url=$4 + local version=${5:-"latest"} + local prerelease=${6:-false} + local delimiter=${7:-'|'} + local word_number=${8:-2} + + if [[ -z "$file_name" ]]; then + echo "File name is not specified." + exit 1 + fi + + if [[ -n "$url" ]]; then + release_url="$url" + else + if [ "$version" == "latest" ]; then + release_url="https://api.github.com/repos/${repo_owner}/${repo_name}/releases/latest" + else + json=$(curl -fsSL "https://api.github.com/repos/${repo_owner}/${repo_name}/releases?per_page=100") + tags=$(echo "$json" | jq -r --arg prerelease "$prerelease" '.[] | select(.prerelease == ($prerelease | test("true"; "i"))) | .tag_name') + tag=$(echo "$tags" | grep -o "$version") + if [[ "$(echo "$tag" | wc -l)" -gt 1 ]]; then + echo "Multiple tags found matching the version $version. Please specify a more specific version." + exit 1 + fi + if [[ -z "$tag" ]]; then + echo "Failed to get a tag name for version $version." + exit 1 + fi + release_url="https://api.github.com/repos/${repo_owner}/${repo_name}/releases/tags/$tag" + fi + fi + + body=$(curl -fsSL "$release_url" | jq -r '.body' | tr -d '`') + matching_line=$(echo "$body" | grep "$file_name") + if [[ "$(echo "$matching_line" | wc -l)" -gt 1 ]]; then + echo "Multiple lines found included the file $file_name. Please specify a more specific file name." + exit 1 + fi + if [[ -z "$matching_line" ]]; then + echo "File name '$file_name' not found in release body." + exit 1 + fi + + result=$(echo "$matching_line" | cut -d "$delimiter" -f "$word_number" | tr -d -c '[:alnum:]') + if [[ -z "$result" ]]; then + echo "Empty result. Check parameters delimiter and/or word_number for the matching line." + exit 1 + fi + + echo "$result" +} + +get_hash_from_remote_file() { + local url=$1 + local keywords=("$2" "$3") + local delimiter=${4:-' '} + local word_number=${5:-1} + + if [[ -z "${keywords[0]}" || -z "$url" ]]; then + echo "File name and/or URL is not specified." + exit 1 + fi + + matching_line=$(curl -fsSL "$url" | sed 's/ */ /g' | tr -d '`') + for keyword in "${keywords[@]}"; do + matching_line=$(echo "$matching_line" | grep "$keyword") + done + + if [[ "$(echo "$matching_line" | wc -l)" -gt 1 ]]; then + echo "Multiple lines found including the words: ${keywords[*]}. Please use a more specific filter." + exit 1 + fi + + if [[ -z "$matching_line" ]]; then + echo "Keywords (${keywords[*]}) not found in the file with hashes." + exit 1 + fi + + result=$(echo "$matching_line" | cut -d "$delimiter" -f "$word_number" | tr -d -c '[:alnum:]') + if [[ ${#result} -ne 64 && ${#result} -ne 128 ]]; then + echo "Invalid result length. Expected 64 or 128 characters. Please check delimiter and/or word_number parameters." + echo "Result: $result" + exit 1 + fi + + echo "$result" +} + +use_checksum_comparison() { + local file_path=$1 + local checksum=$2 + local sha_type=${3:-"256"} + + echo "Performing checksum verification" + + if [[ ! -f "$file_path" ]]; then + echo "File not found: $file_path" + exit 1 + fi + + local_file_hash=$(shasum --algorithm "$sha_type" "$file_path" | awk '{print $1}') + + if [[ "$local_file_hash" != "$checksum" ]]; then + echo "Checksum verification failed. Expected hash: $checksum; Actual hash: $local_file_hash." + exit 1 + else + echo "Checksum verification passed" + fi } diff --git a/linux/ubuntu/scripts/yq.sh b/linux/ubuntu/scripts/yq.sh new file mode 100755 index 0000000..00a8584 --- /dev/null +++ b/linux/ubuntu/scripts/yq.sh @@ -0,0 +1,29 @@ +#!/bin/bash -e +################################################################################ +## File: yq.sh +## Desc: Installs YQ +## Supply chain security: YQ - checksum validation +################################################################################ +# source: https://github.com/actions/runner-images/blob/5d6938f680075d63fa71f8aa70990866cd12884b/images/linux/scripts/installers/yq.sh + +# Source the helpers for use with the script +. /imagegeneration/installers/helpers/install.sh + +yq_arch() { + case "$(uname -m)" in + 'aarch64') echo 'arm64' ;; + 'x86_64') echo 'amd64' ;; + 'armv7l') echo 'arm' ;; + *) exit 1 ;; + esac +} + +# Download YQ +base_url="https://github.com/mikefarah/yq/releases/latest/download" +filename="yq_linux_$(yq_arch)" +download_with_retries "${base_url}/${filename}" "/tmp" "yq" +# Supply chain security - YQ +external_hash=$(get_hash_from_remote_file "${base_url}/checksums" "${filename} " "" " " "19") +use_checksum_comparison "/tmp/yq" "${external_hash}" +# Install YQ +sudo install /tmp/yq /usr/bin/yq