Update bootstrappers

build.ps1

@@ -1,67 +1,186 @@
-$ErrorActionPreference = "Stop"
+#!/usr/bin/env powershell
+#requires -version 4
 
-function DownloadWithRetry([string] $url, [string] $downloadLocation, [int] $retries)
+<#
+.SYNOPSIS
+Build this repository
+
+.DESCRIPTION
+Downloads korebuild if required. Then builds the repository.
+
+.PARAMETER Path
+The folder to build. Defaults to the folder containing this script.
+
+.PARAMETER Channel
+The channel of KoreBuild to download. Overrides the value from the config file.
+
+.PARAMETER DotNetHome
+The directory where .NET Core tools will be stored.
+
+.PARAMETER ToolsSource
+The base url where build tools can be downloaded. Overrides the value from the config file.
+
+.PARAMETER Update
+Updates KoreBuild to the latest version even if a lock file is present.
+
+.PARAMETER ConfigFile
+The path to the configuration file that stores values. Defaults to version.props.
+
+.PARAMETER MSBuildArgs
+Arguments to be passed to MSBuild
+
+.NOTES
+This function will create a file $PSScriptRoot/korebuild-lock.txt. This lock file can be committed to source, but does not have to be.
+When the lockfile is not present, KoreBuild will create one using latest available version from $Channel.
+
+The $ConfigFile is expected to be an JSON file. It is optional, and the configuration values in it are optional as well. Any options set
+in the file are overridden by command line parameters.
+
+.EXAMPLE
+Example config file:
+```json
 {
-    while($true)
+  "$schema": "https://raw.githubusercontent.com/aspnet/BuildTools/dev/tools/korebuild.schema.json",
-    {
+  "channel": "dev",
-        try
+  "toolsSource": "https://aspnetcore.blob.core.windows.net/buildtools"
-        {
-            Invoke-WebRequest $url -OutFile $downloadLocation
-            break
-        }
-        catch
-        {
-            $exceptionMessage = $_.Exception.Message
-            Write-Host "Failed to download '$url': $exceptionMessage"
-            if ($retries -gt 0) {
-                $retries--
-                Write-Host "Waiting 10 seconds before retrying. Retries left: $retries"
-                Start-Sleep -Seconds 10
-
-            }
-            else
-            {
-                $exception = $_.Exception
-                throw $exception
-            }
-        }
-    }
 }
+```
+#>
+[CmdletBinding(PositionalBinding = $false)]
+param(
+    [string]$Path = $PSScriptRoot,
+    [Alias('c')]
+    [string]$Channel,
+    [Alias('d')]
+    [string]$DotNetHome,
+    [Alias('s')]
+    [string]$ToolsSource,
+    [Alias('u')]
+    [switch]$Update,
+    [string]$ConfigFile = $null,
+    [Parameter(ValueFromRemainingArguments = $true)]
+    [string[]]$MSBuildArgs
+)
 
-cd $PSScriptRoot
+Set-StrictMode -Version 2
+$ErrorActionPreference = 'Stop'
 
-$repoFolder = $PSScriptRoot
+#
-$env:REPO_FOLDER = $repoFolder
+# Functions
+#
 
-$koreBuildZip="https://github.com/aspnet/KoreBuild/archive/rel/2.0.0.zip"
+function Get-KoreBuild {
-if ($env:KOREBUILD_ZIP)
-{
-    $koreBuildZip=$env:KOREBUILD_ZIP
-}
 
-$buildFolder = ".build"
+    $lockFile = Join-Path $Path 'korebuild-lock.txt'
-$buildFile="$buildFolder\KoreBuild.ps1"
 
-if (!(Test-Path $buildFolder)) {
+    if (!(Test-Path $lockFile) -or $Update) {
-    Write-Host "Downloading KoreBuild from $koreBuildZip"
+        Get-RemoteFile "$ToolsSource/korebuild/channels/$Channel/latest.txt" $lockFile
+    }
 
-    $tempFolder=$env:TEMP + "\KoreBuild-" + [guid]::NewGuid()
+    $version = Get-Content $lockFile | Where-Object { $_ -like 'version:*' } | Select-Object -first 1
-    New-Item -Path "$tempFolder" -Type directory | Out-Null
+    if (!$version) {
+        Write-Error "Failed to parse version from $lockFile. Expected a line that begins with 'version:'"
+    }
+    $version = $version.TrimStart('version:').Trim()
+    $korebuildPath = Join-Paths $DotNetHome ('buildtools', 'korebuild', $version)
 
-    $localZipFile="$tempFolder\korebuild.zip"
+    if (!(Test-Path $korebuildPath)) {
-
+        Write-Host -ForegroundColor Magenta "Downloading KoreBuild $version"
-    DownloadWithRetry -url $koreBuildZip -downloadLocation $localZipFile -retries 6
+        New-Item -ItemType Directory -Path $korebuildPath | Out-Null
+        $remotePath = "$ToolsSource/korebuild/artifacts/$version/korebuild.$version.zip"
 
+        try {
+            $tmpfile = Join-Path ([IO.Path]::GetTempPath()) "KoreBuild-$([guid]::NewGuid()).zip"
+            Get-RemoteFile $remotePath $tmpfile
+            if (Get-Command -Name 'Expand-Archive' -ErrorAction Ignore) {
+                # Use built-in commands where possible as they are cross-plat compatible
+                Expand-Archive -Path $tmpfile -DestinationPath $korebuildPath
+            }
+            else {
+                # Fallback to old approach for old installations of PowerShell
                 Add-Type -AssemblyName System.IO.Compression.FileSystem
-    [System.IO.Compression.ZipFile]::ExtractToDirectory($localZipFile, $tempFolder)
+                [System.IO.Compression.ZipFile]::ExtractToDirectory($tmpfile, $korebuildPath)
+            }
+        }
+        catch {
+            remove-item -Recurse -Force $korebuildPath -ErrorAction Ignore
+            throw
+        }
+        finally {
+            remove-item $tmpfile -ErrorAction Ignore
+        }
+    }
 
-    New-Item -Path "$buildFolder" -Type directory | Out-Null
+    return $korebuildPath
-    copy-item "$tempFolder\**\build\*" $buildFolder -Recurse
+}
 
-    # Cleanup
+function Join-Paths([string]$path, [string[]]$childPaths) {
-    if (Test-Path $tempFolder) {
+    $childPaths | ForEach-Object { $path = Join-Path $path $_ }
-        Remove-Item -Recurse -Force $tempFolder
+    return $path
+}
+
+function Get-RemoteFile([string]$RemotePath, [string]$LocalPath) {
+    if ($RemotePath -notlike 'http*') {
+        Copy-Item $RemotePath $LocalPath
+        return
+    }
+
+    $retries = 10
+    while ($retries -gt 0) {
+        $retries -= 1
+        try {
+            Invoke-WebRequest -UseBasicParsing -Uri $RemotePath -OutFile $LocalPath
+            return
+        }
+        catch {
+            Write-Verbose "Request failed. $retries retries remaining"
+        }
+    }
+
+    Write-Error "Download failed: '$RemotePath'."
+}
+
+#
+# Main
+#
+
+# Load configuration or set defaults
+
+$Path = Resolve-Path $Path
+if (!$ConfigFile) { $ConfigFile = Join-Path $Path 'korebuild.json' }
+
+if (Test-Path $ConfigFile) {
+    try {
+        $config = Get-Content -Raw -Encoding UTF8 -Path $ConfigFile | ConvertFrom-Json
+        if ($config) {
+            if (!($Channel) -and (Get-Member -Name 'channel' -InputObject $config)) { [string] $Channel = $config.channel }
+            if (!($ToolsSource) -and (Get-Member -Name 'toolsSource' -InputObject $config)) { [string] $ToolsSource = $config.toolsSource}
+        }
+    } catch {
+        Write-Warning "$ConfigFile could not be read. Its settings will be ignored."
+        Write-Warning $Error[0]
     }
 }
 
-&"$buildFile" @args
+if (!$DotNetHome) {
+    $DotNetHome = if ($env:DOTNET_HOME) { $env:DOTNET_HOME } `
+        elseif ($env:USERPROFILE) { Join-Path $env:USERPROFILE '.dotnet'} `
+        elseif ($env:HOME) {Join-Path $env:HOME '.dotnet'}`
+        else { Join-Path $PSScriptRoot '.dotnet'}
+}
+
+if (!$Channel) { $Channel = 'dev' }
+if (!$ToolsSource) { $ToolsSource = 'https://aspnetcore.blob.core.windows.net/buildtools' }
+
+# Execute
+
+$korebuildPath = Get-KoreBuild
+Import-Module -Force -Scope Local (Join-Path $korebuildPath 'KoreBuild.psd1')
+
+try {
+    Install-Tools $ToolsSource $DotNetHome
+    Invoke-RepositoryBuild $Path @MSBuildArgs
+}
+finally {
+    Remove-Module 'KoreBuild' -ErrorAction Ignore
+}

build.sh

@@ -1,46 +1,216 @@
 #!/usr/bin/env bash
-repoFolder="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
-cd $repoFolder
 
-koreBuildZip="https://github.com/aspnet/KoreBuild/archive/rel/2.0.0.zip"
+set -euo pipefail
-if [ ! -z $KOREBUILD_ZIP ]; then
-    koreBuildZip=$KOREBUILD_ZIP
-fi
 
-buildFolder=".build"
+#
-buildFile="$buildFolder/KoreBuild.sh"
+# variables
+#
 
-if test ! -d $buildFolder; then
+RESET="\033[0m"
-    echo "Downloading KoreBuild from $koreBuildZip"
+RED="\033[0;31m"
+YELLOW="\033[0;33m"
+MAGENTA="\033[0;95m"
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+[ -z "${DOTNET_HOME:-}"] && DOTNET_HOME="$HOME/.dotnet"
+config_file="$DIR/korebuild.json"
+verbose=false
+update=false
+repo_path="$DIR"
+channel=''
+tools_source=''
 
-    tempFolder="/tmp/KoreBuild-$(uuidgen)"
+#
-    mkdir $tempFolder
+# Functions
+#
+__usage() {
+    echo "Usage: $(basename ${BASH_SOURCE[0]}) [options] [[--] <MSBUILD_ARG>...]"
+    echo ""
+    echo "Arguments:"
+    echo "    <MSBUILD_ARG>...         Arguments passed to MSBuild. Variable number of arguments allowed."
+    echo ""
+    echo "Options:"
+    echo "    --verbose                Show verbose output."
+    echo "    -c|--channel <CHANNEL>   The channel of KoreBuild to download. Overrides the value from the config file.."
+    echo "    --config-file <FILE>     The path to the configuration file that stores values. Defaults to korebuild.json."
+    echo "    -d|--dotnet-home <DIR>   The directory where .NET Core tools will be stored. Defaults to '\$DOTNET_HOME' or '\$HOME/.dotnet."
+    echo "    --path <PATH>            The directory to build. Defaults to the directory containing the script."
+    echo "    -s|--tools-source <URL>  The base url where build tools can be downloaded. Overrides the value from the config file."
+    echo "    -u|--update              Update to the latest KoreBuild even if the lock file is present."
+    echo ""
+    echo "Description:"
+    echo "    This function will create a file \$DIR/korebuild-lock.txt. This lock file can be committed to source, but does not have to be."
+    echo "    When the lockfile is not present, KoreBuild will create one using latest available version from \$channel."
 
-    localZipFile="$tempFolder/korebuild.zip"
+    if [[ "${1:-}" != '--no-exit' ]]; then
+        exit 2
+    fi
+}
 
-    retries=6
+get_korebuild() {
-    until (wget -O $localZipFile $koreBuildZip 2>/dev/null || curl -o $localZipFile --location $koreBuildZip 2>/dev/null)
+    local lock_file="$repo_path/korebuild-lock.txt"
-    do
+    if [ ! -f $lock_file ] || [ "$update" = true ]; then
-        echo "Failed to download '$koreBuildZip'"
+        __get_remote_file "$tools_source/korebuild/channels/$channel/latest.txt" $lock_file
-        if [ "$retries" -le 0 ]; then
+    fi
+    local version="$(grep 'version:*' -m 1 $lock_file)"
+    if [[ "$version" == '' ]]; then
+        __error "Failed to parse version from $lock_file. Expected a line that begins with 'version:'"
+        return 1
+    fi
+    version="$(echo ${version#version:} | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//')"
+    local korebuild_path="$DOTNET_HOME/buildtools/korebuild/$version"
+
+    {
+        if [ ! -d "$korebuild_path" ]; then
+            mkdir -p "$korebuild_path"
+            local remote_path="$tools_source/korebuild/artifacts/$version/korebuild.$version.zip"
+            tmpfile="$(mktemp)"
+            echo -e "${MAGENTA}Downloading KoreBuild ${version}${RESET}"
+            if __get_remote_file $remote_path $tmpfile; then
+                unzip -q -d "$korebuild_path" $tmpfile
+            fi
+            rm $tmpfile || true
+        fi
+
+        source "$korebuild_path/KoreBuild.sh"
+    } || {
+        if [ -d "$korebuild_path" ]; then
+            echo "Cleaning up after failed installation"
+            rm -rf "$korebuild_path" || true
+        fi
+        return 1
+    }
+}
+
+__error() {
+    echo -e "${RED}error: $*${RESET}" 1>&2
+}
+
+__warn() {
+    echo -e "${YELLOW}warning: $*${RESET}"
+}
+
+__machine_has() {
+    hash "$1" > /dev/null 2>&1
+    return $?
+}
+
+__get_remote_file() {
+    local remote_path=$1
+    local local_path=$2
+
+    if [[ "$remote_path" != 'http'* ]]; then
+        cp $remote_path $local_path
+        return 0
+    fi
+
+    failed=false
+    if __machine_has curl ; then
+        curl --retry 10 -sSL -f --create-dirs -o $local_path $remote_path || failed=true
+    elif __machine_has wget; then
+        wget --tries 10 -O $local_path $remote_path || failed=true
+    else
+        failed=true
+    fi
+
+    if [ "$failed" = true ]; then
+        __error "Download failed: $remote_path" 1>&2
+        return 1
+    fi
+}
+
+#
+# main
+#
+
+while [[ $# > 0 ]]; do
+    case $1 in
+        -\?|-h|--help)
+            __usage --no-exit
+            exit 0
+            ;;
+        -c|--channel|-Channel)
+            shift
+            channel=${1:-}
+            [ -z "$channel" ] && __usage
+            ;;
+        --config-file|-ConfigFile)
+            shift
+            config_file="${1:-}"
+            [ -z "$config_file" ] && __usage
+            if [ ! -f "$config_file" ]; then
+                __error "Invalid value for --config-file. $config_file does not exist."
                 exit 1
             fi
-        retries=$((retries - 1))
+            ;;
-        echo "Waiting 10 seconds before retrying. Retries left: $retries"
+        -d|--dotnet-home|-DotNetHome)
-        sleep 10s
+            shift
-    done
+            DOTNET_HOME=${1:-}
+            [ -z "$DOTNET_HOME" ] && __usage
+            ;;
+        --path|-Path)
+            shift
+            repo_path="${1:-}"
+            [ -z "$repo_path" ] && __usage
+            ;;
+        -s|--tools-source|-ToolsSource)
+            shift
+            tools_source="${1:-}"
+            [ -z "$tools_source" ] && __usage
+            ;;
+        -u|--update|-Update)
+            update=true
+            ;;
+        --verbose|-Verbose)
+            verbose=true
+            ;;
+        --)
+            shift
+            break
+            ;;
+        *)
+            break
+            ;;
+    esac
+    shift
+done
 
-    unzip -q -d $tempFolder $localZipFile
+if ! __machine_has unzip; then
-
+    __error 'Missing required command: unzip'
-    mkdir $buildFolder
+    exit 1
-    cp -r $tempFolder/**/build/** $buildFolder
-
-    chmod +x $buildFile
-
-    # Cleanup
-    if test -d $tempFolder; then
-        rm -rf $tempFolder
-    fi
 fi
 
-$buildFile -r $repoFolder "$@"
+if ! __machine_has curl && ! __machine_has wget; then
+    __error 'Missing required command. Either wget or curl is required.'
+    exit 1
+fi
+
+[ -z "${config_file:-}" ] && config_file="$repo_path/korebuild.json"
+if [ -f "$config_file" ]; then
+    if __machine_has jq ; then
+        if jq '.' "$config_file" >/dev/null ; then
+            config_channel="$(jq -r 'select(.channel!=null) | .channel' "$config_file")"
+            config_tools_source="$(jq -r 'select(.toolsSource!=null) | .toolsSource' "$config_file")"
+        else
+            __warn "$config_file is invalid JSON. Its settings will be ignored."
+        fi
+    elif __machine_has python ; then
+        if python -c "import json,codecs;obj=json.load(codecs.open('$config_file', 'r', 'utf-8-sig'))" >/dev/null ; then
+            config_channel="$(python -c "import json,codecs;obj=json.load(codecs.open('$config_file', 'r', 'utf-8-sig'));print(obj['channel'] if 'channel' in obj else '')")"
+            config_tools_source="$(python -c "import json,codecs;obj=json.load(codecs.open('$config_file', 'r', 'utf-8-sig'));print(obj['toolsSource'] if 'toolsSource' in obj else '')")"
+        else
+            __warn "$config_file is invalid JSON. Its settings will be ignored."
+        fi
+    else
+        __warn 'Missing required command: jq or pyton. Could not parse the JSON file. Its settings will be ignored.'
+    fi
+
+    [ ! -z "${config_channel:-}" ] && channel="$config_channel"
+    [ ! -z "${config_tools_source:-}" ] && tools_source="$config_tools_source"
+fi
+
+[ -z "$channel" ] && channel='dev'
+[ -z "$tools_source" ] && tools_source='https://aspnetcore.blob.core.windows.net/buildtools'
+
+get_korebuild
+install_tools "$tools_source" "$DOTNET_HOME"
+invoke_repository_build "$repo_path" $@