~ knotless Search/
↑↓ move↵ open⇧↵ all results

johnandersen777.bsky.social/socialweb-computer-console

console.socialweb.computer: kcp, kubectl and ssh compiled to wasm in a browser tab with a terminal; a kcp fork

0
0
main
651 lines · 18.8 KB Raw History
#!/bin/sh

# SPDX-FileCopyrightText: 2025 Christoph Mewes, https://codeberg.org/xrstf/uget
# SPDX-License-Identifier: MIT
#
# µget 0.4.0 – your friendly downloader
# -------------------------------------
#
# µget can download software as binaries, archives or Go modules.
#
# Usage: ./uget.sh URL_PATTERN BINARY_NAME VERSION [EXTRACT_PATTERN=**/$BINARY_NAME]

set -eu

###############################################################################
# Configuration

# µget supports a large range of environment variables to customize its
# behaviour. Global configuration settings are prefixed with UGET_, settings
# usually meant for a single tool only have no prefix (like GO_MODULE).

# GO_MODULE can be set to true to create a dummy Go module, add the given
# URL and version as a dependency and then go build the desired binary. Note
# that Go modules do not use checksums by default, see $UGET_GO_CHECKSUMS.
GO_MODULE=${GO_MODULE:-false}

# UNCOMPRESSED can be set to true if the downloaded file is not an archive,
# but the binary itself and doesn't need decompressing.
UNCOMPRESSED=${UNCOMPRESSED:-false}

# UGET_UPDATE can be set to true when the VERSION parameter of a program has
# been updated and you want µget to update all known variants (based on the
# checksums file) before installing the correct variant for the current system.
# When UGET_CHECKSUMS is not in use, this variable has no effect.
# Use UGET_UPDATE_ONLY if you want to just update the checksums without also
# installing the binary.
UGET_UPDATE=${UGET_UPDATE:-false}

# UGET_UPDATE_ONLY is like UGET_UPDATE, but does not install the binary on
# the current system (i.e. it only touches the checksum file).
UGET_UPDATE_ONLY=${UGET_UPDATE_ONLY:-false}

# UGET_DIRECTORY is where downloaded binaries will be placed.
UGET_DIRECTORY="${UGET_DIRECTORY:-_tools}"

# UGET_CHECKSUMS is an optional path to a checksum file that µget should use to
# ensure subsequent downloads match previously known checksums to prevent
# tampering on the server side.
UGET_CHECKSUMS="${UGET_CHECKSUMS:-}"

# UGET_VERSIONED_BINARIES can be set to true to append the binary's version to
# its filename, leading to final paths like "_tools/mytool-v1.2.3". This can
# help when µget is being used in Makefiles to improve staleness detection.
# Note that µget never deletes any installed binaries, so enabling this can
# lead to leftover binaries that you can cleanup at your own convenience.
UGET_VERSIONED_BINARIES=${UGET_VERSIONED_BINARIES:-false}

# UGET_TEMPDIR is the root directory to use when creating new temporary dirs.
UGET_TEMPDIR="${UGET_TEMPDIR:-/tmp}"

# UGET_CACHE is an optional path to a directory where binaries are stored/read
# from instead of being downloaded from the internet. This can be useful if you
# have many projects using µget and do not want to re-download the same binary
# for all of them. This does not apply for Go modules since those are already
# effectively cached by Go itself.
UGET_CACHE="${UGET_CACHE:-}"

# UGET_PRINT_PATH can be set to "relative" to make µget only omit log output
# and only print the relative path to the binary, and set to "absolute" to
# output the absolute path.
UGET_PRINT_PATH="${UGET_PRINT_PATH:-no}"

# UGET_HASHFUNC is the hashing function used to calculate file checksums. The
# output of this program is processed with awk to only print the first column.
UGET_HASHFUNC="${UGET_HASHFUNC:-sha256sum}"

# UGET_GO_BUILD_CMD overwrites the default call to "go build" when installing
# a Go module. Use this to inject custom Go flags or toolchains.
# The given command is called with "-o BINARY_NAME MODULE_URL" with the pwd
# being inside a fake module that depends on the given module.
UGET_GO_BUILD_CMD="${UGET_GO_BUILD_CMD:-go build}"

# UGET_GO_CHECKSUMS can be set to true to force checksums even for Go modules.
# This is disabled by default because the exact binaries being built depend on
# a lot of factors and usually it's a hassle to ensure everyone in a project
# has the *exact* same build environment. Go modules already make use of Google's
# GOSUMDB and should be "safe enough" by default with µget checksums.
UGET_GO_CHECKSUMS=${UGET_GO_CHECKSUMS:-false}

###############################################################################
# Function library

uget_mktemp() {
  set -e
  # --tmpdir does not work on MacOS
  mktemp -d -p "$ABS_UGET_TEMPDIR"
}

uget_log() {
  if [ "$UGET_PRINT_PATH" = "no" ]; then
    echo "$@" >&2
  fi
}

uget_error() {
  echo "$@" >&2
}

uget_lowercase() {
  set -e
  cat | tr '[:upper:]' '[:lower:]'
}

uget_checksum_enabled() {
  set -e
  [ -n "$UGET_CHECKSUMS" ]
}

uget_cache_enabled() {
  set -e
  [ -n "$UGET_CACHE" ]
}

uget_checksum_check() {
  set -e

  local kvString="$1"
  local downloadedBinary="$2"

  if ! uget_checksum_enabled; then return; fi

  local newChecksum
  newChecksum="$(uget_checksum_calculate "$downloadedBinary")"

  local oldChecksum
  oldChecksum="$(uget_checksum_read "$kvString")"

  if [ -n "$oldChecksum" ] && [ "$oldChecksum" != "$newChecksum" ]; then
    uget_error
    uget_error "  *************************************************************************"
    uget_error "  SECURITY ERROR"
    uget_error
    uget_error "  The downloaded file $downloadedBinary does not have the expected checksum."
    uget_error
    uget_error "  Expected: $oldChecksum"
    uget_error "  Actual  : $newChecksum"
    uget_error
    uget_error "  If you are updating $IDENTIFIER, this error is expected."
    uget_error "  Re-run this command with the environment variable UGET_UPDATE=true to make"
    uget_error "  µget update the checksums for all known variants of $IDENTIFIER."
    uget_error "  Use UGET_UPDATE_ONLY=true if you want to just update the checksums and not"
    uget_error "  install the given binary on this machine."
    uget_error "  *************************************************************************"
    uget_error

    return 1
  fi

  if [ -z "$oldChecksum" ]; then
    uget_checksum_write "$kvString" "$newChecksum"
  fi
}

uget_checksum_read() {
  set -e

  local kvString="$1"

  if [ -f "$UGET_CHECKSUMS" ]; then
    awk -F'|' -v "binary=$IDENTIFIER" -v "kv=$kvString" '{ if ($1 == binary && $2 == kv) print $3 }' "$UGET_CHECKSUMS"
  fi
}

uget_checksum_calculate() {
  set -e
  "$UGET_HASHFUNC" "$1" | awk '{ print $1 }'
}

# hash_string includes a trailing newline in the hashed data
# (because of echo), but since *all* hashes contain it, it
# doesn't matter.
uget_hash_string() {
  set -e
  echo "$1" | "$UGET_HASHFUNC" | awk '{ print $1 }'
}

uget_checksum_write() {
  set -e

  local kvString="$1"
  local checksum="$2"

  if [ -f "$UGET_CHECKSUMS" ]; then
    local tempDir
    tempDir="$(uget_mktemp)"

    # use awk to drop any existing hash for this binary/keyvalue combo
    # (for better readability, do not invert the condition here);
    # checking for NF (number of fields) to drop empty lines
    awk \
      -F'|' -v "binary=$IDENTIFIER" -v "kv=$kvString" \
      '{ if (NF == 0 || ($1 == binary && $2 == kv)) {} else print }' \
      "$UGET_CHECKSUMS" > "$tempDir/checksums.txt"

    # add our new checksum
    echo "$IDENTIFIER|$kvString|$checksum" >> "$tempDir/checksums.txt"

    # sort the file because it looks nicer and prevents ugly git diffs
    sort "$tempDir/checksums.txt" > "$UGET_CHECKSUMS"

    rm -rf -- "$tempDir"
  else
    # start a new file
    echo "$IDENTIFIER|$kvString|$checksum" > "$UGET_CHECKSUMS"
  fi
}

uget_url_placeholder() {
  set -e

  case "$1" in
    GOARCH) go env GOARCH ;;
    GOOS)   go env GOOS ;;
    UARCH)  uname -m | uget_lowercase ;;
    UOS)    uname -s | uget_lowercase ;;
    *)      uget_error "Unexpected placeholder $1."; return 1 ;;
  esac
}

# valueFromPair returns "foo" for "myvalue=foo;myothervalue=bar" when called with
# "myvalue" as the key.
uget_url_valueFromPair() {
  set -e

  local kvString="$1"
  local key="$2"

  # adding semicolons makes matching full keys easier
  echo ";$kvString;" | sed -E "s/.*;$key=([^;]+).*/\\1/"
}

uget_url_setKeyInPairs() {
  set -e

  local kvString="$1"
  local key="$2"
  local value="$3"

  # first take the existing pairs and turn it into a multiline string; then
  # awk out the existing pair for $key,
  # then add a new pair and sort it all together, strip empty lines in case
  # kvstring as empty, then join the multiple lines back into a single line
  # and drop the trailing ';'
  (
    echo "$kvString" | tr ';' "\n" | awk -F'=' -v "key=$key" '{ if ($1 != key) print }'
    echo "$key=$value"
  ) | sort | sed '/^[[:space:]]*$/d' | tr "\n" ';' | sed 's/;$//'
}

uget_url_findPlaceholders() {
  set -e

  # match all {...},
  # then sort and return only unique values (no need to replace the
  # same placeholder multiple times) (this is important to allow for consistent
  # matches when awk'ing through the checksum file),
  # then remove braces and
  # finally turn into a singleline string
  echo "$1" |
    grep -oE '\{[A-Z0-9_]+\}' |
    sort -u |
    tr -d '{}' |
    awk '{printf("%s ", $0)}'
}

# replaceLive() use live system-data to replace placeholders in the given pattern.
# It returns a string of form "KVSTRING|URL".
uget_url_replaceLive() {
  set -e

  local urlPattern="$1"
  local usedPlaceholders=""

  for placeholder in $(uget_url_findPlaceholders "$urlPattern"); do
    # version is treated specially
    [ "$placeholder" = "VERSION" ] && continue

    local replacement
    replacement="$(uget_url_placeholder "$placeholder")"
    urlPattern="$(echo "$urlPattern" | sed "s|{$placeholder}|$replacement|g")"

    # remember this placeholder and its value
    usedPlaceholders="$usedPlaceholders;$placeholder=$replacement"
  done

  # trim leading ";"
  usedPlaceholders="$(echo "$usedPlaceholders" | sed 's/^;//')"

  echo "$usedPlaceholders|$urlPattern"
}

# replaceWithArgs() does not ask the current system for the values when replacing
# a placeholder, but uses a given key-value pair string as the source. It also
# only returns the resulting string, since the used placeholders are known to
# the caller already.
uget_url_replaceWithArgs() {
  set -e

  local urlPattern="$1"
  local kvString="$2"

  for placeholder in $(uget_url_findPlaceholders "$urlPattern"); do
    # version is treated specially
    [ "$placeholder" = "VERSION" ] && continue

    local replacement
    replacement="$(uget_url_valueFromPair "$kvString" "$placeholder")"
    if [ -z "$replacement" ]; then
      uget_error "Found no replacement string for placeholder $placeholder."
      exit 1
    fi

    urlPattern="$(echo "$urlPattern" | sed "s|{$placeholder}|$replacement|g")"
  done

  echo "$urlPattern"
}

# returns "KVSTRING URL"
uget_url_build() {
  set -e

  local kvString="${1:-}"
  local pattern

  if [ -z "$kvString" ]; then
    local result
    result="$(uget_url_replaceLive "$URL_PATTERN")"
    kvString="$(echo "$result" | cut -d'|' -f1)"
    pattern="$(echo "$result" | cut -d'|' -f2)"
  else
    pattern="$(uget_url_replaceWithArgs "$URL_PATTERN" "$kvString")"
  fi

  pattern="$(echo "$pattern" | sed "s|{VERSION}|$VERSION|g")"

  echo "$kvString|$pattern"
}

# uget_http_download() uses either curl or wget to download the given URL into
# the current directory.
uget_http_download() {
  set -e

  local url="$1"

  if command -v curl >/dev/null 2>&1; then
    curl --fail -LO "$url"
  elif command -v wget >/dev/null 2>&1; then
    wget "$url"
  else
    uget_error "Neither curl nor wget are available."
    return 1
  fi
}

# uget_go_install creates a temporary Go module, adds the given URL as a
# dependency and then builds $BINARY.
uget_go_install() {
  set -e

  local destinationDir="$1"
  local kvString="$2"
  local url="$3"

  # make sure GOARCH and GOOS will be set correctly
  local os
  local arch
  os="$(uget_url_valueFromPair "$kvString" "GOOS")"
  arch="$(uget_url_valueFromPair "$kvString" "GOARCH")"

  # since we crosscompile, we cannot do "GOBIN=(somewhere) go install url@version",
  # because Go considers this to be dangerous behaviour:
  # https://github.com/golang/go/issues/57485
  # Instead we create a dummy module and use the desired program as a dependency,
  # *then* we're allowed to crosscompile it anywhere using "go build". Go figure.

  go mod init temp 2>/dev/null
  go get "$url@$VERSION"

  local tmpFilename="__tmp.bin"

  # go build command is meant to be expanded
  # shellcheck disable=SC2086
  GOFLAGS=-trimpath GOARCH="$arch" GOOS="$os" $UGET_GO_BUILD_CMD -o "$tmpFilename" "$url"

  mv "$tmpFilename" "$destinationDir/$BINARY"
}

# uget_extract_archive extracts a downloaded archive and moves the one interesting
# file out of it.
uget_extract_archive() {
  set -e

  local destinationDir="$1"
  local archive="$2"

  if ! $UNCOMPRESSED; then
    case "$archive" in
      *.tar.gz | *.tgz)
        tar xzf "$archive"
        ;;
      *.tar.bz2 | *.tbz2)
        tar xjf "$archive"
        ;;
      *.tar.xz | *.txz)
        tar xJf "$archive"
        ;;
      *.zip)
        unzip "$archive"
        ;;
      *)
        uget_error "Unknown file type: $archive"
        return 1
    esac
  fi

  # pattern is explicitly meant to be interpreted by the shell
  # shellcheck disable=SC2086
  mv $BINARY_PATTERN "$destinationDir/$BINARY"
  chmod +x "$destinationDir/$BINARY"
}

# uget_download performs the actual download or reads a binary
# from the configured cache (if any), and finally places it
# in a given directory.
uget_download() {
  set -e

  local destinationDir="$1"
  local kvString="$2"
  local url="$3"

  local startDir
  startDir="$(pwd)"

  local tempDir
  tempDir="$(uget_mktemp)"

  cd "$tempDir"

  if $GO_MODULE; then
    uget_go_install "$destinationDir" "$kvString" "$url"
  else
    local urlHash
    urlHash="$(uget_hash_string "$url")"

    cacheFile="$UGET_CACHE/$IDENTIFIER-$VERSION-$urlHash"

    if uget_cache_enabled && [ -f "$cacheFile" ]; then
      cp "$cacheFile" "$destinationDir/$BINARY"
      uget_log "Copied $BINARY from µget cache."
    else
      uget_http_download "$url"

      local archive
      archive="$(ls)"

      uget_extract_archive "$destinationDir" "$archive"

      if uget_cache_enabled; then
        cp "$destinationDir/$BINARY" "$cacheFile"
      fi
    fi
  fi

  cd "$startDir"
  rm -rf -- "$tempDir"
}

# ready() checks if the desired binary already exists in the desired version.
uget_ready() {
  local fullFinalPath="$ABS_UGET_DIRECTORY/$BINARY"
  if ! [ -f "$fullFinalPath" ]; then
    return 1
  fi

  # skip further checks if we're using versioned binaries
  if $UGET_VERSIONED_BINARIES; then
    return 0
  fi

  local versionFile="$fullFinalPath.version"
  [ -f "$versionFile" ] && [ "$VERSION" = "$(cat "$versionFile")" ]
}

# install() downloads the binary, checks the checksum and places it in UGET_DIRECTORY.
uget_install() {
  set -e

  local kvString="$1"
  local url="$2"
  local fullFinalPath="$ABS_UGET_DIRECTORY/$BINARY"
  local versionFile="$fullFinalPath.version"

  local startDir
  startDir="$(pwd)"

  local tempDir
  tempDir="$(uget_mktemp)"

  uget_log "Downloading $IDENTIFIER version $VERSION ..."
  uget_download "$tempDir" "$kvString" "$url"

  local fullTempBinary="$tempDir/$BINARY"
  uget_checksum_check "$kvString" "$fullTempBinary"

  # if everything is fine, place the binary in its final location
  mv "$fullTempBinary" "$fullFinalPath"

  if ! $UGET_VERSIONED_BINARIES; then
    echo "$VERSION" > "$versionFile"
  fi

  uget_log "Installed at $UGET_DIRECTORY/$BINARY."

  cd "$startDir"
  rm -rf -- "$tempDir"
}

# update() downloads the binary, updates the checksums and discards the binary.
uget_update() {
  set -e

  local kvString="$1"
  local url="$2"

  local startDir
  startDir="$(pwd)"

  local tempDir
  tempDir="$(uget_mktemp)"

  uget_log "  ~> $kvString"
  uget_download "$tempDir" "$kvString" "$url"

  local fullBinaryPath="$tempDir/$BINARY"

  local checksum
  checksum="$(uget_checksum_calculate "$fullBinaryPath")"

  uget_checksum_write "$kvString" "$checksum"

  cd "$startDir"
  rm -rf -- "$tempDir"
}

###############################################################################
# General Setup Logic

# get CLI flags
export URL_PATTERN="$1"
export IDENTIFIER="$2"
export VERSION="$3"
BINARY_PATTERN="${4:-**/$IDENTIFIER}"

export BINARY="$IDENTIFIER"
if $UGET_VERSIONED_BINARIES; then
  BINARY="$IDENTIFIER-$VERSION"
fi

# ensure target directory exists
mkdir -p "$UGET_DIRECTORY" "$UGET_TEMPDIR"

ABS_UGET_DIRECTORY="$(realpath "$UGET_DIRECTORY")"
ABS_UGET_TEMPDIR="$(realpath "$UGET_TEMPDIR")"

if uget_cache_enabled; then
  mkdir -p "$UGET_CACHE"
  UGET_CACHE="$(realpath "$UGET_CACHE")"
fi

if $GO_MODULE && ! $UGET_GO_CHECKSUMS; then
  UGET_CHECKSUMS=""
fi

if uget_checksum_enabled; then
  touch "$UGET_CHECKSUMS"
  UGET_CHECKSUMS="$(realpath "$UGET_CHECKSUMS")"
else
  if $GO_MODULE; then
    if $UGET_UPDATE || $UGET_UPDATE_ONLY; then
      uget_error "Checksums are disabled for Go modules, cannot update $IDENTIFIER checksums."
      # This is not an error because in complex Makefiles, there might be 3 binaries required
      # for one make target, and if one of them is a Go module and you have no direct way
      # to just update a single binary, then running "UGET_UPDATE make complex-task" would
      # fail at the Go module. It's simply more convenient to just warn and move on to the next
      # binary.
    fi
  fi

  UGET_UPDATE=false
  UGET_UPDATE_ONLY=false
fi

###############################################################################
# Main application logic

# When in update-only mode, we do not download the binary for the current system
# only, but instead for all known variants based on the checksums file,
# recalculate the checksums and then discard the temporary binaries. Otherwise
# after updating the checksums we continue with the regular install logic.

if $UGET_UPDATE || $UGET_UPDATE_ONLY; then
  uget_log "Updating checksums for $IDENTIFIER ..."

  # Find and process all known variants...
  awk -F'|' -v "binary=$IDENTIFIER" '{ if ($1 == binary) print $2 }' "$UGET_CHECKSUMS" | while IFS= read -r kvString; do
    result="$(uget_url_build "$kvString")"
    url="$(echo "$result" | cut -d'|' -f2)"

    # download binary into tempdir, update checksums, but then delete it again
    uget_update "$kvString" "$url"
  done

  uget_log "All checksums were updated."
fi

if ! $UGET_UPDATE_ONLY; then
  if ! uget_ready; then
    # Replace placeholders in the URL with system-specific data, like arch or OS
    result="$(uget_url_build)"
    kvString="$(echo "$result" | cut -d'|' -f1)"
    url="$(echo "$result" | cut -d'|' -f2)"

    # Go modules usually do not have arch/os infos in their module names, but
    # the resulting binary still depends on the local system. To support checksums
    # for different machines, we always assume GOARCH and GOOS are "used placeholders".
    if $GO_MODULE; then
      kvString="$(uget_url_setKeyInPairs "$kvString" "GOARCH" "$(uget_url_placeholder GOARCH)")"
      kvString="$(uget_url_setKeyInPairs "$kvString" "GOOS" "$(uget_url_placeholder GOOS)")"
    fi

    uget_install "$kvString" "$url"
  fi

  case "$UGET_PRINT_PATH" in
    absolute) echo "$ABS_UGET_DIRECTORY/$BINARY" ;;
    relative) echo "$UGET_DIRECTORY/$BINARY" ;;
  esac
fi