From 35dcb815e217c1135012ba81616496c3ad10b3f5 Mon Sep 17 00:00:00 2001 From: davidovski Date: Thu, 17 Feb 2022 02:15:19 +0000 Subject: added file installing --- src/get.sh | 134 ++++++++++++++++++++++++++++++++++++++++++-------------- src/install.sh | 62 ++++++++++++++++++++++++++ src/profile.sh | 16 +++++-- src/sync.sh | 36 +++++---------- src/util.sh | 78 +++++++++++++++++++++++++++++++++ src/validate.sh | 25 +++++++++++ src/xi.sh | 39 ++++++++++++++--- 7 files changed, 326 insertions(+), 64 deletions(-) create mode 100644 src/install.sh create mode 100644 src/util.sh create mode 100644 src/validate.sh diff --git a/src/get.sh b/src/get.sh index 3e0be79..3da2591 100755 --- a/src/get.sh +++ b/src/get.sh @@ -22,10 +22,12 @@ resolve_deps () { local package=${to_check[-1]} unset to_check[-1] - deps+=($package) + #only add if not already added + echo ${deps[*]} | grep -q "\b$dep\b" || deps+=($package) + for dep in $(list_deps $package); do # if not already checked - if echo ${deps[*]} | grep -qv "\b$dep\b"; then + if echo ${deps[@]} | grep -qv "\b$dep\b"; then to_check+=($dep) fi done @@ -38,11 +40,7 @@ resolve_deps () { } get_package_download_info() { - tail -1 ${PACKAGES_DIR}/*/$1 -} - -get_available_version () { - echo "${info[1]}" + sed 1q ${PACKAGES_DIR}/*/$1 } is_installed() { @@ -57,14 +55,65 @@ get_installed_version () { } # bad implementation -exists () { +package_exists () { [ "$(find ${PACKAGES_DIR} -mindepth 2 -name "$1" | wc -l)" != "0" ] } -download () { +download_packages () { + local total_download=$1; shift + local packages=($@) + local outputs=() + + local out_dir="${PACKAGE_CACHE}" + mkdir -p "$out_dir" + + for package in ${packages[*]}; do + local info=($(get_package_download_info $package)) + local url=${info[0]} + local checksum=${info[1]} + + local output="${out_dir}/${checksum}.${package}.xipkg" + local output_info="${output}.info" + + if validate_checksum $output $checksum; then + ${VERBOSE} && printf "${LIGHT_BLACK}skipping download for %s already exists with checksum %s${RESET}\n" $package $checksum + else + touch $output + + curl ${CURL_OPTS} -o "$output_info" "$url.info" & + curl ${CURL_OPTS} -o "$output" "$url" & + fi + + outputs+=($output) + done + + wait_for_download $total_download ${outputs[*]} + + + local i=0 + for pkg_file in ${outputs[*]}; do + + ${QUIET} || hbar -T "${LARGE_CIRCLE} validating downloads..." $i ${#outputs[*]} + + info_file="${pkg_file}.info" + if ! validate_sig $pkg_file $info_file; then + printf "${RED}Failed to verify signature for ${LIGHT_RED}%s${RED}\n" $(basename -s .xipkg $pkg_file) + mv "$pkg_file" "${pkg_file}.invalid" + else + i=$((i+1)) + fi + done + ${QUIET} || hbar -t ${HBAR_COMPLETE} -T "${CHECKMARK} validated downloads" $i ${#outputs[*]} + + install ${outputs[*]} + +} + +fetch () { local requested=($@) local missing=() + local already=() local install=() local update=() local urls=() @@ -72,7 +121,7 @@ download () { local total_download=0 for package in $(resolve_deps $@); do - if exists $package; then + if package_exists $package; then info=($(get_package_download_info $package)) url=${info[0]} checksum=${info[1]} @@ -80,9 +129,11 @@ download () { files=${info[3]} if is_installed $package; then - if [ "$(get_installed_version $package)" != "$(get_available_version $package)" ]; then + if [ "$(get_installed_version $package)" != "$checksum" ]; then update+=($package) total_download=$((total_download+size)) + else + already+=($package) fi else install+=($package) @@ -93,29 +144,48 @@ download () { fi done - if [ "${#missing[@]}" != "0" ]; then - printf "${LIGHT_RED}The following packages could not be located:" - for package in ${missing[*]}; do - printf "${RED} $package" - done - printf "${RESET}\n" - fi - if [ "${#update[@]}" != "0" ]; then - printf "${LIGHT_GREEN}The following packages will be updated:\n\t" - for package in ${update[*]}; do - printf "${GREEN} $package" - done - printf "${RESET}\n" - fi - if [ "${#install[@]}" != "0" ]; then - printf "${LIGHT_BLUE}The following packages will be updated:\n\t" - for package in ${install[*]}; do - printf "${BLUE} $package" - done - printf "${RESET}\n" + if ! ${QUIET}; then + if [ "${#missing[@]}" != "0" ]; then + printf "${LIGHT_RED}The following packages could not be located:" + for package in ${missing[*]}; do + printf "${RED} $package" + done + printf "${RESET}\n" + fi + if [ "${#update[@]}" != "0" ]; then + printf "${LIGHT_GREEN}The following packages will be updated:\n\t" + for package in ${update[*]}; do + printf "${GREEN} $package" + done + printf "${RESET}\n" + fi + if [ "${#install[@]}" != "0" ]; then + printf "${LIGHT_BLUE}The following packages will be installed:\n\t" + for package in ${install[*]}; do + printf "${BLUE} $package" + done + printf "${RESET}\n" + fi + if [ "${#already[@]}" != "0" ]; then + printf "${LIGHT_WHITE}The following packages are already up to date:\n\t" + for package in ${already[*]}; do + printf "${WHITE} $package" + done + printf "${RESET}\n" + fi fi - echo "total download size: ${total_download} bytes" + [ "${#install[@]}" = "0" ] && [ "${#update[@]}" = 0 ] && printf "${LIGHT_RED}Nothing to do!\n" && return 0 + + + ${QUIET} || [ "${SYSROOT}" = "/" ] || printf "${WHITE}To install to ${LIGHT_WHITE}${SYSROOT}${RESET}\n" + ${QUIET} || printf "${WHITE}Total download size:${LIGHT_WHITE} $(format_bytes $total_download)\n" + + if prompt_question "${WHITE}Continue?"; then + download_packages $total_download ${install[*]} ${update[*]} + else + ${QUIET} || printf "${RED}Action canceled by user\n" + fi } diff --git a/src/install.sh b/src/install.sh new file mode 100644 index 0000000..3623473 --- /dev/null +++ b/src/install.sh @@ -0,0 +1,62 @@ +#!/bin/bash + +extract () { + tar -h --no-overwrite-dir -vvxf $1 -C ${SYSROOT} | grep ^- +} + +install_package () { + local pkg_file="$1" + local name="$2" + local info_file="$pkg_file.info" + + local installed_dir="${INSTALLED_DIR}/$name" + local info="$installed_dir/info" + local files="$installed_dir/files" + local checksum="$installed_dir/checksum" + + mkdir -p "$installed_dir" + extract $1 > $files + cp $info_file $info + + md5sum $pkg_file | cut -d' ' -f1 > $checksum +} + +get_package_filecount() { + local info=($(get_package_download_info $1)) + echo ${info[3]} +} + +total_filecount() { + local packages=($@) + local count=0 + for package in ${packages[*]}; do + local name=$(basename -s .xipkg $package | cut -d. -f2) + local c=$(get_package_filecount $name) + count=$((count+c)) + done + echo $count +} + +install () { + local packages=($@) + + local missing=() + for package in ${packages[*]}; do + [ ! -f $package ] && missing+=($(basename $package)) + done + + if [ "${#missing[@]}" != "0" ]; then + # warning: potential recursion loop here + fetch ${missing[*]} + else + + local total=$(total_filecount ${packages[*]}) + local files_files=() + for package in ${packages[*]}; do + local name=$(basename -s .xipkg $package | cut -d. -f2) + install_package $package $name & + files_files+=("${INSTALLED_DIR}/$name/files") + done + wait_for_extract $total ${files_files[*]} + fi +} diff --git a/src/profile.sh b/src/profile.sh index 12bd41c..b150d74 100755 --- a/src/profile.sh +++ b/src/profile.sh @@ -1,6 +1,10 @@ #!/bin/bash -. /usr/lib/colors.sh +. /usr/lib/colors.sh && + export HBAR_COMPLETE="-c ${GREEN}${BG_DEFAULT}" + +. /usr/lib/glyphs.sh + export CONF_FILE="/etc/xipkg.conf" export CURL_OPTS="-SsL" @@ -8,7 +12,13 @@ export CURL_OPTS="-SsL" export DEP_DIR=$(parseconf -v dir.deps) export REPOS=($(parseconf -v repos)) export SOURCES=($(parseconf sources.*)) + export PACKAGES_DIR=$(parseconf -v dir.packages) -export INSTALLED_DIR=$(parseconf -v dir.installed) +export INSTALLED_DIR=${SYSROOT}$(parseconf -v dir.installed) +export KEYCHAIN_DIR=$(parseconf -v dir.keychain) + +export CACHE_DIR=$(parseconf -v dir.cache) +export PACKAGE_CACHE="${CACHE_DIR}/packages" +export SYNC_CACHE="${CACHE_DIR}/sync" + -export TMP_DIR="/tmp/xi" diff --git a/src/sync.sh b/src/sync.sh index 9589c43..5cc7a29 100755 --- a/src/sync.sh +++ b/src/sync.sh @@ -1,19 +1,5 @@ #!/bin/bash -download_file() { - curl ${CURL_OPTS} -o $1 -w "%{http_code}" $2 2> /dev/null -} - -wait_for_jobs () { - local total=$(jobs -r | wc -l) - local completed=0 - while [ "$completed" != "$total" ]; do - completed=$(( $total - $(jobs -r | wc -l))) - hbar -T "$1" $completed $total - done - hbar -t -T "$2" $completed $total - wait -} # save each listed package in a relevant directory, based on checksum # @@ -42,7 +28,7 @@ list_source () { local name=$(echo $src | cut -d":" -f1) local repo_url="${url}${repo}" local full_url="${repo_url}/packages.list" - local tmp_file="$TMP_DIR/$name.$repo" + local tmp_file="${SYNC_CACHE}/$name.$repo" local status=$(download_file $tmp_file $full_url) @@ -58,7 +44,7 @@ dep_graph () { local url=$(echo $src | cut -d":" -f2-) local name=$(echo $src | cut -d":" -f1) local full_url="${url}deps.graph" - local tmp_file="$TMP_DIR/$name.deps" + local tmp_file="${SYNC_CACHE}/$name.deps.graph" [ -f $tmp_file ] && rm $tmp_file; touch $tmp_file if [ "$(download_file $tmp_file $full_url)" = "200" ]; then @@ -89,7 +75,7 @@ popularity_contest () { contest $package_dir & done - wait_for_jobs "contesting packages..." "contested packages" + wait_for_jobs "${LARGE_CIRCLE} contesting packages..." "${CHECKMARK} contested packages" } index_deps () { @@ -100,9 +86,9 @@ index_deps () { for src in ${SOURCES[*]}; do dep_graph $src completed=$((completed+1)) - hbar -l $l -T "indexing dependencies..." $completed $total + ${QUIET} || hbar -l $l -T "${LARGE_CIRCLE} indexing dependencies..." $completed $total done - hbar -l $l -T "indexed dependencies" $completed $total + ${QUIET} || hbar -l $l ${HBAR_COMPLETE} -T "${CHECKMARK} indexed dependencies" $completed $total } index_repo () { @@ -113,21 +99,22 @@ index_repo () { for src in ${SOURCES[*]}; do list_source $repo $src completed=$((completed+1)) - hbar -l $l -T "syncing $repo..." $completed $total + ${QUIET} || hbar -l $l -T "${LARGE_CIRCLE} syncing $repo..." $completed $total done - hbar -l $l -T "synced $repo" $completed $total + ${QUIET} || hbar -l $l ${HBAR_COMPLETE} -T "${CHECKMARK} synced $repo" $completed $total } sync () { # prepare the file structure for the sync - mkdir -pv $TMP_DIR - rm -r $PACKAGES_DIR/* + mkdir -p ${SYNC_CACHE} + + [ "$(ls -A $PACKAGES_DIR)" ] && rm -r $PACKAGES_DIR/* rm -r $DEP_DIR mkdir $DEP_DIR # create padding spaces for each hbar - for repo in ${REPOS[*]}; do + ${QUIET} || for repo in ${REPOS[*]}; do hbar done @@ -135,6 +122,7 @@ sync () { index_deps 0 & local i=1 for repo in ${REPOS[*]}; do + mkdir -p ${PACKAGES_DIR}/$repo index_repo $repo $i & i=$((i+1)) done diff --git a/src/util.sh b/src/util.sh new file mode 100644 index 0000000..d20db86 --- /dev/null +++ b/src/util.sh @@ -0,0 +1,78 @@ +#!/bin/sh + +download_file() { + curl ${CURL_OPTS} -o $1 -w "%{http_code}" $2 2> /dev/null +} + +wait_for_jobs () { + if ! $QUIET; then + local total=$(jobs -r | wc -l) + local completed=0 + while [ "$completed" != "$total" ]; do + completed=$(( $total - $(jobs -r | wc -l))) + hbar -T "$1" $completed $total + done + hbar -t ${HBAR_COMPLETE} -T "$2" $completed $total + fi + + wait +} + +wait_for_download () { + if ! $QUIET && [ "$(jobs -r | wc -l)" != "0" ]; then + local total_download=$1 + shift + local files=($@) + + unset downloaded + while [ "$(jobs -r | wc -l)" != "0" ]; do + local downloaded=0 + + for output in ${files[*]}; do + size=$(stat -c %s $output) + downloaded=$((downloaded+size)) + done + + hbar -h -T " downloading packages" $downloaded $total_download + done + hbar -th ${HBAR_COMPLETE} -T "${CHECKMARK} downloaded packages" $downloaded $total_download + fi + + wait +} + +wait_for_extract () { + if ! $QUIET && [ "$(jobs -r | wc -l)" != "0" ]; then + local total_filecount=$1 + shift + local files=($@) + + unset extracted + while [ "$(jobs -r | wc -l)" != "0" ]; do + local extracted=0 + + for output in ${files[*]}; do + size=$(cat $output | wc -l) + extracted=$((extracted+size)) + done + + hbar -T " extracting files" $extracted $total_filecount + done + hbar -t ${HBAR_COMPLETE} -T "${CHECKMARK} extracted packages" $extracted $total_filecount + fi + + wait +} + + +format_bytes () { + echo $@ | numfmt --to iec + +} + +prompt_question () { + $NOCONFIRM && return 0 + printf "$1 [Y/n] " + read response + [ "${response:0}" != "n" ] +} diff --git a/src/validate.sh b/src/validate.sh new file mode 100644 index 0000000..4f73729 --- /dev/null +++ b/src/validate.sh @@ -0,0 +1,25 @@ +#!/bin/bash + +validate_checksum () { + local file=$1 + local checksum=$2 + [ ! -f $file ] && return 1 + [ "$(md5sum $file | awk '{ print $1; }')" = "$checksum" ] +} + +validate_sig () { + local pkg_file=$1 + local info_file=$2 + local keychain + + local sig_encoded=$(sed -rn "s/^SIGNATURE=(.*)/\1/p" $info_file) + local sig_file="${pkg_file}.sig" + + echo $sig_encoded | tr ' ' '\n' | base64 -d > $sig_file + + for key in ${KEYCHAIN_DIR}/*.pub; do + ${VERBOSE} && printf "${LIGHT_BLACK}Checking verification against $(basename $key) for $(basename $pkg_file)\n${RESET}" + openssl dgst -verify $key -signature $sig_file $pkg_file | grep -q "OK" && return 0 + done + return 1 +} diff --git a/src/xi.sh b/src/xi.sh index aaad140..8c8d816 100755 --- a/src/xi.sh +++ b/src/xi.sh @@ -1,19 +1,17 @@ #!/bin/bash [ -z "${LIBDIR}" ] && LIBDIR=/usr/lib/xipkg -. ${LIBDIR}/profile.sh -. ${LIBDIR}/sync.sh -. ${LIBDIR}/get.sh export SYSROOT=/ export CONF_FILE="/etc/xipkg.conf" export VERBOSE=false +export QUIET=false export RESOLVE_DEPS=true export DO_SYNC=true export UNSAFE=false export NOCONFIRM=false -while getopts ":r:c:nluyv" opt; do +while getopts ":r:c:qnluyv" opt; do case "${opt}" in r) SYSROOT="${OPTARG}" @@ -36,9 +34,40 @@ while getopts ":r:c:nluyv" opt; do v) VERBOSE=true ;; + q) + QUIET=true + ;; esac done +. ${LIBDIR}/profile.sh +. ${LIBDIR}/util.sh +. ${LIBDIR}/validate.sh + +. ${LIBDIR}/sync.sh +. ${LIBDIR}/install.sh +. ${LIBDIR}/get.sh + shift $((OPTIND-1)) -download $@ +if [ "$#" = "0" ]; then + echo "xilinux running xipkg (palceholder text)" +else + + + case "$1" in + "sync") + sync + ;; + "install" | "update") + shift + $DO_SYNC && sync + install $@ + ;; + *) + $DO_SYNC && sync + fetch $@ + ;; + esac +fi +printf "${RESET}" -- cgit v1.2.1