From 1117f2bef50ec65aa6bfe55e8e22beb5be092275 Mon Sep 17 00:00:00 2001 From: davidovski Date: Wed, 16 Feb 2022 16:58:36 +0000 Subject: added dependency resolving --- src/get.sh | 121 +++++++++++++++++++++++++++++++++++++++++ src/profile.sh | 14 +++++ src/sync.sh | 148 ++++++++++++++++++++++++++++++++++++++++++++++++++ src/xi.sh | 44 +++++++++++++++ src/xiget.sh | 12 ----- src/xisync.sh | 166 --------------------------------------------------------- 6 files changed, 327 insertions(+), 178 deletions(-) create mode 100755 src/get.sh create mode 100755 src/profile.sh create mode 100755 src/sync.sh create mode 100755 src/xi.sh delete mode 100644 src/xiget.sh delete mode 100755 src/xisync.sh diff --git a/src/get.sh b/src/get.sh new file mode 100755 index 0000000..3e0be79 --- /dev/null +++ b/src/get.sh @@ -0,0 +1,121 @@ +#!/bin/sh + + +# list all direct dependencies of a package +# +list_deps() { + local name=$1 + local tree_file="${DEP_DIR}/$name" + [ -f $tree_file ] && + for dep in $(cat "$tree_file"); do + echo $dep + done | sort -u +} + +# list all dependencies and sub dependencies of a package +# +resolve_deps () { + local deps=() + local to_check=($@) + if ${RESOLVE_DEPS}; then + while [ "${#to_check[@]}" != "0" ]; do + local package=${to_check[-1]} + unset to_check[-1] + + deps+=($package) + for dep in $(list_deps $package); do + # if not already checked + if echo ${deps[*]} | grep -qv "\b$dep\b"; then + to_check+=($dep) + fi + done + done + echo ${deps[@]} + else + echo $@ + fi + +} + +get_package_download_info() { + tail -1 ${PACKAGES_DIR}/*/$1 +} + +get_available_version () { + echo "${info[1]}" +} + +is_installed() { + [ -f "${INSTALLED_DIR}/$1/checksum" ] +} + +get_installed_version () { + local name=$1 + local file="${INSTALLED_DIR}/$name/checksum" + [ -f $file ] && + cat $file +} + +# bad implementation +exists () { + [ "$(find ${PACKAGES_DIR} -mindepth 2 -name "$1" | wc -l)" != "0" ] +} + +download () { + local requested=($@) + + local missing=() + local install=() + local update=() + local urls=() + + local total_download=0 + + for package in $(resolve_deps $@); do + if exists $package; then + info=($(get_package_download_info $package)) + url=${info[0]} + checksum=${info[1]} + size=${info[2]} + files=${info[3]} + + if is_installed $package; then + if [ "$(get_installed_version $package)" != "$(get_available_version $package)" ]; then + update+=($package) + total_download=$((total_download+size)) + fi + else + install+=($package) + total_download=$((total_download+size)) + fi + else + missing+=($package) + fi + done + + if [ "${#missing[@]}" != "0" ]; then + printf "${LIGHT_RED}The following packages could not be located:" + for package in ${missing[*]}; do + printf "${RED} $package" + done + printf "${RESET}\n" + fi + if [ "${#update[@]}" != "0" ]; then + printf "${LIGHT_GREEN}The following packages will be updated:\n\t" + for package in ${update[*]}; do + printf "${GREEN} $package" + done + printf "${RESET}\n" + fi + if [ "${#install[@]}" != "0" ]; then + printf "${LIGHT_BLUE}The following packages will be updated:\n\t" + for package in ${install[*]}; do + printf "${BLUE} $package" + done + printf "${RESET}\n" + fi + + echo "total download size: ${total_download} bytes" +} + + diff --git a/src/profile.sh b/src/profile.sh new file mode 100755 index 0000000..12bd41c --- /dev/null +++ b/src/profile.sh @@ -0,0 +1,14 @@ +#!/bin/bash + +. /usr/lib/colors.sh +export CONF_FILE="/etc/xipkg.conf" + +export CURL_OPTS="-SsL" + +export DEP_DIR=$(parseconf -v dir.deps) +export REPOS=($(parseconf -v repos)) +export SOURCES=($(parseconf sources.*)) +export PACKAGES_DIR=$(parseconf -v dir.packages) +export INSTALLED_DIR=$(parseconf -v dir.installed) + +export TMP_DIR="/tmp/xi" diff --git a/src/sync.sh b/src/sync.sh new file mode 100755 index 0000000..9589c43 --- /dev/null +++ b/src/sync.sh @@ -0,0 +1,148 @@ +#!/bin/bash + +download_file() { + curl ${CURL_OPTS} -o $1 -w "%{http_code}" $2 2> /dev/null +} + +wait_for_jobs () { + local total=$(jobs -r | wc -l) + local completed=0 + while [ "$completed" != "$total" ]; do + completed=$(( $total - $(jobs -r | wc -l))) + hbar -T "$1" $completed $total + done + hbar -t -T "$2" $completed $total + wait +} + +# save each listed package in a relevant directory, based on checksum +# +parse_line() { + local repo=$1 + local repo_url=$2 + local package=$3 + local checksum=$4 + local size=$5 + local files=$6 + + local package_name=$(basename -s ".xipkg" $package) + + local package_dir="$PACKAGES_DIR/$repo/$package_name.versions" + local checksum_file=$package_dir/$checksum + + [ -d $package_dir ] || mkdir -p $package_dir + printf "$repo_url/$package $checksum $size $files\n" >> $checksum_file +} + +list_source () { + local repo=$1 + local src=$2 + + local url=$(echo $src | cut -d":" -f2-) + local name=$(echo $src | cut -d":" -f1) + local repo_url="${url}${repo}" + local full_url="${repo_url}/packages.list" + local tmp_file="$TMP_DIR/$name.$repo" + + local status=$(download_file $tmp_file $full_url) + + if [ "$status" = "200" ]; then + while IFS= read -r line; do + parse_line $repo $repo_url $line + done < "$tmp_file" + fi +} + +dep_graph () { + local src=$1 + local url=$(echo $src | cut -d":" -f2-) + local name=$(echo $src | cut -d":" -f1) + local full_url="${url}deps.graph" + local tmp_file="$TMP_DIR/$name.deps" + [ -f $tmp_file ] && rm $tmp_file; touch $tmp_file + + if [ "$(download_file $tmp_file $full_url)" = "200" ]; then + while IFS= read -r line; do + local package=$(echo $line | cut -d: -f1) + local new=$(echo $line | cut -d: -f2-) + echo $new >> $DEP_DIR/$package + done < "$tmp_file" + fi +} + + +contest () { + local package_dir=$1 + + local popular=$(wc -l $package_dir/* | sort -n | head -1 | awk '{ print $2 }' ) + + local info_file=$(sed "s/.versions//g" <<< "$package_dir") + mv $popular $info_file + rm -r $package_dir +} + +popularity_contest () { + local list=$(ls -1 -d $PACKAGES_DIR/*/*) + local total=$(echo $list | wc -l) + + for package_dir in $list; do + contest $package_dir & + done + + wait_for_jobs "contesting packages..." "contested packages" +} + +index_deps () { + local l=$1 + local total=${#SOURCES[*]} + local completed=0 + + for src in ${SOURCES[*]}; do + dep_graph $src + completed=$((completed+1)) + hbar -l $l -T "indexing dependencies..." $completed $total + done + hbar -l $l -T "indexed dependencies" $completed $total +} + +index_repo () { + local repo=$1 l=$2 + local total=${#SOURCES[*]} + local completed=0 + + for src in ${SOURCES[*]}; do + list_source $repo $src + completed=$((completed+1)) + hbar -l $l -T "syncing $repo..." $completed $total + done + hbar -l $l -T "synced $repo" $completed $total +} + + +sync () { + # prepare the file structure for the sync + mkdir -pv $TMP_DIR + rm -r $PACKAGES_DIR/* + rm -r $DEP_DIR + mkdir $DEP_DIR + + # create padding spaces for each hbar + for repo in ${REPOS[*]}; do + hbar + done + + # download package lists and dep graphs at the same time + index_deps 0 & + local i=1 + for repo in ${REPOS[*]}; do + index_repo $repo $i & + i=$((i+1)) + done + + # wait for all jobs to complete + wait + + # determine which version of which package should be regarded + hbar + popularity_contest +} diff --git a/src/xi.sh b/src/xi.sh new file mode 100755 index 0000000..aaad140 --- /dev/null +++ b/src/xi.sh @@ -0,0 +1,44 @@ +#!/bin/bash + +[ -z "${LIBDIR}" ] && LIBDIR=/usr/lib/xipkg +. ${LIBDIR}/profile.sh +. ${LIBDIR}/sync.sh +. ${LIBDIR}/get.sh + +export SYSROOT=/ +export CONF_FILE="/etc/xipkg.conf" +export VERBOSE=false +export RESOLVE_DEPS=true +export DO_SYNC=true +export UNSAFE=false +export NOCONFIRM=false + +while getopts ":r:c:nluyv" opt; do + case "${opt}" in + r) + SYSROOT="${OPTARG}" + ;; + c) + CONF_FILE="${OPTARG}" + ;; + n) + RESOLVE_DEPS=false + ;; + l) + DO_SYNC=false + ;; + u) + UNSAFE=true + ;; + y) + NOCONFIRM=true + ;; + v) + VERBOSE=true + ;; + esac +done + +shift $((OPTIND-1)) + +download $@ diff --git a/src/xiget.sh b/src/xiget.sh deleted file mode 100644 index 815c47f..0000000 --- a/src/xiget.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/sh - -export CONF_FILE="/etc/xipkg.conf" - -CURL_OPTS="-SsL" - -DEP_GRAPH=$(parseconf -v dir.deps) - -get_deps() { - local name=$1 - [ -f $DEP_GRAPH ] && sed -rn "s/^$name: (.*)/\1/p" $DEP_GRAPH || echo -} diff --git a/src/xisync.sh b/src/xisync.sh deleted file mode 100755 index 5349dd2..0000000 --- a/src/xisync.sh +++ /dev/null @@ -1,166 +0,0 @@ -#!/bin/bash - -export CONF_FILE="/etc/xipkg.conf" - -CURL_OPTS="-SsL" - -REPOS=($(parseconf -v repos)) -SOURCES=($(parseconf sources.*)) -PACKAGES_DIR=$(parseconf -v dir.packages) -DEP_GRAPH=$(parseconf -v dir.deps) - -TMP_DIR="/tmp/xi" - -get_deps() { - local name=$1 - [ -f $DEP_GRAPH ] && sed -rn "s/^$name: (.*)/\1/p" $DEP_GRAPH || echo -} - -download_file() { - curl ${CURL_OPTS} -o $1 -w "%{http_code}" $2 2> /dev/null -} - -wait_for_jobs () { - local total=$(jobs -r | wc -l) - local completed=0 - while [ "$completed" != "$total" ]; do - completed=$(( $total - $(jobs -r | wc -l))) - hbar -T "$1" $completed $total - done - hbar -t -T "$2" $completed $total - wait -} - -# save each listed package in a relevant directory, based on checksum -# -parse_line() { - local repo=$1 - local repo_url=$2 - local package=$3 - local checksum=$4 - local size=$5 - local files=$6 - - local package_name=$(basename -s ".xipkg" $package) - - local package_dir="$PACKAGES_DIR/$repo/$package_name.versions" - local checksum_file=$package_dir/$checksum - - [ -d $package_dir ] || mkdir -p $package_dir - printf "$repo_url/$package $checksum $size $files\n" >> $checksum_file -} - -list_source () { - local repo=$1 - local src=$2 - - local url=$(echo $src | cut -d":" -f2-) - local name=$(echo $src | cut -d":" -f1) - local repo_url="${url}${repo}" - local full_url="${repo_url}/packages.list" - local tmp_file="$TMP_DIR/$name.$repo" - - local status=$(download_file $tmp_file $full_url) - - if [ "$status" = "200" ]; then - while IFS= read -r line; do - parse_line $repo $repo_url $line - done < "$tmp_file" - fi -} - -dep_graph () { - local src=$1 - local url=$(echo $src | cut -d":" -f2-) - local name=$(echo $src | cut -d":" -f1) - local full_url="${url}deps.graph" - local tmp_file="$TMP_DIR/$name.deps" - [ -f $tmp_file ] && rm $tmp_file; touch $tmp_file - - if [ "$(download_file $tmp_file $full_url)" = "200" ]; then - while IFS= read -r line; do - local package=$(echo $line | cut -d: -f1) - local new=$(echo $line | cut -d: -f2-) - echo $new >> $DEP_GRAPH/$package - done < "$tmp_file" - fi -} - - -contest () { - local package_dir=$1 - - local popular=$(wc -l $package_dir/* | sort -n | head -1 | awk '{ print $2 }' ) - - local info_file=$(sed "s/.versions//g" <<< "$package_dir") - mv $popular $info_file - rm -r $package_dir -} - -popularity_contest () { - local list=$(ls -1 -d $PACKAGES_DIR/*/*) - local total=$(echo $list | wc -l) - - for package_dir in $list; do - contest $package_dir & - done - - wait_for_jobs "contesting packages..." "contested packages" -} - -index_deps () { - local l=$1 - local total=${#SOURCES[*]} - local completed=0 - - for src in ${SOURCES[*]}; do - dep_graph $src - completed=$((completed+1)) - hbar -l $l -T "indexing dependencies..." $completed $total - done - hbar -l $l -T "indexed dependencies" $completed $total -} - -index_repo () { - local repo=$1 l=$2 - local total=${#SOURCES[*]} - local completed=0 - - for src in ${SOURCES[*]}; do - list_source $repo $src - completed=$((completed+1)) - hbar -l $l -T "syncing $repo..." $completed $total - done - hbar -l $l -T "synced $repo" $completed $total -} - - -sync () { - # prepare the file structure for the sync - mkdir -pv $TMP_DIR - rm -r $PACKAGES_DIR/* - rm -r $DEP_GRAPH - mkdir $DEP_GRAPH - - # create padding spaces for each hbar - for repo in ${REPOS[*]}; do - hbar - done - - # download package lists and dep graphs at the same time - index_deps 0 & - local i=1 - for repo in ${REPOS[*]}; do - index_repo $repo $i & - i=$((i+1)) - done - - # wait for all jobs to complete - wait - - # determine which version of which package should be regarded - hbar - popularity_contest -} - -sync -- cgit v1.2.1