summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rwxr-xr-xinstall-system.sh46
-rw-r--r--src/verbs/install.py183
-rw-r--r--src/verbs/sync.py43
3 files changed, 201 insertions, 71 deletions
diff --git a/install-system.sh b/install-system.sh
index 33766e1..eeb7e9c 100755
--- a/install-system.sh
+++ b/install-system.sh
@@ -5,8 +5,23 @@
KEY="davidovski https://xi.davidovski.xyz/repo/xi.pub"
+XI_OPTS="-nyl"
+
R=$1
+[ $# -eq 0 ] && echo "Please specify where you would like to instal the system" && exit 1
+
+if [ -e $R ]; then
+ printf "Remove existing system? [Y/n] "
+ read response
+
+ if [ "$response" != "n" ]; then
+ rm -rf $R
+ echo "removed $R"
+ fi
+fi
+
+
mkdir -p $R
mkdir -p $R/tmp
mkdir -p $R/dev
@@ -28,17 +43,31 @@ ln -s usr/lib usr/lib64
ln -s usr/local usr
-xi sync
-
-xi -nyl --root . install $(ls /var/lib/xipkg/packages/core)
-xi -nyl --root . keyimport $KEY
+xi sync &&
+xi $XI_OPTS --root . install $(ls /var/lib/xipkg/packages/core) &&
+xi $XI_OPTS --root . keyimport $KEY &&
# chroot into the system to install xipkg and any postinstall scripts
-xi -nyl --root . install xipkg
+xi $XI_OPTS --root . install xipkg &&
+
+cd ../.. &&
+
+echo "base system installed next to do:" &&
+echo " xichroot into system" &&
+echo " set hostname at /etc/hostname" &&
+echo " configure DNS at /etc/resolv.conf" &&
+echo " xi sync" &&
+echo " install any additional packages" &&
+echo " compile and install kernel" &&
+echo " configure and install grub" &&
+echo " * hope that the system works!" &&
+echo "have fun!" &&
+exit 0;
+
+echo "something went wrong"
+exit 0;
-cd bin
-ln -s bash sh
-cd ../..
+## leftover autoconfig scripts
mkdir -p $R/var/lib/xipkg/
cp -r /var/lib/xipkg/packages $R/var/lib/xipkg
@@ -46,6 +75,7 @@ cp -r /var/lib/xipkg/keychain $R/var/lib/xipkg
# Autoconfiguring some things like network
+mkdir -p $R/etc
echo "xilinux" > $R/etc/hostname
cat > $R/etc/resolv.conf << "EOF"
diff --git a/src/verbs/install.py b/src/verbs/install.py
index 60e1809..9832e2b 100644
--- a/src/verbs/install.py
+++ b/src/verbs/install.py
@@ -32,14 +32,17 @@ def find_package(query, repos, packages_dir, sources):
requested_repo = repo
with open(os.path.join(repo_dir, query)) as file:
checksum = file.readline().strip().split("=")[-1]
+ size = file.readline().strip().split("=")[-1]
+ filecount = file.readline().strip().split("=")[-1]
listed_sources = file.readline().strip().split("=")[-1].split()
found_sources = {
source: util.add_path(url, repo)
for source, url in sources.items()
if source in listed_sources
}
- return checksum, found_sources, requested_repo
- return None, [], None
+ return checksum, found_sources, requested_repo, int(size)*1000, int(filecount)
+ return None, [], None, 0, 0
+
def verify_signature(package_file, package_info,
cache_dir="/var/cache/xipkg", keychain_dir="/var/lib/xipkg/keychain",
@@ -69,6 +72,7 @@ def verify_signature(package_file, package_info,
print(colors.BLACK + "There are no keys to verify with")
return ""
+
def retrieve_package_info(sources, checksum, package_name, config,
verbose=False, skip_verification=False):
@@ -76,7 +80,6 @@ def retrieve_package_info(sources, checksum, package_name, config,
cache_dir=config["dir"]["cache"]
# TODO we may potentially do this a few times while resolving deps, might want to cache things here
-
# TODO or find cached package checksum from the cache folder
for source in get_best_source(sources, sources_list=sources_list):
url = sources[source]
@@ -97,7 +100,21 @@ def retrieve_package_info(sources, checksum, package_name, config,
print(colors.RED + f"No matching hashes found" + colors.RESET)
return {}
-def retrieve_package(sources, package_info, package_name, config,
+# Does not verify the package itself, will only blindly accept the best size it can
+def query_package_size(sources, package_info, package_name, config, verbose=False):
+ sources_list=config["dir"]["sources"]
+ for source in get_best_source(sources, sources_list=sources_list):
+ url = sources[source]
+ if verbose:
+ print(colors.LIGHT_BLACK + f"using source {source} at {url} for {package_name}")
+
+ package_url = util.add_path(url, package_name + ".xipkg")
+ size = util.query_size(package_url)
+ if size > 0:
+ return size
+ return 0
+
+def retrieve_package(sources, package_info, package_name, config, completed=0, total_download=-1,
verbose=False, skip_verification=False):
sources_list=config["dir"]["sources"]
@@ -114,8 +131,15 @@ def retrieve_package(sources, package_info, package_name, config,
package_dir = util.add_path(cache_dir, source)
util.mkdir(package_dir)
- # TODO if exists maybe just use cached version
- status, package_path = util.curl_to_file(package_url, util.add_path(package_dir, package_name + ".xipkg"), text=package_name + ".xipkg")
+
+ if total_download == -1:
+ text = package_name + ".xipkg"
+ else:
+ text = "packages..."
+
+ # TODO if package already downloaded maybe just use cached version
+ status, package_path, size = util.curl_to_file(package_url, util.add_path(package_dir, package_name + ".xipkg"),
+ start=completed, total=total_download, text=text)
if status == 200:
downloaded_checksum = util.md5sum(package_path)
@@ -125,8 +149,7 @@ def retrieve_package(sources, package_info, package_name, config,
sig = verify_signature(package_path, package_info,
cache_dir=cache_dir, keychain_dir=keychain_dir, verbose=verbose)
if len(sig) > 0:
- print(colors.RESET)
- return package_path, source, sig
+ return package_path, source, sig, size
elif verbose:
print(colors.RED
+ f"Failed to verify signature for {package_name} in {source}"
@@ -136,8 +159,7 @@ def retrieve_package(sources, package_info, package_name, config,
+ f"Checksum verification failed for {package_name} in {source}"
+ colors.RESET)
else:
- print(colors.RESET)
- return package_path, source, "none"
+ return package_path, source, "none", size
print(colors.RESET + colors.RED + f"No valid packages found for {package_name}" + colors.RESET)
return ""
@@ -171,15 +193,13 @@ def find_all_dependencies(package_names, options, config):
# this is all assuming that the order of deps installed doesn't matter
to_check = [p for p in package_names]
all_deps = []
+ failed = []
while len(to_check) > 0:
util.loading_bar(len(all_deps), len(all_deps) + len(to_check), "Resolving dependencies...")
dep = to_check.pop()
- # probably better way to implement this obligatory wildcard
- # 100% sure there is a better way of doing this than installing all packages from a repo
- # maybe some sort of package grouping (or empty package with deps on all needed)
- dep_checksum, dep_sources, dep_repo = find_package(dep, config["repos"], config["dir"]["packages"], config["sources"])
+ dep_checksum, dep_sources, dep_repo, size, files = find_package(dep, config["repos"], config["dir"]["packages"], config["sources"])
if dep_checksum is not None:
info = retrieve_package_info(
@@ -191,15 +211,18 @@ def find_all_dependencies(package_names, options, config):
if not dep in all_deps:
all_deps.append(dep)
deps = resolve_dependencies(info)
- for dep in deps:
- if not dep in all_deps:
- if is_installed(dep, config, options["r"]):
+ for d in deps:
+ if not d in all_deps:
+ if is_installed(d, config, options["r"]):
if options["v"]: print(colors.YELLOW + f"Package {dep} has already been installed")
else:
- to_check.append(dep)
- elif options["v"]:
- util.print_reset(colors.CLEAR_LINE + colors.RED + f"Failed to retrieve info for {dep}")
+ to_check.append(d)
+ else:
+ if not dep in failed: failed.append(dep)
+ if options["v"]:
+ util.print_reset(colors.CLEAR_LINE + colors.RED + f"Failed to retrieve info for {dep}")
else:
+ if not dep in failed: failed.append(dep)
if options["v"]: util.print_reset(colors.CLEAR_LINE + colors.RED + f"Failed to find package {dep}")
if len(all_deps) > 0:
@@ -209,7 +232,7 @@ def find_all_dependencies(package_names, options, config):
# assuming that the latter packages are core dependencies
# we can reverse the array to reflect the more important packages to install
all_deps.reverse()
- return all_deps
+ return all_deps, failed
def is_installed(package_name, config, root="/"):
installed_dir = util.add_path(root, config["dir"]["installed"])
@@ -227,6 +250,7 @@ def install_package(package_name, package_path, package_info,
if post_install:
run_post_install(config, verbose=verbose, root=root)
save_installed_info(package_name, package_info, files, repo, source_url, key, config, root=root)
+ return files
@@ -261,7 +285,20 @@ def save_installed_info(package_name, package_info,
with open(files_file, "w") as file:
file.write(files)
- pass
+
+def run_post_install(config, verbose=False, root="/"):
+ installed_dir = util.add_path(root, config["dir"]["postinstall"])
+ if os.path.exists(installed_dir):
+ files = os.listdir(installed_dir)
+ for file in files:
+ f = util.add_path(config["dir"]["postinstall"], file)
+ command = f"sh {f}"
+ if root != "/":
+ os.chroot(root)
+ os.chdir("/")
+ os.system(command)
+ os.remove(f)
+
def install_single(package, options, config, post_install=True, verbose=False, unsafe=False):
checksum, sources, repo = find_package(package, config["repos"],
@@ -280,18 +317,6 @@ def install_single(package, options, config, post_install=True, verbose=False, u
repo, sources[source], key, post_install,
config, verbose=verbose, root=options["r"])
-def run_post_install(config, verbose=False, root="/"):
- installed_dir = util.add_path(root, config["dir"]["postinstall"])
- if os.path.exists(installed_dir):
- files = os.listdir(installed_dir)
- for file in files:
- f = util.add_path(config["dir"]["postinstall"], file)
- command = f"sh {f}"
- if root != "/":
- os.chroot(root)
- os.chdir("/")
- os.system(command)
- os.remove(f)
def install(args, options, config):
if not options["l"]:
@@ -308,27 +333,89 @@ def install(args, options, config):
# have some interaction with sudo when necessary rather than always require it
# this check may need to be done sooner?
if util.is_root() or options["r"] != "/":
- to_install = args if options["n"] else find_all_dependencies(args, options, config)
+ to_install, location_failed = args, []
+ if not options["n"]:
+ to_install, location_failed = find_all_dependencies(args, options, config)
- if len(to_install) > 0:
- print(colors.CLEAR_LINE + colors.RESET, end="")
- print(colors.BLUE + "The following packages will be installed:")
+ if len(location_failed) > 0:
+ print(colors.LIGHT_RED + "Failed to locate the following packages:")
print(end="\t")
- for d in to_install:
- print(colors.BLUE if d in args else colors.LIGHT_BLUE, d, end="")
+ for d in location_failed:
+ print(colors.RED if d in args else colors.LIGHT_RED, d, end="")
print()
- if util.ask_confirmation(colors.BLUE + "Continue?", no_confirm=options["y"]):
+ if len(to_install) > 0:
+ length = 0
+ total_files = 0
+ infos = []
+ for package in to_install:
+ util.loading_bar(len(infos), len(to_install), "Gathering package infos")
+ checksum, sources, repo, size, filecount = find_package(package, config["repos"],
+ config["dir"]["packages"], config["sources"])
+
+ if checksum != None:
+ info = retrieve_package_info(
+ sources, checksum, package, config,
+ verbose=v, skip_verification=unsafe
+ )
+
+ # TODO make package_size be written in package info or sync list instead
+ length += int(size)
+ total_files += int(filecount)
+
+ infos.append(
+ (package, sources, repo, info)
+ )
+
+ divisor, unit = util.get_unit(length)
+
+ util.loading_bar(len(infos), len(to_install), "Gathered package infos")
+ print(colors.RESET)
+
+ if not options["y"]:
+ print(colors.BLUE + "The following packages will be installed:")
+ print(end="\t")
+ for d in to_install:
+ print(colors.BLUE if d in args else colors.LIGHT_BLUE, d, end="")
+ print()
+
+ print(colors.BLUE + "Total download size: " + colors.LIGHT_BLUE + str(round(length / divisor, 2)) + unit)
+
+ if options["y"] or util.ask_confirmation(colors.BLUE + "Continue?"):
+ # TODO try catch over each package in each stage so that we can know if there are errors
+
+ downloaded = 0
+ pkg_files = []
+ for package_info in infos:
+ (package, sources, repo, info) = package_info
+
+ package_path, source, key, size = retrieve_package(sources,
+ info, package, config,
+ completed=downloaded, total_download=length,
+ verbose=v, skip_verification=unsafe)
+
+ downloaded += size
+
+ pkg_files.append(
+ (package, package_path, sources[source], key, repo, info)
+ )
+
+ util.loading_bar(int(length/divisor), int(length/divisor), "Downloaded packages", unit=unit)
+ print(colors.RESET)
- for package in to_install:
- try:
- install_single(package, options, config, verbose=v, unsafe=unsafe)
- util.fill_line(f"Installed {package}", colors.BG_CYAN + colors.LIGHT_BLACK, end="\n")
- except Exception as e:
- util.fill_line(f"Failed to install {package}", colors.BG_RED + colors.LIGHT_BLACK, end="\n")
- util.fill_line(str(e), colors.CLEAR_LINE + colors.RESET + colors.RED, end="\n")
+ extracted = 0
+ for f in pkg_files:
+ util.loading_bar(extracted, total_files, "Installing files")
+ (package, package_path, source, key, repo, info) = f
+ files = install_package(package, package_path, info,
+ repo, source, key, True,
+ config, verbose=v, root=options["r"])
+ extracted += len(files.split("\n"))
+
+ util.loading_bar(extracted, total_files, "Installed files")
+ print(colors.RESET)
else:
print(colors.RED + "Action cancelled by user")
else:
diff --git a/src/verbs/sync.py b/src/verbs/sync.py
index f47e0db..866a821 100644
--- a/src/verbs/sync.py
+++ b/src/verbs/sync.py
@@ -6,6 +6,9 @@ import time
CACHE_DIR = "/var/cache/xipkg"
+# returns a dictionary, and duration:
+# key: package name
+# value: list of info [checksum, size]
def list_packages(url):
start = time.time()
status, response = util.curl(url + "/packages.list")
@@ -15,12 +18,13 @@ def list_packages(url):
else:
duration /= len(response)
return {
- line.split()[0].split(".")[0]: line.split()[1]
+ line.split()[0].split(".")[0]: " ".join(line.split()[1:])
for line in response.split("\n") if len(line.split()) > 0
}, duration
+
def sync_packages(repo, sources, verbose=False):
- packages = {}
+ versions = {}
speeds = {}
for source,url in sources.items():
@@ -29,24 +33,26 @@ def sync_packages(repo, sources, verbose=False):
if speed > 0:
speeds[source] = speed
- if len(listed) == 0 and verbose:
- print(colors.RED + f"No packages found in {source}/{repo}" + colors.RESET)
+ if verbose:
+ if len(listed) == 0:
+ print(colors.RED + f"No packages found in {source}/{repo}" + colors.RESET)
+ else:
+ print(colors.BLACK + f"{len(listed)} packages found in {source}/{repo}" + colors.RESET)
for p in listed:
- if not p in packages:
- packages[p] = []
- packages[p].append((listed[p], source))
+ if not p in versions: versions[p] = []
+ versions[p].append((listed[p], source))
- return packages, speeds
+ return versions, speeds
def validate_package(package, versions, repo, verbose=False):
popularity = {}
for v in versions:
- checksum = v[0]
+ info = v[0]
source = v[1]
- if not checksum in popularity:
- popularity[checksum] = 0
- popularity[checksum] += 1
+ if not info in popularity:
+ popularity[info] = 0
+ popularity[info] += 1
most_popular = ""
p_count = -1
@@ -56,11 +62,16 @@ def validate_package(package, versions, repo, verbose=False):
p_count = c
sources = [v[1] for v in versions if v[0] == most_popular]
+
# change the packages dict to list all the sources
- return {
- "checksum": most_popular,
+ # maybe some validation here
+ info = {
+ "checksum": most_popular.split()[0],
+ "size": most_popular.split()[1],
+ "files": most_popular.split()[2],
"sources" : sources
}
+ return info
def save_package(package, info, location):
util.mkdir(location)
@@ -75,6 +86,8 @@ def save_package(package, info, location):
content = ""
with open(package_file, "w") as file:
file.write("checksum=" + info["checksum"] + "\n")
+ file.write("size=" + info["size"] + "\n")
+ file.write("files=" + info["files"] + "\n")
file.write("sources=" + " ".join([source for source in info["sources"]]))
return exists
@@ -139,7 +152,7 @@ def sync(args, options, config):
# find the most popular hash to use
done = 0
total = len(packages.items())
- for package,versions in packages.items():
+ for package, versions in packages.items():
info = validate_package(package, versions, repo, verbose=v)
if not save_package(package, info, repo_dir):
new += 1
generated by cgit v1.2.1 (git 2.18.0) at 2024-09-21 14:00:31 +0000