grouped together loading bars and merged file sizes into the sync phase

2 files changed, 163 insertions, 63 deletions

diff --git a/src/verbs/install.py b/src/verbs/install.py
index 60e1809..9832e2b 100644
--- a/src/verbs/install.py
+++ b/src/verbs/install.py
@@ -32,14 +32,17 @@ def find_package(query, repos, packages_dir, sources):
             requested_repo = repo
             with open(os.path.join(repo_dir, query)) as file:
                 checksum = file.readline().strip().split("=")[-1]
+                size = file.readline().strip().split("=")[-1]
+                filecount = file.readline().strip().split("=")[-1]
                 listed_sources = file.readline().strip().split("=")[-1].split()
                 found_sources = {
                             source: util.add_path(url, repo) 
                             for source, url in sources.items() 
                             if source in listed_sources
                         }
-                return checksum, found_sources, requested_repo
-    return None, [], None
+                return checksum, found_sources, requested_repo, int(size)*1000, int(filecount)
+    return None, [], None, 0, 0
+
 
 def verify_signature(package_file, package_info, 
                             cache_dir="/var/cache/xipkg", keychain_dir="/var/lib/xipkg/keychain",
@@ -69,6 +72,7 @@ def verify_signature(package_file, package_info,
         print(colors.BLACK + "There are no keys to verify with")
     return ""
 
+
 def retrieve_package_info(sources, checksum, package_name, config,
                             verbose=False, skip_verification=False):
 
@@ -76,7 +80,6 @@ def retrieve_package_info(sources, checksum, package_name, config,
     cache_dir=config["dir"]["cache"]
     
     # TODO we may potentially do this a few times while resolving deps, might want to cache things here
-
     # TODO or find cached package checksum from the cache folder
     for source in get_best_source(sources, sources_list=sources_list):
         url = sources[source]
@@ -97,7 +100,21 @@ def retrieve_package_info(sources, checksum, package_name, config,
         print(colors.RED + f"No matching hashes found" + colors.RESET)
     return {}
 
-def retrieve_package(sources, package_info, package_name, config,
+# Does not verify the package itself, will only blindly accept the best size it can
+def query_package_size(sources, package_info, package_name, config, verbose=False):
+    sources_list=config["dir"]["sources"]
+    for source in get_best_source(sources, sources_list=sources_list):
+        url = sources[source]
+        if verbose:
+            print(colors.LIGHT_BLACK + f"using source {source} at {url} for {package_name}")
+
+        package_url = util.add_path(url, package_name + ".xipkg")
+        size = util.query_size(package_url)
+        if size > 0:
+            return size
+    return 0
+
+def retrieve_package(sources, package_info, package_name, config, completed=0, total_download=-1,
                             verbose=False, skip_verification=False):
 
     sources_list=config["dir"]["sources"]
@@ -114,8 +131,15 @@ def retrieve_package(sources, package_info, package_name, config,
         package_dir = util.add_path(cache_dir, source)
 
         util.mkdir(package_dir)
-        # TODO if exists maybe just use cached version
-        status, package_path = util.curl_to_file(package_url, util.add_path(package_dir, package_name + ".xipkg"), text=package_name + ".xipkg")
+
+        if total_download == -1:
+            text = package_name + ".xipkg"
+        else:
+            text = "packages..."
+
+        # TODO if package already downloaded maybe just use cached version
+        status, package_path, size = util.curl_to_file(package_url, util.add_path(package_dir, package_name + ".xipkg"),
+                start=completed, total=total_download, text=text)
 
         if status == 200:
             downloaded_checksum = util.md5sum(package_path)
@@ -125,8 +149,7 @@ def retrieve_package(sources, package_info, package_name, config,
                     sig = verify_signature(package_path, package_info, 
                             cache_dir=cache_dir, keychain_dir=keychain_dir, verbose=verbose)
                     if len(sig) > 0:
-                        print(colors.RESET)
-                        return package_path, source, sig
+                        return package_path, source, sig, size
                     elif verbose:
                         print(colors.RED 
                                 + f"Failed to verify signature for {package_name} in {source}" 
@@ -136,8 +159,7 @@ def retrieve_package(sources, package_info, package_name, config,
                                 + f"Checksum verification failed for {package_name} in {source}" 
                                 + colors.RESET)
             else:
-                print(colors.RESET)
-                return package_path, source, "none"
+                return package_path, source, "none", size
     print(colors.RESET + colors.RED + f"No valid packages found for {package_name}" + colors.RESET)
     return ""
 
@@ -171,15 +193,13 @@ def find_all_dependencies(package_names, options, config):
     # this is all assuming that the order of deps installed doesn't matter
     to_check = [p for p in package_names]
     all_deps = []
+    failed = []
 
     while len(to_check) > 0:
         util.loading_bar(len(all_deps), len(all_deps) + len(to_check), "Resolving dependencies...")
         dep = to_check.pop()
         
-        # probably better way to implement this obligatory wildcard
-        # 100% sure there is a better way of doing this than installing all packages from a repo
-        # maybe some sort of package grouping (or empty package with deps on all needed)
-        dep_checksum, dep_sources, dep_repo = find_package(dep, config["repos"], config["dir"]["packages"], config["sources"])
+        dep_checksum, dep_sources, dep_repo, size, files = find_package(dep, config["repos"], config["dir"]["packages"], config["sources"])
 
         if dep_checksum is not None:
             info = retrieve_package_info(
@@ -191,15 +211,18 @@ def find_all_dependencies(package_names, options, config):
                 if not dep in all_deps:
                     all_deps.append(dep)
                     deps = resolve_dependencies(info)
-                    for dep in deps:
-                        if not dep in all_deps:
-                            if is_installed(dep, config, options["r"]):
+                    for d in deps:
+                        if not d in all_deps:
+                            if is_installed(d, config, options["r"]):
                                 if options["v"]: print(colors.YELLOW + f"Package {dep} has already been installed")
                             else:
-                                to_check.append(dep)
-            elif options["v"]:
-                    util.print_reset(colors.CLEAR_LINE + colors.RED + f"Failed to retrieve info for {dep}")
+                                to_check.append(d)
+            else:
+                if not dep in failed: failed.append(dep)
+                if options["v"]:
+                        util.print_reset(colors.CLEAR_LINE + colors.RED + f"Failed to retrieve info for {dep}")
         else:
+            if not dep in failed: failed.append(dep)
             if options["v"]: util.print_reset(colors.CLEAR_LINE + colors.RED + f"Failed to find package {dep}")
 
     if len(all_deps) > 0:
@@ -209,7 +232,7 @@ def find_all_dependencies(package_names, options, config):
     # assuming that the latter packages are core dependencies
     # we can reverse the array to reflect the more important packages to install
     all_deps.reverse()
-    return all_deps
+    return all_deps, failed
 
 def is_installed(package_name, config, root="/"):
     installed_dir = util.add_path(root, config["dir"]["installed"])
@@ -227,6 +250,7 @@ def install_package(package_name, package_path, package_info,
     if post_install:
         run_post_install(config, verbose=verbose, root=root)
     save_installed_info(package_name, package_info, files, repo, source_url, key, config, root=root)
+    return files
 
     
 
@@ -261,7 +285,20 @@ def save_installed_info(package_name, package_info,
     with open(files_file, "w") as file:
         file.write(files)
 
-    pass
+
+def run_post_install(config, verbose=False, root="/"):
+    installed_dir = util.add_path(root, config["dir"]["postinstall"])
+    if os.path.exists(installed_dir):
+        files = os.listdir(installed_dir)
+        for file in files:
+            f = util.add_path(config["dir"]["postinstall"], file)
+            command = f"sh {f}"
+            if root != "/":
+                os.chroot(root)
+            os.chdir("/")
+            os.system(command)
+            os.remove(f)
+
 
 def install_single(package, options, config, post_install=True, verbose=False, unsafe=False):
     checksum, sources, repo = find_package(package, config["repos"],
@@ -280,18 +317,6 @@ def install_single(package, options, config, post_install=True, verbose=False, u
             repo, sources[source], key, post_install,
             config, verbose=verbose, root=options["r"])
 
-def run_post_install(config, verbose=False, root="/"):
-    installed_dir = util.add_path(root, config["dir"]["postinstall"])
-    if os.path.exists(installed_dir):
-        files = os.listdir(installed_dir)
-        for file in files:
-            f = util.add_path(config["dir"]["postinstall"], file)
-            command = f"sh {f}"
-            if root != "/":
-                os.chroot(root)
-            os.chdir("/")
-            os.system(command)
-            os.remove(f)
 
 def install(args, options, config):
     if not options["l"]:
@@ -308,27 +333,89 @@ def install(args, options, config):
     # have some interaction with sudo when necessary rather than always require it
     # this check may need to be done sooner?
     if util.is_root() or options["r"] != "/":
-        to_install = args if options["n"] else find_all_dependencies(args, options, config)
+        to_install, location_failed = args, []
+        if not options["n"]:
+            to_install, location_failed = find_all_dependencies(args, options, config)
 
-        if len(to_install) > 0:
-            print(colors.CLEAR_LINE + colors.RESET, end="")
-            print(colors.BLUE + "The following packages will be installed:")
+        if len(location_failed) > 0:
+            print(colors.LIGHT_RED + "Failed to locate the following packages:")
             print(end="\t")
-            for d in to_install:
-                print(colors.BLUE if d in args else colors.LIGHT_BLUE, d, end="")
+            for d in location_failed:
+                print(colors.RED if d in args else colors.LIGHT_RED, d, end="")
             print()
 
-            if util.ask_confirmation(colors.BLUE + "Continue?", no_confirm=options["y"]):
+        if len(to_install) > 0:
+            length = 0
+            total_files = 0
+            infos = []
+            for package in to_install:
+                util.loading_bar(len(infos), len(to_install), "Gathering package infos")
+                checksum, sources, repo, size, filecount = find_package(package, config["repos"],
+                        config["dir"]["packages"], config["sources"])
+
+                if checksum != None:
+                    info = retrieve_package_info(
+                                sources, checksum, package, config,
+                                verbose=v, skip_verification=unsafe
+                        )
+
+                    # TODO make package_size be written in package info or sync list instead
+                    length += int(size)
+                    total_files += int(filecount)
+
+                    infos.append(
+                            (package, sources, repo, info)
+                            )
+
+            divisor, unit = util.get_unit(length)
+
+            util.loading_bar(len(infos), len(to_install), "Gathered package infos")
+            print(colors.RESET)
+
+            if not options["y"]:
+                print(colors.BLUE + "The following packages will be installed:")
+                print(end="\t")
+                for d in to_install:
+                    print(colors.BLUE if d in args else colors.LIGHT_BLUE, d, end="")
+                print()
+
+                print(colors.BLUE + "Total download size: " + colors.LIGHT_BLUE + str(round(length / divisor, 2)) + unit)
+
+            if options["y"] or util.ask_confirmation(colors.BLUE + "Continue?"):
+                # TODO try catch over each package in each stage so that we can know if there are errors
+
+                downloaded = 0
+                pkg_files = []
+                for package_info in infos:
+                    (package, sources, repo, info) = package_info
+
+                    package_path, source, key, size = retrieve_package(sources, 
+                            info, package, config, 
+                            completed=downloaded, total_download=length,
+                            verbose=v, skip_verification=unsafe)
+
+                    downloaded += size
+
+                    pkg_files.append(
+                            (package, package_path, sources[source], key, repo, info)
+                            )
+                
+                util.loading_bar(int(length/divisor), int(length/divisor), "Downloaded packages", unit=unit)
+                print(colors.RESET)
 
-                for package in to_install:
-                    try:
-                        install_single(package, options, config, verbose=v, unsafe=unsafe)
-                        util.fill_line(f"Installed {package}", colors.BG_CYAN + colors.LIGHT_BLACK, end="\n")
-                    except Exception as e:
-                        util.fill_line(f"Failed to install {package}", colors.BG_RED + colors.LIGHT_BLACK, end="\n")
-                        util.fill_line(str(e), colors.CLEAR_LINE + colors.RESET + colors.RED, end="\n")
+                extracted = 0
+                for f in pkg_files:
+                    util.loading_bar(extracted, total_files, "Installing files")
 
+                    (package, package_path, source, key, repo, info) = f
 
+                    files = install_package(package, package_path, info, 
+                            repo, source, key, True,
+                            config, verbose=v, root=options["r"])
+                    extracted += len(files.split("\n"))
+
+                util.loading_bar(extracted, total_files, "Installed files")
+                print(colors.RESET)
             else:
                 print(colors.RED + "Action cancelled by user")
         else:
diff --git a/src/verbs/sync.py b/src/verbs/sync.py
index f47e0db..866a821 100644
--- a/src/verbs/sync.py
+++ b/src/verbs/sync.py
@@ -6,6 +6,9 @@ import time
 
 CACHE_DIR = "/var/cache/xipkg"
 
+# returns a dictionary, and duration:
+#   key: package name
+#   value: list of info [checksum, size]
 def list_packages(url):
     start = time.time()
     status, response = util.curl(url + "/packages.list")
@@ -15,12 +18,13 @@ def list_packages(url):
     else:
         duration /= len(response)
         return {
-                line.split()[0].split(".")[0]: line.split()[1]
+                line.split()[0].split(".")[0]: " ".join(line.split()[1:])
                 for line in response.split("\n") if len(line.split()) >  0
                 }, duration
         
+
 def sync_packages(repo, sources, verbose=False):
-    packages = {}
+    versions = {}
 
     speeds = {}
     for source,url in sources.items():
@@ -29,24 +33,26 @@ def sync_packages(repo, sources, verbose=False):
         if speed > 0:
             speeds[source] = speed
 
-        if len(listed) == 0 and verbose:
-            print(colors.RED + f"No packages found in {source}/{repo}" + colors.RESET)
+        if verbose:
+            if len(listed) == 0:
+                print(colors.RED + f"No packages found in {source}/{repo}" + colors.RESET)
+            else:
+                print(colors.BLACK + f"{len(listed)} packages found in {source}/{repo}" + colors.RESET)
 
         for p in listed:
-            if not p in packages:
-                packages[p] = []
-            packages[p].append((listed[p], source))
+            if not p in versions: versions[p] = []
+            versions[p].append((listed[p], source))
 
-    return packages, speeds
+    return versions, speeds
 
 def validate_package(package, versions, repo, verbose=False):
     popularity = {}
     for v in versions:
-        checksum = v[0]
+        info = v[0]
         source = v[1]
-        if not checksum in popularity:
-            popularity[checksum] = 0
-        popularity[checksum] += 1
+        if not info in popularity:
+            popularity[info] = 0
+        popularity[info] += 1
 
     most_popular = ""
     p_count = -1
@@ -56,11 +62,16 @@ def validate_package(package, versions, repo, verbose=False):
             p_count = c
 
     sources = [v[1] for v in versions if v[0] == most_popular]
+    
     # change the packages dict to list all the sources
-    return {
-            "checksum": most_popular,
+    # maybe some validation here
+    info = {
+            "checksum": most_popular.split()[0],
+            "size": most_popular.split()[1],
+            "files": most_popular.split()[2],
             "sources" : sources
             }
+    return info
 
 def save_package(package, info, location):
     util.mkdir(location)
@@ -75,6 +86,8 @@ def save_package(package, info, location):
     content = ""
     with open(package_file, "w") as file:
         file.write("checksum=" + info["checksum"] + "\n")
+        file.write("size=" + info["size"] + "\n")
+        file.write("files=" + info["files"] + "\n")
         file.write("sources=" + " ".join([source for source in info["sources"]]))
 
     return exists
@@ -139,7 +152,7 @@ def sync(args, options, config):
         # find the most popular hash to use
         done = 0 
         total = len(packages.items())
-        for package,versions in packages.items():
+        for package, versions in packages.items():
             info = validate_package(package, versions, repo, verbose=v)
             if not save_package(package, info, repo_dir):
                 new += 1