synced packages from repos to list

6 files changed, 130 insertions, 47 deletions

diff --git a/src/colors.py b/src/colors.py
index 760c3f3..c05d49b 100644
--- a/src/colors.py
+++ b/src/colors.py
@@ -19,3 +19,13 @@ LIGHT_BLUE      = esc(94)
 LIGHT_MAGENTA   = esc(95)
 LIGHT_CYAN      = esc(96)
 LIGHT_WHITE     = esc(97)
+
+BG_BLACK           = esc(40)
+BG_RED             = esc(41)
+BG_GREEN           = esc(42)
+BG_YELLOW          = esc(43)
+BG_BLUE            = esc(44)
+BG_MAGENTA         = esc(45)
+BG_CYAN            = esc(46)
+BG_WHITE           = esc(47)
+BG_DEFAULT         = esc(49)
diff --git a/src/config.py b/src/config.py
index c98b076..b7cf915 100644
--- a/src/config.py
+++ b/src/config.py
@@ -78,7 +78,7 @@ def _parse_line(line, config_file):
 
         # if starting with include, then include another file in the same config
         if key == "include":
-            included = parse_conf(value)
+            included = parse_file(value)
             return included
         elif value[-1].endswith("{"):
             return {key: _parse_config(config_file)}
diff --git a/src/options.py b/src/options.py
index c910ef2..4c419f2 100644
--- a/src/options.py
+++ b/src/options.py
@@ -22,6 +22,11 @@ options = {
                 "flag" : True,
                 "desc" : "skip syncing with repo sources (not recommended)"
             },
+        "v": {
+                "name" : "verbose",
+                "flag" : True,
+                "desc" : "print more"
+            },
         "c": {
                 "name" : "config",
                 "flag" : False,
diff --git a/src/util.py b/src/util.py
new file mode 100644
index 0000000..de05b08
--- /dev/null
+++ b/src/util.py
@@ -0,0 +1,37 @@
+import shutil
+import requests
+import colors
+import time
+import os
+
+def loading_bar(completed, total, text, 
+        unit=""):
+
+    columns, rows = shutil.get_terminal_size((80, 20))
+    
+    count = f"[{completed}{unit}/{total}{unit}]"
+    
+    spaces = columns - (len(count) + len(text))
+    info = text +  "".join([" " for i in range(spaces)]) + count
+
+
+    reset_at = int((completed/total)*len(info)) if total > 0 else 0
+    info = "".join([info[i] + (colors.RESET if i == reset_at else "") for i in range(len(info))]) 
+
+    print(colors.BLACK + colors.BG_GREEN + info, end="\r")
+
+
+
+
+def curl(url):
+    r = requests.get(url)
+    return r.status_code, r.text
+
+def mkdir(path):
+    if not os.path.exists(path):
+        os.makedirs(path)
+
+if __name__ == "__main__":
+    for i in range(1000):
+        loading_bar(i, 1000, "it is loading...")
+        time.sleep(0.01)
diff --git a/src/verbs/sync.py b/src/verbs/sync.py
index 95c96f1..fa78112 100644
--- a/src/verbs/sync.py
+++ b/src/verbs/sync.py
@@ -1,61 +1,92 @@
 import os
-import requests
+import util
+import colors
+import time
 
-TEMP_DIR = "/tmp/xipkg"
+CACHE_DIR = "/var/cache/xipkg"
 
-def curl(url):
-    r = requests.get(url)
-    return r.status_code, r.text
-
-def mkdir(path):
-    if not os.path.exists(path):
-        os.makedirs(path)
+def list_packages(url):
+    status, response = util.curl(url + "/packages.list")
+    if status != 200:
+        return {}
+    else:
+        return {
+                line.split()[0].split(".")[0]: line.split()[1]
+                for line in response.split("\n") if len(line.split()) >  0
+                }
+        
+def sync_packages(repo, sources):
+    packages = {}
+    total = 0
+    completed = 0
+    for source,url in sources.items():
 
-def download_repo(output, url):
-    pkg_list_url = url + "/packages.txt"
+        listed = list_packages(url + repo if url[-1] == "/" else f"/{repo}")
+        total += len(listed)
+        for p in listed:
+            if not p in packages:
+                packages[p] = []
 
-    status, response = curl(pkg_list_url)
-    if status == 404:
-        print("repo does not exist at", pkg_list_url)
-    else:
-        packages = response.split("\n")
-        for package in packages:
-            if len(package) > 0:
-                pkg_url = url + "/" + package
-                status, package_info = curl(pkg_url)
+            packages[p].append((listed[p], source))
+            completed += 1
+            util.loading_bar(completed, total, f"Syncing {repo}")
+    return packages
 
-                if status == 200:
-                    with open(os.path.join(output, package), "w") as file:
-                        file.write(package_info)
-                else:
-                    print("package is missing at", pkg_url)
+def validate_packages(packages, repo):
+    output = {}
+    completed = 0
+    total = len(packages)
+    for package, versions in packages.items():
+        popularity = {}
+        for v in versions:
+            checksum = v[0]
+            source = v[1]
+            if not checksum in popularity:
+                popularity[checksum] = 0
+            popularity[checksum] += 1
 
-# have separate list and download methods for each scheme
-def sync_package_infos(source_name, url, repos):
+        most_popular = sorted(popularity)[0]
+        
+        # change the packages dict to list all the sources
+        output[package] = {
+                "checksum": most_popular,
+                "sources" : [v[1] for v in versions if v[0] == most_popular]
+                }
+        completed += 1
+        util.loading_bar(completed, total, f"Validating {repo}")
+    return output
 
-    source_dir = os.path.join(TEMP_DIR, source_name)
+def save_package_list(validated, location):
+    util.mkdir(location)
+    for package,info in validated.items():
+        package_file = os.path.join(location, package)
+        
+        content = ""
+        with open(package_file, "w") as file:
+            file.write("checksum=" + info["checksum"] + "\n")
+            file.write("sources=" + " ".join([source for source in info["sources"]]))
 
-    scheme = url.split(":")[0]
     
-    print(url)
-    # TODO: add ftp
-    if scheme.startswith("http"):
-        sync_func = download_repo
-    else:
-        # Assume its a location on the file system
-        sync_func = copy_repo
 
-    for repo in repos:
-        out = os.path.join(TEMP_DIR, repo)
-        mkdir(out)
-        sync_func(out, url + repo if url[-1] == "/" else f"/{repo}")
 
-        
-def sync(options, config):
+def sync(args, options, config):
     sources = config["sources"]
     repos = config["repos"]
 
-    mkdir(TEMP_DIR)
-    for source, url in sources.items():
-        sync_package_infos(source, url, repos)
+    for repo in repos:
+        packages = sync_packages(repo, sources)
+        
+        # find the most popular hash to use
+        validated = validate_packages(packages, repo)
+
+        save_package_list(validated, os.path.join(config["dir"]["packages"], repo))
+
+        num_packages = len(validated)
+        util.loading_bar(num_packages, num_packages, f"Synced {repo}")
+        print(colors.RESET)
+
+            
+            
+            
+
     print("Synced!")
diff --git a/src/xi.py b/src/xi.py
index 9e723de..0b11c56 100644
--- a/src/xi.py
+++ b/src/xi.py
@@ -32,7 +32,7 @@ def main():
         (
             verbs[verb] if verb in verbs else search
         )(
-            args[1:] if len(args) > 1 else [],  conf
+            args[1:] if len(args) > 1 else [], opts, conf
         )
     else:
         options.print_usage()