From a4618ee489a1cc393e195d61e965bd41885e8f14 Mon Sep 17 00:00:00 2001 From: davidovski Date: Tue, 9 Nov 2021 17:36:30 +0000 Subject: synced packages from repos to list --- src/colors.py | 10 +++++ src/config.py | 2 +- src/options.py | 5 +++ src/util.py | 37 +++++++++++++++++ src/verbs/sync.py | 121 ++++++++++++++++++++++++++++++++++-------------------- src/xi.py | 2 +- 6 files changed, 130 insertions(+), 47 deletions(-) create mode 100644 src/util.py (limited to 'src') diff --git a/src/colors.py b/src/colors.py index 760c3f3..c05d49b 100644 --- a/src/colors.py +++ b/src/colors.py @@ -19,3 +19,13 @@ LIGHT_BLUE = esc(94) LIGHT_MAGENTA = esc(95) LIGHT_CYAN = esc(96) LIGHT_WHITE = esc(97) + +BG_BLACK = esc(40) +BG_RED = esc(41) +BG_GREEN = esc(42) +BG_YELLOW = esc(43) +BG_BLUE = esc(44) +BG_MAGENTA = esc(45) +BG_CYAN = esc(46) +BG_WHITE = esc(47) +BG_DEFAULT = esc(49) diff --git a/src/config.py b/src/config.py index c98b076..b7cf915 100644 --- a/src/config.py +++ b/src/config.py @@ -78,7 +78,7 @@ def _parse_line(line, config_file): # if starting with include, then include another file in the same config if key == "include": - included = parse_conf(value) + included = parse_file(value) return included elif value[-1].endswith("{"): return {key: _parse_config(config_file)} diff --git a/src/options.py b/src/options.py index c910ef2..4c419f2 100644 --- a/src/options.py +++ b/src/options.py @@ -22,6 +22,11 @@ options = { "flag" : True, "desc" : "skip syncing with repo sources (not recommended)" }, + "v": { + "name" : "verbose", + "flag" : True, + "desc" : "print more" + }, "c": { "name" : "config", "flag" : False, diff --git a/src/util.py b/src/util.py new file mode 100644 index 0000000..de05b08 --- /dev/null +++ b/src/util.py @@ -0,0 +1,37 @@ +import shutil +import requests +import colors +import time +import os + +def loading_bar(completed, total, text, + unit=""): + + columns, rows = shutil.get_terminal_size((80, 20)) + + count = f"[{completed}{unit}/{total}{unit}]" + + spaces = columns - (len(count) + len(text)) + info = text + "".join([" " for i in range(spaces)]) + count + + + reset_at = int((completed/total)*len(info)) if total > 0 else 0 + info = "".join([info[i] + (colors.RESET if i == reset_at else "") for i in range(len(info))]) + + print(colors.BLACK + colors.BG_GREEN + info, end="\r") + + + + +def curl(url): + r = requests.get(url) + return r.status_code, r.text + +def mkdir(path): + if not os.path.exists(path): + os.makedirs(path) + +if __name__ == "__main__": + for i in range(1000): + loading_bar(i, 1000, "it is loading...") + time.sleep(0.01) diff --git a/src/verbs/sync.py b/src/verbs/sync.py index 95c96f1..fa78112 100644 --- a/src/verbs/sync.py +++ b/src/verbs/sync.py @@ -1,61 +1,92 @@ import os -import requests +import util +import colors +import time -TEMP_DIR = "/tmp/xipkg" +CACHE_DIR = "/var/cache/xipkg" -def curl(url): - r = requests.get(url) - return r.status_code, r.text - -def mkdir(path): - if not os.path.exists(path): - os.makedirs(path) +def list_packages(url): + status, response = util.curl(url + "/packages.list") + if status != 200: + return {} + else: + return { + line.split()[0].split(".")[0]: line.split()[1] + for line in response.split("\n") if len(line.split()) > 0 + } + +def sync_packages(repo, sources): + packages = {} + total = 0 + completed = 0 + for source,url in sources.items(): -def download_repo(output, url): - pkg_list_url = url + "/packages.txt" + listed = list_packages(url + repo if url[-1] == "/" else f"/{repo}") + total += len(listed) + for p in listed: + if not p in packages: + packages[p] = [] - status, response = curl(pkg_list_url) - if status == 404: - print("repo does not exist at", pkg_list_url) - else: - packages = response.split("\n") - for package in packages: - if len(package) > 0: - pkg_url = url + "/" + package - status, package_info = curl(pkg_url) + packages[p].append((listed[p], source)) + completed += 1 + util.loading_bar(completed, total, f"Syncing {repo}") + return packages - if status == 200: - with open(os.path.join(output, package), "w") as file: - file.write(package_info) - else: - print("package is missing at", pkg_url) +def validate_packages(packages, repo): + output = {} + completed = 0 + total = len(packages) + for package, versions in packages.items(): + popularity = {} + for v in versions: + checksum = v[0] + source = v[1] + if not checksum in popularity: + popularity[checksum] = 0 + popularity[checksum] += 1 -# have separate list and download methods for each scheme -def sync_package_infos(source_name, url, repos): + most_popular = sorted(popularity)[0] + + # change the packages dict to list all the sources + output[package] = { + "checksum": most_popular, + "sources" : [v[1] for v in versions if v[0] == most_popular] + } + completed += 1 + util.loading_bar(completed, total, f"Validating {repo}") + return output - source_dir = os.path.join(TEMP_DIR, source_name) +def save_package_list(validated, location): + util.mkdir(location) + for package,info in validated.items(): + package_file = os.path.join(location, package) + + content = "" + with open(package_file, "w") as file: + file.write("checksum=" + info["checksum"] + "\n") + file.write("sources=" + " ".join([source for source in info["sources"]])) - scheme = url.split(":")[0] - print(url) - # TODO: add ftp - if scheme.startswith("http"): - sync_func = download_repo - else: - # Assume its a location on the file system - sync_func = copy_repo - for repo in repos: - out = os.path.join(TEMP_DIR, repo) - mkdir(out) - sync_func(out, url + repo if url[-1] == "/" else f"/{repo}") - -def sync(options, config): +def sync(args, options, config): sources = config["sources"] repos = config["repos"] - mkdir(TEMP_DIR) - for source, url in sources.items(): - sync_package_infos(source, url, repos) + for repo in repos: + packages = sync_packages(repo, sources) + + # find the most popular hash to use + validated = validate_packages(packages, repo) + + save_package_list(validated, os.path.join(config["dir"]["packages"], repo)) + + num_packages = len(validated) + util.loading_bar(num_packages, num_packages, f"Synced {repo}") + print(colors.RESET) + + + + + print("Synced!") diff --git a/src/xi.py b/src/xi.py index 9e723de..0b11c56 100644 --- a/src/xi.py +++ b/src/xi.py @@ -32,7 +32,7 @@ def main(): ( verbs[verb] if verb in verbs else search )( - args[1:] if len(args) > 1 else [], conf + args[1:] if len(args) > 1 else [], opts, conf ) else: options.print_usage() -- cgit v1.2.1