From 0ff24f5611f8a30f01ae08d866b5276c020a8665 Mon Sep 17 00:00:00 2001 From: davidovski Date: Wed, 17 Nov 2021 12:20:12 +0000 Subject: added server pinging to determine best mirror to use --- default.conf | 2 ++ src/options.py | 2 +- src/verbs/install.py | 21 +++++++++++++++++++-- src/verbs/sync.py | 34 +++++++++++++++++++++++++++++++++- xipkg.conf | 5 +++++ 5 files changed, 60 insertions(+), 4 deletions(-) diff --git a/default.conf b/default.conf index 1a2d6bc..6fac4b2 100644 --- a/default.conf +++ b/default.conf @@ -1,6 +1,8 @@ dir { packages /var/lib/xipkg/packages keychain /var/lib/xipkg/keychain + sources /var/lib/xipkg/sources installed /var/lib/xipkg/installed cache /var/cache/xipkg } + diff --git a/src/options.py b/src/options.py index 704d630..910b8a3 100644 --- a/src/options.py +++ b/src/options.py @@ -26,7 +26,7 @@ options = { "name" : "unsafe", "flag" : True, "desc" : "skip any checksum or signature verification" - } + }, "v": { "name" : "verbose", "flag" : True, diff --git a/src/verbs/install.py b/src/verbs/install.py index 5883c51..f29142c 100644 --- a/src/verbs/install.py +++ b/src/verbs/install.py @@ -20,7 +20,6 @@ def find_package(query, repos, packages_dir): return checksum, sources, requested_repo return None, [], None - def retrieve_package_info(sources, checksum, package_name, verbose=False, skip_verification=False): for source,url in sources.items(): @@ -40,6 +39,25 @@ def retrieve_package_info(sources, checksum, package_name, print(colors.RED + f"No matching hashes found" + colors.RESET) return {} +def retrieve_package(sources, checksum, package_name, + verbose=False, skip_verification=False): + for source,url in sources.items(): + package_info_url = util.add_path(url, package_name + ".xipkg.info") + status, response = util.curl(package_info_url) + + if status == 200: + info = parse_package_info(response) + if info["CHECKSUM"] == checksum or skip_verification: + return info + else: + if verbose: + print(colors.RED + + f"Checksum verification failed for {package_name} in {source}" + + colors.RESET) + if verbose: + print(colors.RED + f"No matching hashes found" + colors.RESET) + return {} + def parse_package_info(packageinfo): info = {} @@ -49,7 +67,6 @@ def parse_package_info(packageinfo): info[split[0]] = "=".join(split[1:]) return info - def install(args, options, config): sources = config["sources"] repos = config["repos"] diff --git a/src/verbs/sync.py b/src/verbs/sync.py index 443a698..e9cc7b3 100644 --- a/src/verbs/sync.py +++ b/src/verbs/sync.py @@ -86,12 +86,44 @@ def import_key(source, url, verbose=False): elif verbose: print(colors.BG_RED + f"" + colors.RESET) + +def test_source(source, url): + start = time.time() + util.curl(util.add_path(url, "index.html")) + duration = time.time() - start + return int(duration * 1000) + +def test_sources(sources, file_path, test_count=10): + if test_count > 0: + pings = {} + checked = 0 + for source,url in sources.items(): + total = 0 + for i in range(test_count): + total += test_source(source, url) + util.loading_bar(checked, len(sources) * test_count, f"Pinging Sources") + checked += 1 + pings[source] = total / test_count if total > 0 else 0 + + + sorted(pings, reverse=True) + + with open(file_path, "w") as file: + for source,ping in pings.items(): + file.write(f"{source} {ping}\n") + + util.loading_bar(checked, len(sources) * test_count, f"Pinged Sources") + print() + + def sync(args, options, config): sources = config["sources"] repos = config["repos"] v = options["v"] + test_sources(sources, config["dir"]["sources"], test_count=int(config["pings"])) + for repo in repos: packages = sync_packages(repo, sources, verbose=v) @@ -103,7 +135,7 @@ def sync(args, options, config): num_packages = len(validated) util.loading_bar(num_packages, num_packages, f"Synced {repo}") print(colors.RESET) - + #total = len(sources) #completed = 0 diff --git a/xipkg.conf b/xipkg.conf index 418eefe..5121484 100644 --- a/xipkg.conf +++ b/xipkg.conf @@ -4,6 +4,7 @@ include /etc/xipkg.d/default.conf sources { davidovski https://xi.davidovski.xyz/repo/ + codeberg https://xi.codeberg.page/repo/ } repos [ @@ -11,3 +12,7 @@ repos [ extra ] +# the number of times to ping a server to find out its speed +# use 0 to disable determining the fastest servers +pings 10 + -- cgit v1.2.1 From 28029bc25090e706d08b6c5735bb5dfc24176f59 Mon Sep 17 00:00:00 2001 From: davidovski Date: Wed, 17 Nov 2021 12:23:24 +0000 Subject: neatened ping number to int --- src/verbs/sync.py | 2 +- xipkg.conf | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/verbs/sync.py b/src/verbs/sync.py index e9cc7b3..bd828ef 100644 --- a/src/verbs/sync.py +++ b/src/verbs/sync.py @@ -103,7 +103,7 @@ def test_sources(sources, file_path, test_count=10): total += test_source(source, url) util.loading_bar(checked, len(sources) * test_count, f"Pinging Sources") checked += 1 - pings[source] = total / test_count if total > 0 else 0 + pings[source] = int(total / test_count) if total > 0 else 0 sorted(pings, reverse=True) diff --git a/xipkg.conf b/xipkg.conf index 5121484..07d1801 100644 --- a/xipkg.conf +++ b/xipkg.conf @@ -14,5 +14,5 @@ repos [ # the number of times to ping a server to find out its speed # use 0 to disable determining the fastest servers -pings 10 +pings 4 -- cgit v1.2.1