diff options
| -rw-r--r-- | Makefile | 7 | ||||
| -rw-r--r-- | default.conf | 6 | ||||
| -rw-r--r-- | notes.txt | 33 | ||||
| -rw-r--r-- | src/colors.py | 10 | ||||
| -rw-r--r-- | src/config.py | 2 | ||||
| -rw-r--r-- | src/options.py | 5 | ||||
| -rw-r--r-- | src/util.py | 37 | ||||
| -rw-r--r-- | src/verbs/sync.py | 121 | ||||
| -rw-r--r-- | src/xi.py | 2 | ||||
| -rw-r--r-- | xipkg.conf (renamed from example.conf) | 2 |
10 files changed, 150 insertions, 75 deletions
@@ -8,3 +8,10 @@ xi: src/xi.py rm src/xi.zip chmod +x ${XI_BINARY} +install: xi xipkg.conf default.conf bin/xi + mkdir -p /etc/xipkg.d/ + cp default.conf /etc/xipkg.d/ + cp xipkg.conf /etc/xipkg.conf + cp bin/xi /usr/bin/xi + + diff --git a/default.conf b/default.conf new file mode 100644 index 0000000..1a2d6bc --- /dev/null +++ b/default.conf @@ -0,0 +1,6 @@ +dir { + packages /var/lib/xipkg/packages + keychain /var/lib/xipkg/keychain + installed /var/lib/xipkg/installed + cache /var/cache/xipkg +} @@ -32,37 +32,12 @@ place to store publickeys for sources. sync ------- -download all xipkg.info and keys from repos, done before all other commands -just go to each source -and each repo -one by one take each xipkg.info -temporarily save each xipkg.info somewhere, per source +go to each source and list packages in a repo +find the most popular package hashes -find the latest xipkg.info and get the VER_HASH for that build -find all of the xipkg.info with that VER_HASH and compare their checksums -the most common CHECKSUM will win, so place that xipkg.info in the correct place -? maybe make a warning about any sources that have outdated / different CHECKSUMS for that version hash (possibly malicious?) - -A database should be made: - -Packages: - [name (String), repo (String)] - -Sources: - [name (String)], url (String) - -PackageSources: - package (String), repo (String), source (String) - -This database should solve the problem of of which package infos come from which sources - - -??? -File: - path (String), package (String), repo (String) -??? +download all keys install ------- @@ -119,6 +94,8 @@ upgrade do a sync if a package is specified, only upgrade that one package +packages need updates when the hash listed in the package sync is different to the installed + create a dependency tree and try upgrade core packages first (shouldnt matter really, since upgrading shouldnt have any dependencies other than xipkg and its subdependencies) to upgrade a package find that package in the repo and compare its pkginfo with the installed's package info diff --git a/src/colors.py b/src/colors.py index 760c3f3..c05d49b 100644 --- a/src/colors.py +++ b/src/colors.py @@ -19,3 +19,13 @@ LIGHT_BLUE = esc(94) LIGHT_MAGENTA = esc(95) LIGHT_CYAN = esc(96) LIGHT_WHITE = esc(97) + +BG_BLACK = esc(40) +BG_RED = esc(41) +BG_GREEN = esc(42) +BG_YELLOW = esc(43) +BG_BLUE = esc(44) +BG_MAGENTA = esc(45) +BG_CYAN = esc(46) +BG_WHITE = esc(47) +BG_DEFAULT = esc(49) diff --git a/src/config.py b/src/config.py index c98b076..b7cf915 100644 --- a/src/config.py +++ b/src/config.py @@ -78,7 +78,7 @@ def _parse_line(line, config_file): # if starting with include, then include another file in the same config if key == "include": - included = parse_conf(value) + included = parse_file(value) return included elif value[-1].endswith("{"): return {key: _parse_config(config_file)} diff --git a/src/options.py b/src/options.py index c910ef2..4c419f2 100644 --- a/src/options.py +++ b/src/options.py @@ -22,6 +22,11 @@ options = { "flag" : True, "desc" : "skip syncing with repo sources (not recommended)" }, + "v": { + "name" : "verbose", + "flag" : True, + "desc" : "print more" + }, "c": { "name" : "config", "flag" : False, diff --git a/src/util.py b/src/util.py new file mode 100644 index 0000000..de05b08 --- /dev/null +++ b/src/util.py @@ -0,0 +1,37 @@ +import shutil +import requests +import colors +import time +import os + +def loading_bar(completed, total, text, + unit=""): + + columns, rows = shutil.get_terminal_size((80, 20)) + + count = f"[{completed}{unit}/{total}{unit}]" + + spaces = columns - (len(count) + len(text)) + info = text + "".join([" " for i in range(spaces)]) + count + + + reset_at = int((completed/total)*len(info)) if total > 0 else 0 + info = "".join([info[i] + (colors.RESET if i == reset_at else "") for i in range(len(info))]) + + print(colors.BLACK + colors.BG_GREEN + info, end="\r") + + + + +def curl(url): + r = requests.get(url) + return r.status_code, r.text + +def mkdir(path): + if not os.path.exists(path): + os.makedirs(path) + +if __name__ == "__main__": + for i in range(1000): + loading_bar(i, 1000, "it is loading...") + time.sleep(0.01) diff --git a/src/verbs/sync.py b/src/verbs/sync.py index 95c96f1..fa78112 100644 --- a/src/verbs/sync.py +++ b/src/verbs/sync.py @@ -1,61 +1,92 @@ import os -import requests +import util +import colors +import time -TEMP_DIR = "/tmp/xipkg" +CACHE_DIR = "/var/cache/xipkg" -def curl(url): - r = requests.get(url) - return r.status_code, r.text - -def mkdir(path): - if not os.path.exists(path): - os.makedirs(path) +def list_packages(url): + status, response = util.curl(url + "/packages.list") + if status != 200: + return {} + else: + return { + line.split()[0].split(".")[0]: line.split()[1] + for line in response.split("\n") if len(line.split()) > 0 + } + +def sync_packages(repo, sources): + packages = {} + total = 0 + completed = 0 + for source,url in sources.items(): -def download_repo(output, url): - pkg_list_url = url + "/packages.txt" + listed = list_packages(url + repo if url[-1] == "/" else f"/{repo}") + total += len(listed) + for p in listed: + if not p in packages: + packages[p] = [] - status, response = curl(pkg_list_url) - if status == 404: - print("repo does not exist at", pkg_list_url) - else: - packages = response.split("\n") - for package in packages: - if len(package) > 0: - pkg_url = url + "/" + package - status, package_info = curl(pkg_url) + packages[p].append((listed[p], source)) + completed += 1 + util.loading_bar(completed, total, f"Syncing {repo}") + return packages - if status == 200: - with open(os.path.join(output, package), "w") as file: - file.write(package_info) - else: - print("package is missing at", pkg_url) +def validate_packages(packages, repo): + output = {} + completed = 0 + total = len(packages) + for package, versions in packages.items(): + popularity = {} + for v in versions: + checksum = v[0] + source = v[1] + if not checksum in popularity: + popularity[checksum] = 0 + popularity[checksum] += 1 -# have separate list and download methods for each scheme -def sync_package_infos(source_name, url, repos): + most_popular = sorted(popularity)[0] + + # change the packages dict to list all the sources + output[package] = { + "checksum": most_popular, + "sources" : [v[1] for v in versions if v[0] == most_popular] + } + completed += 1 + util.loading_bar(completed, total, f"Validating {repo}") + return output - source_dir = os.path.join(TEMP_DIR, source_name) +def save_package_list(validated, location): + util.mkdir(location) + for package,info in validated.items(): + package_file = os.path.join(location, package) + + content = "" + with open(package_file, "w") as file: + file.write("checksum=" + info["checksum"] + "\n") + file.write("sources=" + " ".join([source for source in info["sources"]])) - scheme = url.split(":")[0] - print(url) - # TODO: add ftp - if scheme.startswith("http"): - sync_func = download_repo - else: - # Assume its a location on the file system - sync_func = copy_repo - for repo in repos: - out = os.path.join(TEMP_DIR, repo) - mkdir(out) - sync_func(out, url + repo if url[-1] == "/" else f"/{repo}") - -def sync(options, config): +def sync(args, options, config): sources = config["sources"] repos = config["repos"] - mkdir(TEMP_DIR) - for source, url in sources.items(): - sync_package_infos(source, url, repos) + for repo in repos: + packages = sync_packages(repo, sources) + + # find the most popular hash to use + validated = validate_packages(packages, repo) + + save_package_list(validated, os.path.join(config["dir"]["packages"], repo)) + + num_packages = len(validated) + util.loading_bar(num_packages, num_packages, f"Synced {repo}") + print(colors.RESET) + + + + + print("Synced!") @@ -32,7 +32,7 @@ def main(): ( verbs[verb] if verb in verbs else search )( - args[1:] if len(args) > 1 else [], conf + args[1:] if len(args) > 1 else [], opts, conf ) else: options.print_usage() diff --git a/example.conf b/xipkg.conf index b947794..418eefe 100644 --- a/example.conf +++ b/xipkg.conf @@ -1,4 +1,6 @@ # This is the default configuration for xipkg +# +include /etc/xipkg.d/default.conf sources { davidovski https://xi.davidovski.xyz/repo/ |