From 2c82b81fcd8fc723712ac605041da5142f765730 Mon Sep 17 00:00:00 2001 From: davidovski Date: Sun, 6 Feb 2022 00:40:22 +0000 Subject: moved src files to xibuild files --- build.sh | 9 + sitegen/build_site.sh | 15 ++ src/build.sh | 73 ------ src/build_order.pm | 134 ---------- src/build_package.sh | 228 ----------------- src/make_tools.sh | 612 -------------------------------------------- src/prepare_environment.sh | 32 --- xib_env | 30 +++ xibuild/build_all.sh | 73 ++++++ xibuild/build_order.pm | 134 ++++++++++ xibuild/build_package.sh | 205 +++++++++++++++ xibuild/make_infos.sh | 65 +++++ xibuild/make_tools.sh | 619 +++++++++++++++++++++++++++++++++++++++++++++ 13 files changed, 1150 insertions(+), 1079 deletions(-) create mode 100755 build.sh create mode 100644 sitegen/build_site.sh delete mode 100755 src/build.sh delete mode 100755 src/build_order.pm delete mode 100755 src/build_package.sh delete mode 100755 src/make_tools.sh delete mode 100755 src/prepare_environment.sh create mode 100755 xib_env create mode 100755 xibuild/build_all.sh create mode 100755 xibuild/build_order.pm create mode 100755 xibuild/build_package.sh create mode 100755 xibuild/make_infos.sh create mode 100755 xibuild/make_tools.sh diff --git a/build.sh b/build.sh new file mode 100755 index 0000000..3f424b9 --- /dev/null +++ b/build.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +source xib_env + +cd xibuild +./build_all.sh || exit 1 +./make_infos.sh + + diff --git a/sitegen/build_site.sh b/sitegen/build_site.sh new file mode 100644 index 0000000..8daef8c --- /dev/null +++ b/sitegen/build_site.sh @@ -0,0 +1,15 @@ +#!/bin/bash +# builds the html files required for the xilinux website + +generate_index () { + envsubst < templ/index.html +} + +generate_package_list() { + local repo_dir=$1 + for info_file in $repo_dir/*.xipkg.info; do + + done; +} + +"$XIB_CHROOT"/ diff --git a/src/build.sh b/src/build.sh deleted file mode 100755 index a01368b..0000000 --- a/src/build.sh +++ /dev/null @@ -1,73 +0,0 @@ -#!/bin/bash - -ERROR="\033[0;31m" -INFO="\033[0;34m" -PASS="\033[0;32m" -NEUTRAL="\033[0;33m" -RESET="\033[0m" - -source prepare_environment.sh - -build_package () { - name=$(echo $line | cut -d"+" -f1) - buildfile=$(find $XIB_BUILDFILES -wholename "*/$name.xibuild" | head -1) - - if [ -f "$buildfile" ]; then - printf $INFO - printf "Building$NEUTRAL %s$INFO:\n$RESET" $name - ./build_package.sh $buildfile || return 1 - - # Install the package if it is needed for other builds - if echo $line | grep -q '+'; then - printf "$INFO\tInstalling..." - exported_pkg=$(find $XIB_EXPORT -wholename "*/$name.xipkg" | head -1 | xargs realpath) - if [ -f $exported_pkg ]; then - tar -h --no-overwrite-dir -xf $exported_pkg -C $XIB_CHROOT - - postinstall="$XIB_CHROOT/var/lib/xipkg/postinstall" - if [ -d $postinstall ]; then - for file in "$postinstall/*.sh"; do - f=$(basename $file) - chmod 755 $file - xichroot "$XIB_CHROOT" "/var/lib/xipkg/postinstall/$f" - rm $file - printf "$PASS run postinstall for $f!\n" - done - rmdir $postinstall - fi - fi - - printf "$PASS installed to chroot!\n" - fi - - printf $RESET - printf "Finished building %s!\n" $name - else - printf "$ERROR$name does not exist\n" - fi - - # configure shadow here - if [ "$name" = "shadow" ]; then - xichroot "$XIB_CHROOT" "/usr/sbin/pwconv" - xichroot "$XIB_CHROOT" "/usr/sbin/grpconv" - xichroot "$XIB_CHROOT" "mkdir -p /etc/default" - xichroot "$XIB_CHROOT" "/usr/sbin/useradd -D --gid 999" - fi -} - -build_all () { - for line in $(perl build_order.pm); do - build_package $line || return 1 - done - -} - -if build_all; then - printf "$PASSBuilt all packages!" -else - printf "$ERROR Something went wrong!$NEUTRAL Press enter to view recent log" - read; - - f=$(ls -1 --sort time $XIB_EXPORT/repo/*/*.log | head -1 | xargs realpath) - less $f -fi diff --git a/src/build_order.pm b/src/build_order.pm deleted file mode 100755 index 606839c..0000000 --- a/src/build_order.pm +++ /dev/null @@ -1,134 +0,0 @@ -#!/usr/bin/env perl - -# TODO for building the whole system -# We first need to have a base system that we can build from -# (up to chapter 6 in lfs book, done by make_tools.sh) -# packages need to be build in order of importance -# need to some how find out which packages are the most important and build and install them to the chroot first -# build a package, install it if necessary -# copy the generated package/info to a safe place -# sign the package -# put the package in the export folder - -# TODO for building a single package: -# do all the preliminary checks (exists, deps are installed, etc) -# download source to $chroot/source -# download additional tools -# copy xibuild to the chroot -# create a "build.sh" script in the chroot -# - run the 3 stages of package building -# - create the pacakage in $chroot/$name.xipkg -# - add some info to package info -# - if requested, install to the chroot - -use strict; -use warnings; - -use Env; -use File::Basename; -use lib dirname (__FILE__); -use Sort::TSort "tsort"; - -our $buildfiles = $ENV{XIB_BUILDFILES}; - -sub list_dependencies{ - my $file = $_; - my @deps = (); - - open (my $fh, "<", $file) or warn "Cannot open $file"; - - while (my $line = <$fh>) { - if ($line =~ /DEPS=\((.+)\)/) { - my @words = split(/ /, $1); - push(@deps, @words); - } - } - - return @deps; -} - -sub list_buildfiles{ - my @files = glob("$buildfiles/repo/*/*.xibuild"); - # ignore any meta packages during this stage, they can be added later - @files = grep(!/\/meta\//, @files); - @files = grep(!/\/skip\//, @files); - - return @files -} - -sub list_meta_pkgs{ - return map({basename($_, ".xibuild")} glob("$buildfiles/repo/meta/*.xibuild")); -} - -sub get_packages{ - my %pkgs = (); - - my @files = list_buildfiles(); - - foreach (@files) { - my $pkg_file = $_; - my $pkg_name = basename($pkg_file, ".xibuild"); - - my @deps = list_dependencies($pkg_file); - $pkgs{$pkg_name} = \@deps; - } - - return %pkgs; -} - -# Get a list of all the edges -sub get_edges{ - my %pkgs = @_; - - my @edges = (); - foreach (keys(%pkgs)) { - my $pkg = $_; - my @deps = @{$pkgs{$_}}; - foreach (@deps) { - my $dep = $_; - - my @edge = ($pkg, $dep); - push @edges, [ @edge ]; - - } - } - return @edges; -} - -# Determine which packages are depended on -sub get_depended_on{ - my @edges = @_; - - my %install = (); - foreach (@edges) { - my @edge = @{$_}; - $install{$edge[1]} = $edge[0]; - } - - return keys(%install); -} - -sub determine_build_order{ - my %pkgs = get_packages(); - - my @edges = get_edges(%pkgs); - - my @install = get_depended_on(@edges); - - # use tsort to determine the build order - my @sorted = reverse(@{tsort(\@edges)}); - - my @meta = list_meta_pkgs(); - push(@sorted, @meta); - - foreach(@sorted) { - my $pkg = $_; - print("$pkg"); - print("+") if (grep(/^$pkg/, @install)); - print("\n"); - } -} - -unless (caller) { - determine_build_order(); -} diff --git a/src/build_package.sh b/src/build_package.sh deleted file mode 100755 index a6d4a33..0000000 --- a/src/build_package.sh +++ /dev/null @@ -1,228 +0,0 @@ -#!/bin/bash - -GREEN="\033[0;32m" -BLUE="\033[0;34m" - -BUILDFILE=$1 -REPO=$(echo $BUILDFILE | rev | cut -d/ -f2 | rev) -NAME=$(basename $BUILDFILE .xibuild) - -source $BUILDFILE - -extract () { - FILE=$1 - case "${FILE##*.}" in - "gz" ) - tar -zxf $FILE - ;; - "lz" ) - tar --lzip -xf "$FILE" - ;; - "zip" ) - unzip $FILE - ;; - * ) - tar -xf $FILE - ;; - esac -} - -package_exists () { - local exported="$XIB_EXPORT/repo/$REPO/$NAME" - local exported_pkg="$exported.xipkg" - local exported_pkg_info="$exported.xipkg.info" - local exported_pkg_build="$exported.xibuild" - - if [ -f $exported_pkg ] && [ -f $exported_pkg_info ] && [ -f $exported_pkg_build ]; then - local built_sum=$(md5sum $exported_pkg_build | cut -d" " -f1) - local current_sum=$(md5sum $BUILDFILE | cut -d" " -f1) - - if [ "$built_sum" = "$current_sum" ]; then - return 0 - fi - fi - - return 1 -} - -fetch_source () { - # download additional files - local src_dir="$XIB_CHROOT/build/source" - mkdir -p $src_dir - - cd $src_dir - - if [ ! -z ${SOURCE} ]; then - - if git ls-remote -q $SOURCE $BRANCH &> /dev/null; then - # The source is a git repo - git clone $SOURCE . &> /dev/null - git checkout $BRANCH &> /dev/null - else - # The source is a file - - local downloaded_file=$(basename $SOURCE) - curl -SsL $SOURCE > $downloaded_file - extract $downloaded_file - - # if the extracted file only had one directory - if [ "$(ls -l | wc -l)" = "3" ]; then - for file in */* */.*; do - echo $file | grep -q '\.$' || mv $file . - done; - fi - fi - fi - - # download additional files - if [ ! -z ${ADDITIONAL} ]; then - for url in ${ADDITIONAL[*]}; do - local name=$(basename $url) - curl -SsL $url > $src_dir/$name - done - fi -} - - -clean_chroot () { - local export_dir="$XIB_CHROOT/export" - local build_dir="$XIB_CHROOT/build" - - rm -rf $export_dir - rm -rf $build_dir - - mkdir -p $export_dir - mkdir -p $build_dir - - mkdir -p "$XIB_EXPORT/repo/$REPO/" -} - -make_buildscript () { - - echo MAKEFLAGS="$MAKEFLAGS" >> "$XIB_CHROOT/build/profile" - echo LDFLAGS="$LDFLAGS" >> "$XIB_CHROOT/build/profile" - - cat > "$XIB_CHROOT/build/build.sh" << "EOF" -#!/bin/bash -source /build/profile -export PKG_NAME=$(cat /build/name) -export PKG_DEST=/export - -prepare () { - echo "passing prepare" -} - -build () { - echo "passing build" -} - -check () { - echo "passing check" -} - -package () { - echo "passing package" -} - -cd /build -ls -source $PKG_NAME.xibuild -cd /build/source - -echo "==========================PREPARE STAGE==========================" -prepare || exit 1 -echo "==========================BUILD STAGE==========================" -build || exit 1 -echo "==========================CHECK STAGE==========================" -check || exit 1 -echo "==========================PACKAGE STAGE==========================" -package || exit 1 - -printf "checking for postinstall... " -if command -v postinstall > /dev/null; then - echo "adding postinstall" - POSTINSTALL=$(type postinstall | sed '1,3d;$d') - if [ ${#POSTINSTALL} != 0 ]; then - POST_DIR=$PKG_DEST/var/lib/xipkg/postinstall - mkdir -p $POST_DIR - echo "#!/bin/sh" > $POST_DIR/$PKG_NAME.sh - echo $POSTINSTALL >> $POST_DIR/$PKG_NAME.sh - fi -else - echo "no postinstall" -fi -EOF - chmod 700 "$XIB_CHROOT/build/build.sh" -} - -package () { - local export_repo="$XIB_EXPORT/repo/$REPO" - local export_pkg="$XIB_EXPORT/repo/$REPO/$NAME.xipkg" - local pkg_dest="$XIB_CHROOT/export" - - cd "$pkg_dest" - if [ "$(ls -1 | wc -l)" = "0" ]; then - printf " package is empty;" - [ -z "${SOURCE}" ] || exit 1; - fi - tar -C $pkg_dest -czf $export_pkg ./ -} - -create_info () { - local export_pkg="$XIB_EXPORT/repo/$REPO/$NAME.xipkg" - local pkg_info="$export_pkg.info" - - echo "" > $pkg_info - echo "NAME=$NAME" >> $pkg_info - echo "DESCRIPTION=$DESC" >> $pkg_info - echo "PKG_FILE=$NAME.xipkg" >> $pkg_info - echo "CHECKSUM=$(md5sum $export_pkg | awk '{ print $1 }')" >> $pkg_info - echo "SOURCE=$SOURCE" >> $pkg_info - echo "DATE=$(date)" >> $pkg_info - echo "DEPS=(${DEPS[*]})" >> $pkg_info -} - -sign () { - local export_pkg="$XIB_EXPORT/repo/$REPO/$NAME.xipkg" - local pkg_info="$export_pkg.info" - - echo "SIGNATURE=" >> $pkg_info - openssl dgst -sign $PRIV_KEY $export_pkg >> $pkg_info - -} - -build () { - printf "$BLUE\tCleaning chroot..." - clean_chroot && printf "$GREEN prepared\n" || return 1 - - printf "$BLUE\tfetching source..." - fetch_source && printf "$GREEN fetched $(du -sh "$XIB_CHROOT/build/source" | awk '{ print $1 }')\n" || return 1 - - printf "$BLUE\tgenerating buildscript..." - make_buildscript && printf "$GREEN generated\n" || return 1 - - cp "$BUILDFILE" "$XIB_CHROOT/build/" - printf $NAME > "$XIB_CHROOT/build/name" - - local log_file="$XIB_EXPORT/repo/$REPO/$NAME.log" - - printf "$BLUE\tBuilding package..." - xichroot $XIB_CHROOT /build/build.sh &> $log_file && printf "$GREEN built!\n" || return 1 - - printf "$BLUE\tPackaging package..." - package && printf "$GREEN packaged to $(du -sh "$XIB_EXPORT/repo/$REPO/$NAME.xipkg" | awk '{ print $1 }')!\n" || return 1 - - printf "$BLUE\tCreating package info..." - create_info && printf "$GREEN created info!\n" || return 1 - - # TODO check if the key exists, if not, skip signing - printf "$BLUE\tSigning package..." - sign && printf "$GREEN signed!\n" || return 1 - - cp "$BUILDFILE" "$XIB_EXPORT/repo/$REPO/" -} - -[ -z "${XIB_CHROOT}" ] && echo "CRITICAL! No chroot env variable set!" && exit 1; - -package_exists && printf "\tPackage exists!\n" || build - diff --git a/src/make_tools.sh b/src/make_tools.sh deleted file mode 100755 index e55e3d6..0000000 --- a/src/make_tools.sh +++ /dev/null @@ -1,612 +0,0 @@ -#!/bin/bash -# A small script to generate the chroot environment where building will take place - -export MAKEFLAGS="-j$(grep "processor" /proc/cpuinfo | wc -l)" -export WORKING_DIR="/var/lib/xib" -BUILDFILES_REPO_URL="https://xi.davidovski.xyz/git/buildfiles.git" -export SYSTEM_DIR="$WORKING_DIR/xib-chroot" - -TOOL_DIR="$SYSTEM_DIR/tools" -SOURCES="$SYSTEM_DIR/sources" - -TGT=$(uname -m)-xi-linux-gnu - - -export PATH=/usr/bin -if [ ! -L /bin ]; then export PATH=/bin:$PATH; fi -export PATH=$SYSTEM_DIR/tools/bin:$PATH -export CONFIG_SITE=$SYSTEM_DIR/usr/share/config.site -export LC_ALL=POSIX - - - -packages=(binutils gcc linux glibc mpfr gmp mpc m4 ncurses bash coreutils diffutils file findutils gawk grep gzip make patch sed tar xz) - -get_build_files() { - mkdir -p $WORKING_DIR/buildfiles - git clone "$BUILDFILES_REPO_URL" $WORKING_DIR/buildfiles -} - -list_build_files() { - ls -1 $WORKING_DIR/buildfiles/repo/*/*.xibuild -} - -parse_package_versions() { - for pkg_file in $(list_build_files); do - local pkg_name=$(basename -s .xibuild $pkg_file) - local pkg_ver=$(sed -n "s/^PKG_VER=\(.*\)$/\1/p" $pkg_file) - - [ -z "$pkg_ver" ] && pkg_ver=$(sed -n "s/^BRANCH=\(.*\)$/\1/p" $pkg_file) - [ -z "$pkg_ver" ] && pkg_ver="latest" - printf "%-16s %16s\n" $pkg_name $pkg_ver - done -} - -extract () { - FILE=$1 - case "${FILE#*.}" in - "tar.gz" ) - tar -zxf $FILE - ;; - "tar.lz" ) - tar --lzip -xf "$FILE" - ;; - "zip" ) - unzip $FILE - ;; - * ) - tar -xf $FILE - ;; - esac -} - - -make_dir_struct() { - local system=$1 - - mkdir -pv $system/{etc,var,proc,sys,run,tmp} $system/usr/{bin,lib,sbin} $system/dev/{pts,shm} - - for i in bin lib sbin; do - ln -sv usr/$i $system/$i - done - - case $(uname -m) in - x86_64) mkdir -pv $system/lib64 ;; - esac -} - -extract () { - FILE=$1 - case "${FILE#*.}" in - "tar.gz" ) - tar -zxf $FILE - ;; - "tar.lz" ) - tar --lzip -xf "$FILE" - ;; - "zip" ) - unzip $FILE - ;; - * ) - tar -xf $FILE - ;; - esac -} - -init_versions() { - local versions_file="$SYSTEM_DIR/versions" - - [ -f $versions_file ] || exit 1 - - for i in ${packages[@]}; do - local name=${i^^} - local version=$(sed -n "s/^$i \s*\(.*\)$/\1/p" $versions_file) - eval "${i^^}_VERSION"="$version" - done -} - -get_source() { - local buildfile="$1" - local package_name=$(basename -s .xibuild $buildfile) - - echo "fetching $buildfile" - - if [ ! -d $package_name ]; then - mkdir -p $package_name - cd $package_name - rm -rf * - source $buildfile - - if git ls-remote -q $SOURCE $BRANCH &> /dev/null; then - git clone $SOURCE . - git checkout $BRANCH - - elif hg identify $SOURCE &> /dev/null; then - hg clone $SOURCE package_name . - else - local downloaded=$(basename $SOURCE) - curl -Ls $SOURCE > $downloaded - extract $downloaded - mv */* . - fi - fi -} - -get_sources() { - mkdir -p $SYSTEM_DIR/sources - for pkg in $@; do - local pkg_file="$WORKING_DIR/buildfiles/repo/*/$pkg.xibuild" - cd $SYSTEM_DIR/sources - get_source $pkg_file - done -} - -# builds binutils to toolchain -# -build_binutils1() { - cd $SOURCES/binutils/ - - mkdir -v build - cd build - - ../configure --prefix="$SYSTEM_DIR/tools" \ - --with-sysroot="$SYSTEM_DIR" \ - --target=$TGT \ - --disable-nls \ - --disable-werror && - make && - make install -j1 -} - - -# builds gcc to toolchain -# -build_gcc1() { - cd $SOURCES/gcc/ - - #rm -rf mpfr gmp mpc - - autoreconf -i - - [ -d mpfr ] || cp -r $SOURCES/mpfr mpfr - [ -d gmp ] || cp -r $SOURCES/gmp gmp - [ -d mpc ] || cp -r $SOURCES/mpc mpc - - sed -e '/m64=/s/lib64/lib/' -i.orig gcc/config/i386/t-linux64 - - mkdir -v build - cd build - - ../configure \ - --target=$TGT \ - --prefix=$SYSTEM_DIR/tools \ - --with-glibc-version=$GLIBC_VERSION \ - --with-sysroot=$SYSTEM_DIR \ - --with-newlib \ - --without-headers \ - --without-zstd \ - --enable-initfini-array \ - --disable-nls \ - --disable-shared \ - --disable-multilib \ - --disable-decimal-float \ - --disable-threads \ - --disable-libatomic \ - --disable-libgomp \ - --disable-libquadmath \ - --disable-libssp \ - --disable-libvtv \ - --disable-libstdcxx \ - --enable-languages=c,c++ - - make && - make install || exit 1 - cd .. - cat gcc/limitx.h gcc/glimits.h gcc/limity.h > \ - `dirname $($TGT-gcc -print-libgcc-file-name)`/install-tools/include/limits.h -} - -build_linux_headers() { - cd $SOURCES/linux/ - - make mrproper - - make headers - find usr/include -name '.*' -delete - rm usr/include/Makefile - cp -rv usr/include $SYSTEM_DIR/usr -} - -build_glibc() { - cd $SOURCES/glibc/ - - case $(uname -m) in - i?86) ln -sfv ld-linux.so.2 $SYSTEM_DIR/lib/ld-lsb.so.3 - ;; - x86_64) ln -sfv ../lib/ld-linux-x86-64.so.2 $SYSTEM_DIR/lib64 - ln -sfv ../lib/ld-linux-x86-64.so.2 $SYSTEM_DIR/lib64/ld-lsb-x86-64.so.3 - ;; - esac - - mkdir -v build - cd build - - echo "rootsbindir=/usr/sbin" > configparms - ../configure \ - --prefix=/usr \ - --host=$TGT \ - --build=$(../scripts/config.guess) \ - --enable-kernel=3.2 \ - --with-headers=$SYSTEM_DIR/usr/include \ - libc_cv_slibdir=/usr/lib - - make && - make DESTDIR=$SYSTEM_DIR install - - sed '/RTLDLIST=/s@/usr@@g' -i $SYSTEM_DIR/usr/bin/ldd - - $SYSTEM_DIR/tools/libexec/gcc/$TGT/$GCC_VERSION/install-tools/mkheaders -} - -build_libstdcxx() { - cd $SOURCES/gcc/ - - rm -rf build - mkdir build - cd build - - ../libstdc++-v3/configure \ - --host=$TGT \ - --build=$(../config.guess) \ - --prefix=/usr \ - --disable-multilib \ - --disable-nls \ - --disable-libstdcxx-pch \ - --with-gxx-include-dir=/tools/$SYSTEM_DIR/include/c++/$GCC_VERSION - - make && - make DESTDIR=$SYSTEM_DIR install -} - -build_m4() { - cd $SOURCES/m4/ - ./configure --prefix=/usr \ - --host=$TGT \ - --build=$(build-aux/config.guess) - - make - make DESTDIR=$SYSTEM_DIR install -} - -build_ncurses() { - cd $SOURCES/ncurses/ - - sed -i s/mawk// configure - mkdir build - - pushd build - ../configure - make -C include - make -C progs tic - popd - - ./configure --prefix=/usr \ - --host=$TGT \ - --build=$(./config.guess) \ - --mandir=/usr/share/man \ - --with-manpage-format=normal \ - --with-shared \ - --without-debug \ - --without-ada \ - --without-normal \ - --disable-stripping \ - --enable-widec - - make - make DESTDIR=$SYSTEM_DIR TIC_PATH=$(pwd)/build/progs/tic install - echo "INPUT(-lncursesw)" > $SYSTEM_DIR/usr/lib/libncurses.so -} - -build_bash() { - cd $SOURCES/bash/ - - ./configure --prefix=/usr \ - --build=$(support/config.guess) \ - --host=$TGT \ - --without-bash-malloc - - make - make DESTDIR=$SYSTEM_DIR install - ln -sv bash $SYSTEM_DIR/bin/sh -} - -build_coreutils() { - cd $SOURCES/coreutils/ - - ./configure --prefix=/usr \ - --host=$TGT \ - --build=$(build-aux/config.guess) \ - --enable-install-program=hostname \ - --enable-no-install-program=kill,uptime - - make - make DESTDIR=$SYSTEM_DIR install - - mv -v $SYSTEM_DIR/usr/bin/chroot $SYSTEM_DIR/usr/sbin - mkdir -pv $SYSTEM_DIR/usr/share/man/man8 - mv -v $SYSTEM_DIR/usr/share/man/man1/chroot.1 $SYSTEM_DIR/usr/share/man/man8/chroot.8 - sed -i 's/"1"/"8"/' $SYSTEM_DIR/usr/share/man/man8/chroot.8 -} - -build_diffutils() { - cd $SOURCES/diffutils/ - - ./configure --prefix=/usr --host=$TGT - - make - make DESTDIR=$SYSTEM_DIR install -} - -build_file() { - cd $SOURCES/file/ - - mkdir build - pushd build - ../configure --disable-bzlib \ - --disable-libseccomp \ - --disable-xzlib \ - --disable-zlib - make - popd - - ./configure --prefix=/usr --host=$TGT --build=$(./config.guess) - - make FILE_COMPILE=$(pwd)/build/src/file - make DESTDIR=$SYSTEM_DIR install -} - -build_findutils() { - cd $SOURCES/findutils/ - - ./configure --prefix=/usr \ - --localstatedir=/var/lib/locate \ - --host=$TGT \ - --build=$(build-aux/config.guess) - make - make DESTDIR=$SYSTEM_DIR install -} - -build_gawk() { - cd $SOURCES/gawk/ - sed -i 's/extras//' Makefile.in - ./configure --prefix=/usr \ - --host=$TGT \ - --build=$(build-aux/config.guess) - make - make DESTDIR=$SYSTEM_DIR install -} - -build_grep() { - cd $SOURCES/grep - -./configure --prefix=/usr \ - --host=$TGT -make -make DESTDIR=$SYSTEM_DIR install -} - -build_gzip() { - cd $SOURCES/gzip/ - ./configure --prefix=/usr --host=$TGT - make - make DESTDIR=$SYSTEM_DIR install -} - -build_make() { - cd $SOURCES/make/ - ./configure --prefix=/usr \ - --without-guile \ - --host=$TGT \ - --build=$(build-aux/config.guess) - make - make DESTDIR=$SYSTEM_DIR install -} - -build_patch() { - cd $SOURCES/patch/ - - ./configure --prefix=/usr \ - --host=$TGT \ - --build=$(build-aux/config.guess) - make - make DESTDIR=$SYSTEM_DIR install -} - -build_sed() { - cd $SOURCES/sed/ - - ./configure --prefix=/usr --host=$TGT - - make - make DESTDIR=$SYSTEM_DIR install -} - -build_tar() { - cd $SOURCES/tar/ - ./configure --prefix=/usr \ - --host=$TGT \ - --build=$(build-aux/config.guess) - make - make DESTDIR=$SYSTEM_DIR install -} - -build_xz() { - cd $SOURCES/xz/ - - ./configure --prefix=/usr \ - --host=$TGT \ - --build=$(build-aux/config.guess) \ - --disable-static \ - --docdir=/usr/share/doc/xz-$XZ_VERSION - make - make DESTDIR=$SYSTEM_DIR install -} - -build_binutils2() { - cd $SOURCES/binutils/ - make clean - rm -rf build - mkdir -v build - cd build - ../configure \ - --prefix=/usr \ - --build=$(../config.guess) \ - --host=$TGT \ - --disable-nls \ - --enable-shared \ - --disable-werror \ - --enable-64-bit-bfd - make - make DESTDIR=$SYSTEM_DIR install -j1 - install -vm755 libctf/.libs/libctf.so.0.0.0 $SYSTEM_DIR/usr/lib -} - -build_gcc2() { - cd $SOURCES/gcc/ - - rm -rf build - mkdir -v build - cd build - - mkdir -pv $TGT/libgcc - ln -s ../../../libgcc/gthr-posix.h $TGT/libgcc/gthr-default.h - ../configure \ - --build=$(../config.guess) \ - --host=$TGT \ - --prefix=/usr \ - CC_FOR_TARGET=$TGT-gcc \ - --with-build-sysroot=$SYSTEM_DIR \ - --enable-initfini-array \ - --disable-nls \ - --disable-multilib \ - --disable-decimal-float \ - --disable-libatomic \ - --disable-libgomp \ - --disable-libquadmath \ - --disable-libssp \ - --disable-libvtv \ - --disable-libstdcxx \ - --enable-languages=c,c++ - make - make DESTDIR=$SYSTEM_DIR install - ln -sv gcc $SYSTEM_DIR/usr/bin/cc -} - -if [ -e $SYSTEM_DIR ]; then - printf "Remove existing system? [Y/n] " - read response - - if [ "$response" != "n" ]; then - rm -rf $SYSTEM_DIR - echo "removed $SYSTEM_DIR" - fi -fi - - -mkdir -p "$SYSTEM_DIR" -get_build_files -parse_package_versions > "$SYSTEM_DIR/versions" - -get_sources ${packages[@]} - -make_dir_struct $SYSTEM_DIR -init_versions - -reset_before_build () { - cd $SYSTEM_DIR; printf "\033[0;34mbuilding $1...\033[0m" -} - -for package in ${packages[@]}; do - cd $SYSTEM_DIR - touch $package.build.log -done - -reset_before_build "binutils1" -build_binutils1 &> binutils.build.log && printf "passed\n" || exit 1 - -reset_before_build "gcc1" -build_gcc1 &> gcc1.build.log && printf "passed\n" || exit 1 - -reset_before_build "linux_headers" -build_linux_headers &> linux-headers.build.log && printf "\033[0;32mpassed\n" || exit 1 - -reset_before_build "glibc" -build_glibc &> glibc.build.log && printf "\033[0;32mpassed\n" || exit 1 - -reset_before_build "libstdcxx" -build_libstdcxx &> libstdcxx.build.log && printf "\033[0;32mpassed\n" || exit 1 - -reset_before_build "m4" -build_m4 &> m4.build.log && printf "\033[0;32mpassed\n" || exit 1 - -reset_before_build "ncurses" -build_ncurses &> ncurses.build.log && printf "\033[0;32mpassed\n" || exit 1 - -reset_before_build "bash" -build_bash &> bash.build.log && printf "\033[0;32mpassed\n" || exit 1 - -reset_before_build "coreutils" -build_coreutils &> coreutils.build.log && printf "\033[0;32mpassed\n" || exit 1 - -reset_before_build "diffutils" -build_diffutils &> diffutils.build.log && printf "\033[0;32mpassed\n" || exit 1 - -reset_before_build "file" -build_file &> file.build.log && printf "\033[0;32mpassed\n" || exit 1 - -reset_before_build "findutils" -build_findutils &> findutils.build.log && printf "\033[0;32mpassed\n" || exit 1 - -reset_before_build "gawk" -build_gawk &> gawk.build.log && printf "\033[0;32mpassed\n" || exit 1 - -reset_before_build "grep" -build_grep &> grep.build.log && printf "\033[0;32mpassed\n" || exit 1 - -reset_before_build "gzip" -build_gzip &> gzip.build.log && printf "\033[0;32mpassed\n" || exit 1 - -reset_before_build "make" -build_make &> make.build.log && printf "\033[0;32mpassed\n" || exit 1 - -reset_before_build "patch" -build_patch &> patch.build.log && printf "\033[0;32mpassed\n" || exit 1 - -reset_before_build "sed" -build_sed &> sed.build.log && printf "\033[0;32mpassed\n" || exit 1 - -reset_before_build "tar" -build_tar &> tar.build.log && printf "\033[0;32mpassed\n" || exit 1 - -reset_before_build "xz" -build_xz &> xz.build.log && printf "\033[0;32mpassed\n" || exit 1 - -reset_before_build "binutils2" -build_binutils2 &> bintuils2.build.log && printf "\033[0;32mpassed\n" || exit 1 - -reset_before_build "gcc2" -build_gcc2 &> gcc2.build.log && printf "\033[0;32mpassed\n" || exit 1 - - -echo "DONE" - -printf "Clean sources and logs? [Y/n]" -read response - -if [ "$response" != "n" ]; then - rm -rf $SYSTEM_DIR/*.log - rm -rf $SYSTEM_DIR/sources - echo "removed $SYSTEM_DIR/sources" -fi - - diff --git a/src/prepare_environment.sh b/src/prepare_environment.sh deleted file mode 100755 index 1f4952d..0000000 --- a/src/prepare_environment.sh +++ /dev/null @@ -1,32 +0,0 @@ -#!/bin/sh - -export MAKEFLAGS="-j$(grep "processor" /proc/cpuinfo | wc -l)" -export LDFLAGS="-liconv" -#export LDFLAGS="" - -export XIB_DIR="/var/lib/xib" -export XIB_BUILDFILES="$XIB_DIR/buildfiles" -export XIB_CHROOT="$XIB_DIR/chroot" -export XIB_EXPORT="$XIB_DIR/export" - -export PRIV_KEY="/home/david/.ssh/xi.pem" -export DEVELOPMENT_BUILDFILES="/home/david/docs/proj/xilinux/buildfiles" - -export BUILDFILES_GIT_REPO="https://xi.davidovski.xyz/git/buildfiles.git" - -mkdir -p $XIB_DIR $XIB_BUILDFILES $XIB_CHROOT $XIB_EXPORT - -if [ -d $DEVELOPMENT_BUILDFILES ]; then - cp -r $DEVELOPMENT_BUILDFILES/* $XIB_BUILDFILES/ -else - if [ -d $XIB_BUILDFILES/.git ]; then - cd $XIB_BUILDFILES - git pull - cd $OLDPWD - else - git clone $BUILDFILES_GIT_REPO $XIB_BUILDFILES - fi -fi - - -cp /etc/resolv.conf $XIB_CHROOT/etc/resolv.conf diff --git a/xib_env b/xib_env new file mode 100755 index 0000000..c74a6fb --- /dev/null +++ b/xib_env @@ -0,0 +1,30 @@ +#!/bin/sh + +export MAKEFLAGS="-j$(grep "processor" /proc/cpuinfo | wc -l)" + +export XIB_DIR="/var/lib/xib" +export XIB_BUILDFILES="$XIB_DIR/buildfiles" +export XIB_CHROOT="$XIB_DIR/chroot" +export XIB_EXPORT="$XIB_DIR/export" + +export PRIV_KEY="/home/david/.ssh/xi.pem" +export DEVELOPMENT_BUILDFILES="/home/david/docs/proj/xilinux/buildfiles" + +export BUILDFILES_GIT_REPO="https://xi.davidovski.xyz/git/buildfiles.git" + +mkdir -p $XIB_DIR $XIB_BUILDFILES $XIB_CHROOT $XIB_EXPORT + +if [ -d $DEVELOPMENT_BUILDFILES ]; then + cp -r $DEVELOPMENT_BUILDFILES/* $XIB_BUILDFILES/ +else + if [ -d $XIB_BUILDFILES/.git ]; then + cd $XIB_BUILDFILES + git pull + cd $OLDPWD + else + git clone $BUILDFILES_GIT_REPO $XIB_BUILDFILES + fi +fi + + +cp /etc/resolv.conf $XIB_CHROOT/etc/resolv.conf diff --git a/xibuild/build_all.sh b/xibuild/build_all.sh new file mode 100755 index 0000000..275e50b --- /dev/null +++ b/xibuild/build_all.sh @@ -0,0 +1,73 @@ +#!/bin/bash + +ERROR="\033[0;31m" +INFO="\033[0;34m" +PASS="\033[0;32m" +NEUTRAL="\033[0;33m" +RESET="\033[0m" + +build_package () { + name=$(echo $line | cut -d"+" -f1) + buildfile=$(find $XIB_BUILDFILES -wholename "*/$name.xibuild" | head -1) + + if [ -f "$buildfile" ]; then + printf $INFO + printf "Building$NEUTRAL %s$INFO:\n$RESET" $name + ./build_package.sh $buildfile || return 1 + + # Install the package if it is needed for other builds + if echo $line | grep -q '+'; then + printf "$INFO\tInstalling..." + exported_pkg=$(find $XIB_EXPORT -wholename "*/$name.xipkg" | head -1 | xargs realpath) + if [ -f $exported_pkg ]; then + tar -h --no-overwrite-dir -xf $exported_pkg -C $XIB_CHROOT + + postinstall="$XIB_CHROOT/var/lib/xipkg/postinstall" + if [ -d $postinstall ]; then + for file in "$postinstall/*.sh"; do + f=$(basename $file) + chmod 755 $file + xichroot "$XIB_CHROOT" "/var/lib/xipkg/postinstall/$f" + rm $file + printf "$PASS run postinstall for $f!\n" + done + rmdir $postinstall + fi + fi + + printf "$PASS installed to chroot!\n" + fi + + printf $RESET + printf "Finished building %s!\n" $name + else + printf "$ERROR$name does not exist\n" + fi + + # configure shadow here + if [ "$name" = "shadow" ]; then + xichroot "$XIB_CHROOT" "/usr/sbin/pwconv" + xichroot "$XIB_CHROOT" "/usr/sbin/grpconv" + xichroot "$XIB_CHROOT" "mkdir -p /etc/default" + xichroot "$XIB_CHROOT" "/usr/sbin/useradd -D --gid 999" + fi +} + +build_all () { + for line in $(perl build_order.pm); do + build_package $line || return 1 + done + +} + +if build_all; then + printf "\n${PASS}Built all packages!" + exit 0 +else + printf "$ERROR Something went wrong!$NEUTRAL Press enter to view recent log" + read; + + f=$(ls -1 --sort time $XIB_EXPORT/repo/*/*.log | head -1 | xargs realpath) + less $f + exit 1 +fi diff --git a/xibuild/build_order.pm b/xibuild/build_order.pm new file mode 100755 index 0000000..606839c --- /dev/null +++ b/xibuild/build_order.pm @@ -0,0 +1,134 @@ +#!/usr/bin/env perl + +# TODO for building the whole system +# We first need to have a base system that we can build from +# (up to chapter 6 in lfs book, done by make_tools.sh) +# packages need to be build in order of importance +# need to some how find out which packages are the most important and build and install them to the chroot first +# build a package, install it if necessary +# copy the generated package/info to a safe place +# sign the package +# put the package in the export folder + +# TODO for building a single package: +# do all the preliminary checks (exists, deps are installed, etc) +# download source to $chroot/source +# download additional tools +# copy xibuild to the chroot +# create a "build.sh" script in the chroot +# - run the 3 stages of package building +# - create the pacakage in $chroot/$name.xipkg +# - add some info to package info +# - if requested, install to the chroot + +use strict; +use warnings; + +use Env; +use File::Basename; +use lib dirname (__FILE__); +use Sort::TSort "tsort"; + +our $buildfiles = $ENV{XIB_BUILDFILES}; + +sub list_dependencies{ + my $file = $_; + my @deps = (); + + open (my $fh, "<", $file) or warn "Cannot open $file"; + + while (my $line = <$fh>) { + if ($line =~ /DEPS=\((.+)\)/) { + my @words = split(/ /, $1); + push(@deps, @words); + } + } + + return @deps; +} + +sub list_buildfiles{ + my @files = glob("$buildfiles/repo/*/*.xibuild"); + # ignore any meta packages during this stage, they can be added later + @files = grep(!/\/meta\//, @files); + @files = grep(!/\/skip\//, @files); + + return @files +} + +sub list_meta_pkgs{ + return map({basename($_, ".xibuild")} glob("$buildfiles/repo/meta/*.xibuild")); +} + +sub get_packages{ + my %pkgs = (); + + my @files = list_buildfiles(); + + foreach (@files) { + my $pkg_file = $_; + my $pkg_name = basename($pkg_file, ".xibuild"); + + my @deps = list_dependencies($pkg_file); + $pkgs{$pkg_name} = \@deps; + } + + return %pkgs; +} + +# Get a list of all the edges +sub get_edges{ + my %pkgs = @_; + + my @edges = (); + foreach (keys(%pkgs)) { + my $pkg = $_; + my @deps = @{$pkgs{$_}}; + foreach (@deps) { + my $dep = $_; + + my @edge = ($pkg, $dep); + push @edges, [ @edge ]; + + } + } + return @edges; +} + +# Determine which packages are depended on +sub get_depended_on{ + my @edges = @_; + + my %install = (); + foreach (@edges) { + my @edge = @{$_}; + $install{$edge[1]} = $edge[0]; + } + + return keys(%install); +} + +sub determine_build_order{ + my %pkgs = get_packages(); + + my @edges = get_edges(%pkgs); + + my @install = get_depended_on(@edges); + + # use tsort to determine the build order + my @sorted = reverse(@{tsort(\@edges)}); + + my @meta = list_meta_pkgs(); + push(@sorted, @meta); + + foreach(@sorted) { + my $pkg = $_; + print("$pkg"); + print("+") if (grep(/^$pkg/, @install)); + print("\n"); + } +} + +unless (caller) { + determine_build_order(); +} diff --git a/xibuild/build_package.sh b/xibuild/build_package.sh new file mode 100755 index 0000000..44521a9 --- /dev/null +++ b/xibuild/build_package.sh @@ -0,0 +1,205 @@ +#!/bin/bash + +GREEN="\033[0;32m" +BLUE="\033[0;34m" + +BUILDFILE=$1 +REPO=$(echo $BUILDFILE | rev | cut -d/ -f2 | rev) +NAME=$(basename $BUILDFILE .xibuild) + +source $BUILDFILE + +extract () { + FILE=$1 + case "${FILE##*.}" in + "gz" ) + tar -zxf $FILE + ;; + "lz" ) + tar --lzip -xf "$FILE" + ;; + "zip" ) + unzip $FILE + ;; + * ) + tar -xf $FILE + ;; + esac +} + +package_exists () { + local exported="$XIB_EXPORT/repo/$REPO/$NAME" + local exported_pkg="$exported.xipkg" + local exported_pkg_info="$exported.xipkg.info" + local exported_pkg_build="$exported.xibuild" + + if [ -f $exported_pkg ] && [ -f $exported_pkg_build ]; then + local built_sum=$(md5sum $exported_pkg_build | cut -d" " -f1) + local current_sum=$(md5sum $BUILDFILE | cut -d" " -f1) + + if [ "$built_sum" = "$current_sum" ]; then + return 0 + fi + fi + + return 1 +} + +fetch_source () { + # download additional files + local src_dir="$XIB_CHROOT/build/source" + mkdir -p $src_dir + + cd $src_dir + + if [ ! -z ${SOURCE} ]; then + + if git ls-remote -q $SOURCE $BRANCH &> /dev/null; then + # The source is a git repo + git clone $SOURCE . &> /dev/null + git checkout $BRANCH &> /dev/null + else + # The source is a file + + local downloaded_file=$(basename $SOURCE) + curl -SsL $SOURCE > $downloaded_file + extract $downloaded_file + + # if the extracted file only had one directory + if [ "$(ls -l | wc -l)" = "3" ]; then + for file in */* */.*; do + echo $file | grep -q '\.$' || mv $file . + done; + fi + fi + fi + + # download additional files + if [ ! -z ${ADDITIONAL} ]; then + for url in ${ADDITIONAL[*]}; do + local name=$(basename $url) + curl -SsL $url > $src_dir/$name + done + fi +} + + +clean_chroot () { + local export_dir="$XIB_CHROOT/export" + local build_dir="$XIB_CHROOT/build" + + rm -rf $export_dir + rm -rf $build_dir + + mkdir -p $export_dir + mkdir -p $build_dir + + mkdir -p "$XIB_EXPORT/repo/$REPO/" +} + +make_buildscript () { + + echo MAKEFLAGS="$MAKEFLAGS" >> "$XIB_CHROOT/build/profile" + echo LDFLAGS="$LDFLAGS" >> "$XIB_CHROOT/build/profile" + + cat > "$XIB_CHROOT/build/build.sh" << "EOF" +#!/bin/bash +source /build/profile +export PKG_NAME=$(cat /build/name) +export PKG_DEST=/export + +prepare () { + echo "passing prepare" +} + +build () { + echo "passing build" +} + +check () { + echo "passing check" +} + +package () { + echo "passing package" +} + +cd /build +ls +source $PKG_NAME.xibuild +cd /build/source + +echo "==========================PREPARE STAGE==========================" +prepare || exit 1 +echo "==========================BUILD STAGE==========================" +build || exit 1 +echo "==========================CHECK STAGE==========================" +check || exit 1 +echo "==========================PACKAGE STAGE==========================" +package || exit 1 + +printf "checking for postinstall... " +if command -v postinstall > /dev/null; then + echo "adding postinstall" + POSTINSTALL=$(type postinstall | sed '1,3d;$d') + if [ ${#POSTINSTALL} != 0 ]; then + POST_DIR=$PKG_DEST/var/lib/xipkg/postinstall + mkdir -p $POST_DIR + echo "#!/bin/sh" > $POST_DIR/$PKG_NAME.sh + echo $POSTINSTALL >> $POST_DIR/$PKG_NAME.sh + fi +else + echo "no postinstall" +fi +EOF + chmod 700 "$XIB_CHROOT/build/build.sh" +} + +package () { + local export_repo="$XIB_EXPORT/repo/$REPO" + local export_pkg="$XIB_EXPORT/repo/$REPO/$NAME.xipkg" + local pkg_dest="$XIB_CHROOT/export" + + cd "$pkg_dest" + if [ "$(ls -1 | wc -l)" = "0" ]; then + printf " package is empty;" + [ -z "${SOURCE}" ] || exit 1; + fi + tar -C $pkg_dest -czf $export_pkg ./ +} + +build () { + printf "$BLUE\tCleaning chroot..." + clean_chroot && printf "$GREEN prepared\n" || return 1 + + printf "$BLUE\tfetching source..." + fetch_source && printf "$GREEN fetched $(du -sh "$XIB_CHROOT/build/source" | awk '{ print $1 }')\n" || return 1 + + printf "$BLUE\tgenerating buildscript..." + make_buildscript && printf "$GREEN generated\n" || return 1 + + cp "$BUILDFILE" "$XIB_CHROOT/build/" + printf $NAME > "$XIB_CHROOT/build/name" + + local log_file="$XIB_EXPORT/repo/$REPO/$NAME.log" + + printf "$BLUE\tBuilding package..." + xichroot $XIB_CHROOT /build/build.sh &> $log_file && printf "$GREEN built!\n" || return 1 + + printf "$BLUE\tPackaging package..." + package && printf "$GREEN packaged to $(du -sh "$XIB_EXPORT/repo/$REPO/$NAME.xipkg" | awk '{ print $1 }')!\n" || return 1 + + printf "$BLUE\tCreating package info..." + create_info && printf "$GREEN created info!\n" || return 1 + + # TODO check if the key exists, if not, skip signing + printf "$BLUE\tSigning package..." + sign && printf "$GREEN signed!\n" || return 1 + + cp "$BUILDFILE" "$XIB_EXPORT/repo/$REPO/" +} + +[ -z "${XIB_CHROOT}" ] && echo "CRITICAL! No chroot env variable set!" && exit 1; + +package_exists && printf "\tPackage exists!\n" || build + diff --git a/xibuild/make_infos.sh b/xibuild/make_infos.sh new file mode 100755 index 0000000..4817e6a --- /dev/null +++ b/xibuild/make_infos.sh @@ -0,0 +1,65 @@ +#!/bin/bash + + +# TODO remember to update this if there are ever changes +XIPKG_INFO_VERSION='02' + +get_info() { + local name=$(basename -s ".xipkg" $1) + + local pkg_ver=$PKG_VER + [ -z "$pkg_ver" ] && pkg_ver=$BRANCH + [ -z "$pkg_ver" ] && pkg_ver="latest" + + echo "# XiPKG info file version $XIPKG_INFO_VERSION" + echo "# automatically generated from the built packages" + echo "NAME=$name" + echo "DESCRIPTION=$DESC" + echo "PKG_FILE=$name.xipkg" + echo "CHECKSUM=$(md5sum $1 | awk '{ print $1 }')" + echo "VERSION=$pkg_ver" + echo "SOURCE=$SOURCE" + echo "DATE=$(date -r $1)" + echo "DEPS=(${DEPS[*]})" + echo "MAKE_DEPS=(${MAKE_DEPS[*]})" +} + +sign () { + echo "SIGNATURE=" + openssl dgst -sign $PRIV_KEY $1 +} + +list_line() { + local pkg_file=$1 + + local name=$(basename -s ".xipkg" $pkg_file) + local filecount=$(gzip -cd $pkg_file | tar -tvv | grep -c ^-) + local checksum=$(md5sum $pkg_file | awk '{ print $1 }') + local size=$(du -s $pkg_file | awk '{print $1}') + + echo $name.xipkg $checksum $size $filecount +} + + +for repo in $(ls -d "$XIB_EXPORT"/repo/*); do + file="$repo/packages.list" + [ -e $file ] && rm $file + touch $file + echo "Removed old package lists in $repo" +done + +for pkg in $(ls "$XIB_EXPORT"/repo/*/*.xipkg); do + name=$(basename -s ".xipkg" $pkg) + repo=$(echo $pkg | rev | cut -d/ -f2 | rev) + info_file="$XIB_EXPORT/repo/$repo/$name.xipkg.info" + build_file="$XIB_EXPORT/repo/$repo/$name.xibuild" + + source $build_file + + get_info $pkg > $info_file + sign $pkg >> $info_file + list_line $pkg >> "$XIB_EXPORT"/repo/$repo/packages.list + echo "Enlisted $name to $info_file" + + +done diff --git a/xibuild/make_tools.sh b/xibuild/make_tools.sh new file mode 100755 index 0000000..10b70b5 --- /dev/null +++ b/xibuild/make_tools.sh @@ -0,0 +1,619 @@ +#!/bin/bash +# A small script to generate the chroot environment where building will take place + +export MAKEFLAGS="-j$(grep "processor" /proc/cpuinfo | wc -l)" +export WORKING_DIR="/var/lib/xib" +BUILDFILES_REPO_URL="https://xi.davidovski.xyz/git/buildfiles.git" +export SYSTEM_DIR="$WORKING_DIR/xib-chroot" + +TOOL_DIR="$SYSTEM_DIR/tools" +SOURCES="$SYSTEM_DIR/sources" + +TGT=$(uname -m)-xi-linux-gnu + + +export PATH=/usr/bin +if [ ! -L /bin ]; then export PATH=/bin:$PATH; fi +export PATH=$SYSTEM_DIR/tools/bin:$PATH +export CONFIG_SITE=$SYSTEM_DIR/usr/share/config.site +export LC_ALL=POSIX + +set +h +umask 022 + + + +packages=(binutils gcc linux glibc mpfr gmp mpc m4 ncurses bash coreutils diffutils file findutils gawk grep gzip make patch sed tar xz) + +get_build_files() { + mkdir -p $WORKING_DIR/buildfiles + git clone "$BUILDFILES_REPO_URL" $WORKING_DIR/buildfiles +} + +list_build_files() { + ls -1 $WORKING_DIR/buildfiles/repo/*/*.xibuild +} + +parse_package_versions() { + for pkg_file in $(list_build_files); do + local pkg_name=$(basename -s .xibuild $pkg_file) + local pkg_ver=$(sed -n "s/^PKG_VER=\(.*\)$/\1/p" $pkg_file) + + [ -z "$pkg_ver" ] && pkg_ver=$(sed -n "s/^BRANCH=\(.*\)$/\1/p" $pkg_file) + [ -z "$pkg_ver" ] && pkg_ver="latest" + printf "%-16s %16s\n" $pkg_name $pkg_ver + done +} + +extract () { + FILE=$1 + case "${FILE#*.}" in + "tar.gz" ) + tar -zxf $FILE + ;; + "tar.lz" ) + tar --lzip -xf "$FILE" + ;; + "zip" ) + unzip $FILE + ;; + * ) + tar -xf $FILE + ;; + esac +} + + +make_dir_struct() { + local system=$1 + + mkdir -pv $system/{etc,var,proc,sys,run,tmp} $system/usr/{bin,lib,sbin} $system/dev/{pts,shm} + + for i in bin lib sbin; do + ln -sv usr/$i $system/$i + done + + case $(uname -m) in + x86_64) mkdir -pv $system/lib64 ;; + esac +} + +extract () { + FILE=$1 + case "${FILE#*.}" in + "tar.gz" ) + tar -zxf $FILE + ;; + "tar.lz" ) + tar --lzip -xf "$FILE" + ;; + "zip" ) + unzip $FILE + ;; + * ) + tar -xf $FILE + ;; + esac +} + +init_versions() { + local versions_file="$SYSTEM_DIR/versions" + + [ -f $versions_file ] || exit 1 + + for i in ${packages[@]}; do + local name=${i^^} + local version=$(sed -n "s/^$i \s*\(.*\)$/\1/p" $versions_file) + eval "${i^^}_VERSION"="$version" + done +} + +get_source() { + local buildfile="$1" + local package_name=$(basename -s .xibuild $buildfile) + + echo "fetching $buildfile" + + if [ ! -d $package_name ]; then + mkdir -p $package_name + cd $package_name + rm -rf * + source $buildfile + + if git ls-remote -q $SOURCE $BRANCH &> /dev/null; then + git clone $SOURCE . + git checkout $BRANCH + + elif hg identify $SOURCE &> /dev/null; then + hg clone $SOURCE package_name . + else + local downloaded=$(basename $SOURCE) + curl -Ls $SOURCE > $downloaded + extract $downloaded + mv */* . + fi + fi +} + +get_sources() { + mkdir -p $SYSTEM_DIR/sources + for pkg in $@; do + local pkg_file="$WORKING_DIR/buildfiles/repo/*/$pkg.xibuild" + cd $SYSTEM_DIR/sources + get_source $pkg_file + done +} + +# builds binutils to toolchain +# +build_binutils1() { + cd $SOURCES/binutils/ + + mkdir -v build + cd build + + ../configure --prefix="$SYSTEM_DIR/tools" \ + --with-sysroot="$SYSTEM_DIR" \ + --target=$TGT \ + --disable-nls \ + --disable-werror && + make && + make install -j1 +} + + +# builds gcc to toolchain +# +build_gcc1() { + cd $SOURCES/gcc/ + + #rm -rf mpfr gmp mpc + + autoreconf -i + + [ -d mpfr ] || cp -r $SOURCES/mpfr mpfr + [ -d gmp ] || cp -r $SOURCES/gmp gmp + [ -d mpc ] || cp -r $SOURCES/mpc mpc + + sed -e '/m64=/s/lib64/lib/' -i.orig gcc/config/i386/t-linux64 + + mkdir -v build + cd build + + ../configure \ + --target=$TGT \ + --prefix=$SYSTEM_DIR/tools \ + --with-glibc-version=$GLIBC_VERSION \ + --with-sysroot=$SYSTEM_DIR \ + --with-newlib \ + --without-headers \ + --without-zstd \ + --enable-initfini-array \ + --disable-nls \ + --disable-shared \ + --disable-multilib \ + --disable-decimal-float \ + --disable-threads \ + --disable-libatomic \ + --disable-libgomp \ + --disable-libquadmath \ + --disable-libssp \ + --disable-libvtv \ + --disable-libstdcxx \ + --enable-languages=c,c++ + + make && + make install || exit 1 + cd .. + cat gcc/limitx.h gcc/glimits.h gcc/limity.h > \ + `dirname $($TGT-gcc -print-libgcc-file-name)`/install-tools/include/limits.h +} + +build_linux_headers() { + cd $SOURCES/linux/ + + make mrproper + + make headers + find usr/include -name '.*' -delete + rm usr/include/Makefile + cp -rv usr/include $SYSTEM_DIR/usr +} + +build_glibc() { + cd $SOURCES/glibc/ + + case $(uname -m) in + i?86) ln -sfv ld-linux.so.2 $SYSTEM_DIR/lib/ld-lsb.so.3 + ;; + x86_64) ln -sfv ../lib/ld-linux-x86-64.so.2 $SYSTEM_DIR/lib64 + ln -sfv ../lib/ld-linux-x86-64.so.2 $SYSTEM_DIR/lib64/ld-lsb-x86-64.so.3 + ;; + esac + + mkdir -v build + cd build + + echo "rootsbindir=/usr/sbin" > configparms + ../configure \ + --prefix=/usr \ + --host=$TGT \ + --build=$(../scripts/config.guess) \ + --enable-kernel=3.2 \ + --with-headers=$SYSTEM_DIR/usr/include \ + libc_cv_slibdir=/usr/lib + + make && + make DESTDIR=$SYSTEM_DIR install + + sed '/RTLDLIST=/s@/usr@@g' -i $SYSTEM_DIR/usr/bin/ldd + + $SYSTEM_DIR/tools/libexec/gcc/$TGT/$GCC_VERSION/install-tools/mkheaders +} + +build_libstdcxx() { + cd $SOURCES/gcc/ + + rm -rf build + mkdir build + cd build + + ../libstdc++-v3/configure \ + --host=$TGT \ + --build=$(../config.guess) \ + --prefix=/usr \ + --disable-multilib \ + --disable-nls \ + --disable-libstdcxx-pch \ + --with-gxx-include-dir=/tools/$SYSTEM_DIR/include/c++/$GCC_VERSION + + make && + make DESTDIR=$SYSTEM_DIR install +} + +build_m4() { + cd $SOURCES/m4/ + ./configure --prefix=/usr \ + --host=$TGT \ + --build=$(build-aux/config.guess) + + make + make DESTDIR=$SYSTEM_DIR install +} + +build_ncurses() { + cd $SOURCES/ncurses/ + + sed -i s/mawk// configure + mkdir build + + pushd build + ../configure + make -C include + make -C progs tic + popd + + ./configure --prefix=/usr \ + --host=$TGT \ + --build=$(./config.guess) \ + --mandir=/usr/share/man \ + --with-manpage-format=normal \ + --with-shared \ + --without-debug \ + --without-ada \ + --without-normal \ + --disable-stripping \ + --enable-widec + + make + make DESTDIR=$SYSTEM_DIR TIC_PATH=$(pwd)/build/progs/tic install + echo "INPUT(-lncursesw)" > $SYSTEM_DIR/usr/lib/libncurses.so +} + +build_bash() { + cd $SOURCES/bash/ + + ./configure --prefix=/usr \ + --build=$(support/config.guess) \ + --host=$TGT \ + --without-bash-malloc + + make + make DESTDIR=$SYSTEM_DIR install + ln -sv bash $SYSTEM_DIR/bin/sh +} + +build_coreutils() { + cd $SOURCES/coreutils/ + + ./configure --prefix=/usr \ + --host=$TGT \ + --build=$(build-aux/config.guess) \ + --enable-install-program=hostname \ + --enable-no-install-program=kill,uptime + + make + make DESTDIR=$SYSTEM_DIR install + + mv -v $SYSTEM_DIR/usr/bin/chroot $SYSTEM_DIR/usr/sbin + mkdir -pv $SYSTEM_DIR/usr/share/man/man8 + mv -v $SYSTEM_DIR/usr/share/man/man1/chroot.1 $SYSTEM_DIR/usr/share/man/man8/chroot.8 + sed -i 's/"1"/"8"/' $SYSTEM_DIR/usr/share/man/man8/chroot.8 +} + +build_diffutils() { + cd $SOURCES/diffutils/ + + ./configure --prefix=/usr --host=$TGT + + make + make DESTDIR=$SYSTEM_DIR install +} + +build_file() { + cd $SOURCES/file/ + + mkdir build + pushd build + ../configure --disable-bzlib \ + --disable-libseccomp \ + --disable-xzlib \ + --disable-zlib + make + popd + + ./configure --prefix=/usr --host=$TGT --build=$(./config.guess) + + make FILE_COMPILE=$(pwd)/build/src/file + make DESTDIR=$SYSTEM_DIR install +} + +build_findutils() { + cd $SOURCES/findutils/ + + ./configure --prefix=/usr \ + --localstatedir=/var/lib/locate \ + --host=$TGT \ + --build=$(build-aux/config.guess) + make + make DESTDIR=$SYSTEM_DIR install +} + +build_gawk() { + cd $SOURCES/gawk/ + sed -i 's/extras//' Makefile.in + ./configure --prefix=/usr \ + --host=$TGT \ + --build=$(build-aux/config.guess) + make + make DESTDIR=$SYSTEM_DIR install +} + +build_grep() { + cd $SOURCES/grep + +./configure --prefix=/usr \ + --host=$TGT +make +make DESTDIR=$SYSTEM_DIR install +} + +build_gzip() { + cd $SOURCES/gzip/ + ./configure --prefix=/usr --host=$TGT + make + make DESTDIR=$SYSTEM_DIR install +} + +build_make() { + cd $SOURCES/make/ + ./configure --prefix=/usr \ + --without-guile \ + --host=$TGT \ + --build=$(build-aux/config.guess) + make + make DESTDIR=$SYSTEM_DIR install +} + +build_patch() { + cd $SOURCES/patch/ + + ./configure --prefix=/usr \ + --host=$TGT \ + --build=$(build-aux/config.guess) + make + make DESTDIR=$SYSTEM_DIR install +} + +build_sed() { + cd $SOURCES/sed/ + + ./configure --prefix=/usr --host=$TGT + + make + make DESTDIR=$SYSTEM_DIR install +} + +build_tar() { + cd $SOURCES/tar/ + ./configure --prefix=/usr \ + --host=$TGT \ + --build=$(build-aux/config.guess) + make + make DESTDIR=$SYSTEM_DIR install +} + +build_xz() { + cd $SOURCES/xz/ + + ./configure --prefix=/usr \ + --host=$TGT \ + --build=$(build-aux/config.guess) \ + --disable-static \ + --docdir=/usr/share/doc/xz-$XZ_VERSION + make + make DESTDIR=$SYSTEM_DIR install +} + +build_binutils2() { + cd $SOURCES/binutils/ + make clean + rm -rf build + mkdir -v build + cd build + ../configure \ + --prefix=/usr \ + --build=$(../config.guess) \ + --host=$TGT \ + --disable-nls \ + --enable-shared \ + --disable-werror \ + --enable-64-bit-bfd + make + make DESTDIR=$SYSTEM_DIR install -j1 + install -vm755 libctf/.libs/libctf.so.0.0.0 $SYSTEM_DIR/usr/lib +} + +build_gcc2() { + cd $SOURCES/gcc/ + + rm -rf build + rm -rf mpc gmp mpfr + [ -d mpfr ] || cp -r $SOURCES/mpfr mpfr + [ -d gmp ] || cp -r $SOURCES/gmp gmp + [ -d mpc ] || cp -r $SOURCES/mpc mpc + mkdir -v build + cd build + + mkdir -pv $TGT/libgcc + ln -s ../../../libgcc/gthr-posix.h $TGT/libgcc/gthr-default.h + ../configure \ + --build=$(../config.guess) \ + --host=$TGT \ + --prefix=/usr \ + CC_FOR_TARGET=$TGT-gcc \ + --with-build-sysroot=$SYSTEM_DIR \ + --enable-initfini-array \ + --disable-nls \ + --disable-multilib \ + --disable-decimal-float \ + --disable-libatomic \ + --disable-libgomp \ + --disable-libquadmath \ + --disable-libssp \ + --disable-libvtv \ + --disable-libstdcxx \ + --enable-languages=c,c++ + make + make DESTDIR=$SYSTEM_DIR install + ln -sv gcc $SYSTEM_DIR/usr/bin/cc +} + +if [ -e $SYSTEM_DIR ]; then + printf "Remove existing system? [Y/n] " + read response + + if [ "$response" != "n" ]; then + rm -rf $SYSTEM_DIR + echo "removed $SYSTEM_DIR" + fi +fi + + +mkdir -p "$SYSTEM_DIR" +get_build_files +parse_package_versions > "$SYSTEM_DIR/versions" + +get_sources ${packages[@]} + +make_dir_struct $SYSTEM_DIR +init_versions + +reset_before_build () { + cd $SYSTEM_DIR; printf "\033[0;34mbuilding $1...\033[0m" +} + +for package in ${packages[@]}; do + cd $SYSTEM_DIR + touch $package.build.log +done + +reset_before_build "binutils1" +build_binutils1 &> binutils.build.log && printf "passed\n" || exit 1 + +reset_before_build "gcc1" +build_gcc1 &> gcc1.build.log && printf "passed\n" || exit 1 + +reset_before_build "linux_headers" +build_linux_headers &> linux-headers.build.log && printf "\033[0;32mpassed\n" || exit 1 + +reset_before_build "glibc" +build_glibc &> glibc.build.log && printf "\033[0;32mpassed\n" || exit 1 + +reset_before_build "libstdcxx" +build_libstdcxx &> libstdcxx.build.log && printf "\033[0;32mpassed\n" || exit 1 + +reset_before_build "m4" +build_m4 &> m4.build.log && printf "\033[0;32mpassed\n" || printf "\0330;34mFAIL\n" + +reset_before_build "ncurses" +build_ncurses &> ncurses.build.log && printf "\033[0;32mpassed\n" || printf "\0330;34mFAIL\n" + +reset_before_build "bash" +build_bash &> bash.build.log && printf "\033[0;32mpassed\n" || printf "\0330;34mFAIL\n" + +reset_before_build "coreutils" +build_coreutils &> coreutils.build.log && printf "\033[0;32mpassed\n" || printf "\0330;34mFAIL\n" + +reset_before_build "diffutils" +build_diffutils &> diffutils.build.log && printf "\033[0;32mpassed\n" || printf "\0330;34mFAIL\n" + +reset_before_build "file" +build_file &> file.build.log && printf "\033[0;32mpassed\n" || printf "\0330;34mFAIL\n" + +reset_before_build "findutils" +build_findutils &> findutils.build.log && printf "\033[0;32mpassed\n" || printf "\0330;34mFAIL\n" + +reset_before_build "gawk" +build_gawk &> gawk.build.log && printf "\033[0;32mpassed\n" || printf "\0330;34mFAIL\n" + +reset_before_build "grep" +build_grep &> grep.build.log && printf "\033[0;32mpassed\n" || printf "\0330;34mFAIL\n" + +reset_before_build "gzip" +build_gzip &> gzip.build.log && printf "\033[0;32mpassed\n" || printf "\0330;34mFAIL\n" + +reset_before_build "make" +build_make &> make.build.log && printf "\033[0;32mpassed\n" || printf "\0330;34mFAIL\n" + +reset_before_build "patch" +build_patch &> patch.build.log && printf "\033[0;32mpassed\n" || printf "\0330;34mFAIL\n" + +reset_before_build "sed" +build_sed &> sed.build.log && printf "\033[0;32mpassed\n" || printf "\0330;34mFAIL\n" + +reset_before_build "tar" +build_tar &> tar.build.log && printf "\033[0;32mpassed\n" || printf "\0330;34mFAIL\n" + +reset_before_build "xz" +build_xz &> xz.build.log && printf "\033[0;32mpassed\n" || printf "\0330;34mFAIL\n" + +reset_before_build "binutils2" +build_binutils2 &> bintuils2.build.log && printf "\033[0;32mpassed\n" || printf "\0330;34mFAIL\n" + +reset_before_build "gcc2" +build_gcc2 &> gcc2.build.log && printf "\033[0;32mpassed\n" || printf "\0330;34mFAIL\n" + + +echo "DONE" + +printf "Clean sources and logs? [Y/n]" +read response + +if [ "$response" != "n" ]; then + rm -rf $SYSTEM_DIR/*.log + rm -rf $SYSTEM_DIR/sources + echo "removed $SYSTEM_DIR/sources" +fi + + -- cgit v1.2.1