Pasted

#!/bin/bash
 
directories=( "/var/packages/" "/var/packages/builtin_files" "/var/lib/dpkg" )
outfile=../data-generators/pkg_md5_sums
CWD="$PWD"
 
RP_FN="`which realpath`"
function realpath(){
    case "$RP_TARGET" in
    *busybox*)
      if [ "$1" = -m ]; then
        shift
        A_PATH=$1
        A_PATH=$(echo "$A_path" | sed 's#^./#'"$CWD"'#g' )
        A_PATH=$(echo "$A_path" | sed 's#^../#'"$CWD"'#g' ) && A_PATH=$(dirname $A_PATH)
        echo "Warning simulating '-m' option since xzit isn't supported by busybox" >2
        echo "A_PATH=$A_PATH" >2
        if [ -f  "$A_PATH" ] || [ -d  "$A_PATH" ]; then
          $RP_FN "$@"
        else
          echo "$A_PATH"
        fi
      else
        $RP_FN "$@"
      fi
      ;;
    *)
      $RP_FN "$@"
      ;;
    esac 
}
export -f realpath
outfile=$(realpath -m "$outfile")
 
AWK_fn_map_deb='
/^Package:/     { sub(/^Package: /,"");  PKG=$0; }
/^Version:/     { sub(/^Version: /,"");  PKGVER=$0; }
/^Filename:/    { sub(/^Filename: /,""); PKGPATH=$0; sub(/.*\//,""); PKGFILE=$0; }
 
 
/^$/            { print PKG "|" ARCH "|" PKGVER "|" PKGFILE "|" "dpkg" "|"
                  PKG=""; PKGVER=""; PKGFILE=""; PKGPATH=""; PKGPRIO=""; PKGSECTION=""; PKGMD5="";  PKGDEP=""; }
'
 
 
AWK_fn_prepend_file_list='BEGIN {FS = "|"}
{   db_list=$3
    #print "db_list=" db_list
    sub(/\.[^.]+$/, "."ext, db_list)
    print db_list "|" $1 "|" $2 "|" $3 "|" $4 "|" db_dir}
'   
 
#AWK_fn_DISTRO_PKGS_SPECS=
 
for pfx in "1" "2" "3"; do
  [ -f "$outfile"_$pfx ] && rm "$outfile"_$pfx
  touch "$outfile"
  [ -f "$outfile"_$pfx_md5 ] && rm "$outfile"_$pfx_md5
  touch "$outfile"_$pfx_md5
  [ -f "$outfile"_"$pfx"db ] && rm "$outfile"_"$pfx"_db
  touch "$outfile"_"$pfx"_db
done
 
 
 
for a_dir in "${directories[@]}"; do
  bname="$(basename "$a_dir")"
  case "$bname" in
  packages)
    pfx=1
    cd "$a_dir"; dir_bname=$(basename "$a_dir")
    md5sum $(ls -1 *.files) "" | sed -r 's#^([^[:space:]]+)([[:space:]]+)([^[:space:]].*)$#\3|\1|'"$bname"'#g' | sort >> "$outfile"_"$pfx"_md5
    cut -f2,3,8 -d'|' --output-delimiter="|" "$a_dir"/user-installed-packages | \
      awk -v ext=files -v db_dir=packages "$AWK_fn_prepend_file_list" | sort >> "$outfile"_"$pfx"_db #f1=file_list_name, f2=pkg, f3=version, f4=pkg_file, f5=packages
    join -t "|" -j 1 "$outfile"_"$pfx"_db "$out_file"_"$pfx"_md5 >> "$outfile"_$pfx
    ;;
  builtin_files)
    pfx=2
    cat /var/packages/DISTRO_PKGS_SPECS | \
      "$CWD"/distro_pkgs_specs__to__compat_pkg.awk | sort -k2 -t '|' > ${outfile}"_distro_pkgs_specs" #f1=pkg_name, f2=distro_pkgs_specs_name
    cd "$a_dir"; dir_bname=$(basename "$a_dir")
    md5sum $(ls -1f) "" | sed -r 's#^([^[:space:]]+)([[:space:]]+)([^[:space:]].*)$#\3|\1|'"$bname"'#g' | \
      sort >> "${outfile}"_"$pfx"_md5 #f1=pkg_name, f2=distro_pkgs_specs_nam
      
    join -t "|" -a1 -1 2 -2 1 ${outfile}"_distro_pkgs_specs" "${outfile}"_"$pfx"_md5 | sort > "${outfile}"_specs_and_md5
      
    cut -f2,3,8 -d'|' --output-delimiter="|" "$a_dir"/../woof-installed-packages | \
      awk -v ext=files -v db_dir=builtin_compat_files "$AWK_fn_prepend_file_list" | sort >> "${outfile}"_"$pfx"_db
    join -t "|" -j 1 "${outfile}"_"$pfx"_db "${out_file}"_specs_and_md5 >> "${outfile}"_$pfx
    ;;    
  dpkg)
    pfx=3
    cd "$a_dir/info"; dir_bname=$(basename "$a_dir")
    md5sum $(ls -1 *.list; ls -1 *.md5sums) "" \
       | sed -r 's#^([^[:space:]]+)([[:space:]]+)([^[:space:]]+.*)([.])([^.]*)$#\3||dpkg|\3\4|\1|'"$bname"'#g' \
       | sed -r 's#([^:]+)(:?[^\|]*\|)(.*)#\1|||||\2\3#g' \
       | sort > "${outfile}"_"$pfx"_md5 #f1=pkg, f2=:arch, f3=ver, f4=pkgfile, f5=dir_name, f6=filelist, f7=md5sum
    cd ..
    cat status | awk -v db_dir=dpkg "$AWK_fn_map_deb" > "${outfile}"_"$pfx"_db
    cd $CWD/..
    cat "${outfile}"_"$pfx"_md5  "${outfile}"_"$pfx"_db > "${outfile}"_"$pfx"_data
    cd $CWD/..
    ./bash-reduce -v shuffle_OFS="@@@@" -v shuffle_FS="|" -v Keep_Key=true ./dpkg_mappers/identity.awk ./dpkg_reducers/LB_reducer.awk "${outfile}"_"$pfx"_data > "${outfile}"_"$pfx"
    #./bash-reduce -v shuffle_OFS="@@@@" -v shuffle_FS="|" -v Keep_Key=true ./dpkg_mappers/identity.awk ./core/identity.awk "${outfile}"_"$pfx"_data > "${outfile}"_"$pfx"
    #./bash-reduce -v shuffle_OFS="@@@@" -v shuffle_FS="|" -v Keep_Key=true ./core/identity.awk ./dpkg_reducers/LB_reducer.awk "${outfile}"_"$pfx"_data > "${outfile}"_"$pfx"
    #./bash-reduce -v shuffle_OFS="@@@@" -v shuffle_FS="|" -v Keep_Key=true ./dpkg_mappers/LB_Mapper.awk ./dpkg_reducers/LB_reducer.awk "${outfile}"_"$pfx"_data > "${outfile}"_"$pfx"
    #./bash-reduce -v shuffle_OFS="'@@@@'" -v shuffle_FS="|" -v Keep_Key=true ./dpkg_mappers/LB_Mapper.awk ./core/shuffle.awk "${outfile}"_"$pfx"_data > "${outfile}"_"$pfx"
    #join -t "|" -j 1 "${outfile}"_"$pfx"_db "${out_file}"_specs_and_md5 >> "${outfile}"_$pfx
    ;;      
  esac
done
#cd $CWD
#cat cat "${outfile}"_1_db "$outfile"_2_dpkg "$outfile"_3_dpkg "${outfile}"_"$pfx"_db > "${outfile}"_data
 
#cd $CWD/..
#./bash-reduce ./dpkg_mappers/LB_Mapper ./dpkg_reducers/LB_reducer "${outfile}"_data >> "${outfile}"_$pfx