#!/bin/sh # # Copyright (C) 2020, 2021 Etersoft # Copyright (C) 2020, 2021 Vitaly Lipatov <lav@etersoft.ru> # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU Affero General Public License for more details. # # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. # load_helper epm-sh-altlinux load_helper epm-assure # file prefix suffix __epm_restore_print_comment() { echo "#$2 generated by 'epm restore --dry-run' from $(basename $(dirname $(realpath "$1")))/$(basename "$1")$3" } # FIXME: python modules are packaged into python packages, but we have only module names and rpm package names instead of python packages # enable python3dest(PEP-503 normalized name) provides # https://bugzilla.altlinux.org/show_bug.cgi?id=39003 __epm_filter_pip_to_rpm() { tr "A-Z" "a-z" | sed -e "s|_|-|g" -e "s|^python[-_]||" -e "s|python$||" \ -e "s|bs4|beautifulsoup4|" \ -e "s|pillow|Pillow|" \ -e "s|sqlalchemy|SQLAlchemy|" \ -e "s|flask-SQLAlchemy|flask_sqlalchemy|" \ -e "s|redis|redis-py|" \ -e "s|pyjwt|jwt|" \ -e "s|pymonetdb|monetdb|" \ -e "s|pyyaml|yaml|" \ -e "s|flask-migrate|Flask-Migrate|" \ -e "s|twisted|twisted-core|" \ -e "s|pymacaroons|pymacaroons-pynacl|" \ -e "s|pygments|Pygments|" \ -e "s|memcached|memcache|" \ -e "s|pyinstaller||" \ -e "s|pyopenssl|OpenSSL|" } # TODO: remove me fill_sign() { local sign="$1" echo "$2" | grep -E -- "$sign[[:space:]]*[0-9.]+?" | sed -E -e "s|.*$sign[[:space:]]*([0-9.]+?).*|\1|" } # macro pkg caseline __epm_pi_sign_to_rpm() { local t="$1" local l="$2" local equal="$3" [ -n "$equal" ] || equal=">=" local pi='' local sign ll for sign in "<=" "<" ">=" ">" "==" "!=" "~="; do ll=$(fill_sign "$sign" "$l") [ -n "$ll" ] || continue [ "$sign" = "==" ] && sign="$equal" [ "$sign" = "~=" ] && sign="$equal" [ "$sign" = "!=" ] && sign=">=" [ -n "$pi" ] && pi="$pi " pi="$pi$t $sign $ll" done [ -n "$pi" ] || pi="$t" echo "$pi" } __epm_get_array_name() { echo "$*" | grep "=" | head -n1 | sed -e 's| *=.*||' } __epm_lineprint_python_array() { local a="$*" [ -n "$a" ] || return local name="$(__epm_get_array_name "$a")" (echo "$a" | sed -E -e 's@(\]|\)).*@\1@' ; echo "print('\n'.join($name))" ) | ( a= python3 - || a= python - ) } # translate pip requirement lines to rpm notation # (compare signs and package names) __epm_restore_convert_to_rpm_notation() { local equal="$1" local l while read l ; do if echo "$l" | grep -q 'platform_python_implementation != "PyPy"' ; then [ -n "$verbose" ] && warning " $t is not PyPi requirement, skipped" continue fi if echo "$l" | grep -q 'sys_platform == "darwin"' ; then [ -n "$verbose" ] && warning " $t is darwin only requirement, skipped" continue fi if echo "$l" | grep -q 'sys_platform == "win32"' ; then [ -n "$verbose" ] && warning " $t is win32 only requirement, skipped" continue fi if echo "$l" | grep -q "; *python_version *< *['\"]3" ; then [ -n "$verbose" ] && warning " $t is python2 only requirement, skipped" continue fi if echo "$l" | grep -q "; *python_version *<= *['\"]2\." ; then [ -n "$verbose" ] && warning " $t is python2 only requirement, skipped" continue fi # drop various "python_version > '3.5'" l="$(echo "$l" | sed -e "s| *;.*||")" if echo "$l" | grep -qE "^ *#" || [ -z "$l" ] ; then continue fi local t="$(echo "$l" | sed -E -e "s|[[:space:]]*[<>!=~]+.*||" -e "s| *#.*||" | __epm_filter_pip_to_rpm)" [ -n "$t" ] || continue # until new section if echo "$l" | grep -qE "^\[" ; then break fi # if dependency_links URLs, use egg name if echo "$l" | grep -qE "://" ; then if echo "$l" | grep -q "#egg=" ; then t="$(echo "$l" | sed -e "s|.*#egg=||" -e "s|\[.*||" | __epm_filter_pip_to_rpm)" else warning " skipping URL $l ..." continue fi fi __epm_pi_sign_to_rpm "$t" "$l" "$equal" done } __epm_restore_pip() { local req_file="$1" local reqmacro local ilist if [ -n "$dryrun" ] ; then reqmacro="%py3_use" basename "$req_file" | grep -E -q "(dev|test|coverage)" && reqmacro="%py3_buildrequires" echo __epm_restore_print_comment "$req_file" cat $req_file | __epm_restore_convert_to_rpm_notation | sed -e "s|^|$reqmacro |" return else info "Install requirements from $req_file ..." ilist="$(cat $req_file | __epm_restore_convert_to_rpm_notation | cut -d' ' -f 1 | sed -e "s|^|python3-module-|")" fi ilist="$(estrlist list $ilist)" docmd epm install $ilist } __epm_restore_print_toml() { local lt lt=$(mktemp) || fatal remove_on_exit $lt cat <<EOF >$lt #!//usr/bin/env python3 # https://realpython.com/python-toml/ import sys import toml if len(sys.argv) < 2: raise Exception('Run me with a file') pyproject = sys.argv[1] c = toml.load(pyproject) n = c["tool"]["poetry"]["dependencies"] for key, value in n.items(): if isinstance(value, dict): print('\n' + key + ' ' , value["version"]) else: print('\n' + key + ' ' + value) EOF a= python3 $lt "$1" } __epm_restore_print_pyproject() { local req_file="$1" __epm_restore_print_toml "$req_file" | __epm_restore_convert_to_rpm_notation | sed -e 's|\*||' -e 's|\^|>= |' } __epm_restore_pyproject() { local req_file="$1" local reqmacro local ilist if [ -n "$dryrun" ] ; then reqmacro="%py3_use" echo __epm_restore_print_comment "$req_file" __epm_restore_print_pyproject "$req_file" | sed -e "s|^|$reqmacro |" return else info "Install requirements from $req_file ..." ilist="$(__epm_restore_print_pyproject "$req_file" | cut -d' ' -f 1 | sed -e "s|^|python3-module-|")" fi ilist="$(estrlist list $ilist)" docmd epm install $ilist } __eresection() { rhas "$1" "[[:space:]]*$2[[:space:]]*=[[:space:]]*[\[(]" } __epm_restore_setup_py() { local req_file="$1" if [ -z "$dryrun" ] ; then info "Install requirements from $req_file ..." fi local ar='' local ilist='' local reqmacro local section='' while read l ; do if rhas "$l" "^ *#" ; then continue fi # start of section if __eresection "$l" "REQUIREMENTS" ; then reqmacro="%py3_use" section="$l" fi if __eresection "$l" "install_requires" ; then reqmacro="%py3_use" section="$l" fi if __eresection "$l" "setup_requires" ; then reqmacro="%py3_buildrequires" section="$l" fi if __eresection "$l" "tests_require" ; then reqmacro="%py3_buildrequires" section="$l" fi if [ -n "$section" ] ; then ar="$ar $l" fi # not end of section if [ -z "$section" ] || ! rhas "$l" "(\]|\)),*" ; then continue fi if [ -n "$dryrun" ] ; then echo __epm_restore_print_comment "$req_file" "" " $(__epm_get_array_name "$section")" __epm_lineprint_python_array "$ar" | __epm_restore_convert_to_rpm_notation ">=" | sed -e "s|^|$reqmacro |" else ilist="$ilist $(__epm_lineprint_python_array "$ar" | __epm_restore_convert_to_rpm_notation ">=" | cut -d' ' -f 1 | sed -e "s|^|python3-module-|")" fi section='' ar='' done < $req_file if [ -n "$dryrun" ] ; then return fi ilist="$(estrlist list $ilist)" docmd epm install $ilist } __epm_print_npm_list() { local reqmacro="$1" local req_file="$2" local l while read l ; do # "tap": "^14.10.7" echo "$l" | grep -q '"\(.*\)": "\(.*\)"' || continue local name="$(echo "$l" | sed -e 's|.*"\(.*\)": ".*|\1|')" [ -z "$name" ] && continue local ver="$(echo "$l" | sed -e 's|.*"\(.*\)": "\(.*\)".*|\2|')" #' [ -z "$name" ] && continue if [ -n "$dryrun" ] ; then local pi='' local sign if echo "$ver" | grep -q "^\^" ; then sign=">=" else sign="=" fi ll=$(echo "$ver" | sed -e 's|^[^~]||') pi="$pi$reqmacro node-$name $sign $ll" # [ -n "$pi" ] && pi="$pi #" # [ -n "$pi" ] || pi="$pi$reqmacro: node-$t" echo "$pi" continue else local pi="node-$name" #echo " $l -> $name -> $pi" fi [ -n "$name" ] || continue ilist="$ilist $pi" done < $req_file [ -n "$dryrun" ] || echo "$ilist" } __epm_print_perl_list() { local reqmacro="$1" local req_file="$2" local l for l in $(cat) ; do # perl(Class::ErrorHandler)>=0 echo "$l" | grep -q '^perl(' || continue local name="$(echo "$l" | sed -e 's|>=.*||' -e 's|::|/|g' -e 's|)|.pm)|')" [ "$name" = "perl(perl.pm)" ] && continue [ -z "$name" ] && continue local ver="$(echo "$l" | sed -e 's|.*>=||')" [ -z "$name" ] && continue if [ -n "$dryrun" ] ; then local pi='' local sign='' [ "$ver" = "0" ] || sign=" >= $ver" pi="$pi$reqmacro $name$sign" echo "$pi" continue else local pi="$name" #echo " $l -> $name -> $pi" fi [ -n "$name" ] || continue ilist="$ilist $pi" done < $req_file [ -n "$dryrun" ] || echo "$ilist" } __epm_print_perl_list_shyaml() { local reqmacro="$1" local req_file="$2" local l while read l ; do # Convert::ASN1: 0.10 echo "$l" | grep -q '^ *\(.*\): \(.*\)' || continue local name="$(echo "$l" | sed -e 's| *\(.*\): \(.*\)|\1|' -e 's|::|/|g')".pm [ "$name" = "perl.pm" ] && continue [ -z "$name" ] && continue local ver="$(echo "$l" | sed -e 's| *\(.*\): \(.*\)|\2|')" [ -z "$name" ] && continue if [ -n "$dryrun" ] ; then local pi='' local sign='' [ "$ver" = "0" ] || sign=" >= $ver" pi="$pi$reqmacro perl($name)$sign" echo "$pi" continue else local pi="perl($name)" #echo " $l -> $name -> $pi" fi [ -n "$name" ] || continue ilist="$ilist $pi" done < $req_file [ -n "$dryrun" ] || echo "$ilist" } __epm_print_nupkg_list() { a= dotnet list $1 package | grep "^ > " | while read n name req other; do if [ -n "$dryrun" ] ; then echo "BuildRequires: nupkg($name) >= $req" else echo "nupkg($name)" fi done } __epm_restore_nupkg() { local req_file="$1" if [ -n "$dryrun" ] ; then echo "# generated via dotnet list $(basename $(dirname $(realpath "$req_file")))/$(basename "$req_file") package" __epm_print_nupkg_list $req_file return fi info "Install requirements from $req_file ..." ilist=$(__epm_print_nupkg_list $req_file) ilist="$(estrlist list $ilist)" docmd epm install $ilist } __epm_print_meson_list() { local reqmacro="$1" local req_file="$2" local l while read name sign ver other ; do # gtk4-wayland # gtk4 >= 4.6 [ -n "$other" ] && continue if [ -n "$dryrun" ] ; then local pi='' pi="$reqmacro pkgconfig($name)" [ -n "$sign" ] && pi="$pi $sign $ver" echo "$pi" continue else local pi="pkgconfig($name)" fi [ -n "$name" ] || continue ilist="$ilist $pi" done < $req_file [ -n "$dryrun" ] || echo "$ilist" } __epm_restore_meson() { local req_file="$1" # gtk_dep = dependency('gtk4', version: '>= 4.6') # gtk_wayland_dep = dependency('gtk4-wayland', required: false) # packagekit_dep = dependency('packagekit-glib2', version: '>= 1.2', required: get_option('packagekit')) if [ -n "$dryrun" ] ; then local lt lt=$(mktemp) || fatal remove_on_exit $lt echo __epm_restore_print_comment "$req_file" " dependency" grep "dependency(" $req_file | sed -e 's|.*dependency(||' -e 's|).*||' -e 's|, required.*||' -e 's|, version:||' -e "s|'||g" >$lt __epm_print_meson_list "BuildRequires:" $lt rm -f $lt return fi info "Install requirements from $req_file ..." local lt lt=$(mktemp) || fatal remove_on_exit $lt grep "dependency(" $req_file | sed -e 's|.*dependency(||' -e 's|).*||' -e 's|, required.*||' -e 's|, version:||' -e "s|'||g" >$lt ilist="$ilist $(__epm_print_meson_list "" $lt)" rm -f $lt docmd epm install $ilist } __epm_restore_npm() { local req_file="$1" assure_exists jq || fatal if [ -n "$dryrun" ] ; then local lt lt=$(mktemp) || fatal remove_on_exit $lt a= jq .dependencies <$req_file >$lt echo __epm_restore_print_comment "$req_file" __epm_print_npm_list "Requires:" $lt echo __epm_restore_print_comment "$req_file" " devDependencies" a= jq .devDependencies <$req_file >$lt __epm_print_npm_list "BuildRequires:" $lt rm -f $lt return fi info "Install requirements from $req_file ..." local lt lt=$(mktemp) || fatal remove_on_exit $lt a= jq .dependencies <$req_file >$lt ilist="$(__epm_print_npm_list "" $lt)" a= jq .devDependencies <$req_file >$lt ilist="$ilist $(__epm_print_npm_list "" $lt)" rm -f $lt docmd epm install $ilist } # TODO: check __epm_restore_perl() { local req_file="$1" if [ -n "$dryrun" ] ; then local lt lt=$(mktemp) || fatal remove_on_exit $lt a= /usr/bin/perl $req_file PRINT_PREREQ=1 >$lt # all requirements will autodetected during packing, put it to the buildreq echo __epm_restore_print_comment "$req_file" __epm_print_perl_list "BuildRequires:" $lt rm -f $lt return fi info "Install requirements from $req_file ..." local lt lt=$(mktemp) || exit remove_on_exit $lt a= /usr/bin/perl $req_file PRINT_PREREQ=1 >$lt ilist="$(__epm_print_perl_list "" $lt)" rm -f $lt docmd epm install $ilist } # disabled __epm_restore_perl_shyaml() { local req_file="$1" assure_exists shyaml || fatal if [ -n "$dryrun" ] ; then local lt lt=$(mktemp) || fatal remove_on_exit $lt a= shyaml get-value requires <$req_file >$lt # all requirements will autodetected during packing, put it to the buildreq echo __epm_restore_print_comment "$req_file" __epm_print_perl_list "BuildRequires:" $lt echo __epm_restore_print_comment "$req_file" " build_requires" a= shyaml get-value build_requires <$req_file >$lt __epm_print_perl_list "BuildRequires:" $lt rm -f $lt return fi info "Install requirements from $req_file ..." local lt lt=$(mktemp) || fatal remove_on_exit $lt a= shyaml get-value requires <$req_file >$lt ilist="$(__epm_print_perl_list "" $lt)" a= shyaml get-value build_requires <$req_file >$lt ilist="$ilist $(__epm_print_perl_list "" $lt)" rm -f $lt docmd epm install $ilist } __epm_restore_by() { local req_file="$1" [ -n "$verbose" ] && info "Checking for $req_file ..." [ -s "$req_file" ] || return if file $req_file | grep -q "ELF [3264]*-bit LSB executable" ; then assure_exists ldd-requires showcmd ldd-requires $req_file local TOINSTALL="$(a= ldd-requires $req_file | grep "^apt-get install" | sed -e "s|^apt-get install ||")" if [ -n "$dryrun" ] ; then estrlist list $TOINSTALL return fi [ -n "$TOINSTALL" ] || { info "There are no missed packages is found for $req_file binary." ; return ; } docmd epm install $TOINSTALL return fi case $req_file in requirements/default.txt|requirements/dev.txt|requirements/test.txt|requirements/coverage.txt) [ -s "$req_file" ] && __epm_restore_pip "$req_file" && return ;; esac case $(basename $req_file) in requirements.txt|dev-requirements.txt|requirements-dev.txt|requirements_dev.txt|requirements_test.txt|requirements-test.txt|test-requirements.txt|requires.txt) [ -s "$req_file" ] && __epm_restore_pip "$req_file" ;; setup.py|python_dependencies.py) [ -s "$req_file" ] && __epm_restore_setup_py "$req_file" ;; pyproject.toml) [ -s "$req_file" ] && __epm_restore_pyproject "$req_file" ;; package.json) [ -s "$req_file" ] && __epm_restore_npm "$req_file" ;; meson.build) [ -s "$req_file" ] && __epm_restore_meson "$req_file" ;; # META.yml) # [ -s "$req_file" ] && __epm_restore_perl "$req_file" # ;; Makefile.PL) [ -s "$req_file" ] && __epm_restore_perl "$req_file" ;; *.sln|*.csproj) local PROJ="$(echo $req_file)" [ -s "$PROJ" ] && __epm_restore_nupkg "$PROJ" ;; Gemfile|package.json) info "$req_file support is not implemented yet" ;; esac } epm_restore() { req_file="$pkg_filenames" if [ -n "$pkg_urls" ] && echo "$pkg_urls" | grep -qE "^https?://" ; then req_file="$(basename "$pkg_urls")" #assure eget [ -r "$req_file" ] && fatal "File $req_file is already exists in $(pwd)" info "Downloading '$req_file' from '$pkg_urls' ..." eget "$pkg_urls" [ -s "$req_file" ] || fatal "Can't download $req_file from '$pkg_urls'" fi if [ -n "$req_file" ] ; then __epm_restore_by $req_file return fi # TODO: nowhere works: python3 setup.py --requires # if run with empty args for i in requirements.txt requirements/default.txt requirements_dev.txt requirements-dev.txt requirements/dev.txt dev-requirements.txt \ requirements-test.txt requirements_test.txt requirements/test.txt test-requirements.txt requirements/coverage.txt \ Gemfile requires.txt package.json setup.py python_dependencies.py Makefile.PL meson.build pyproject.toml \ *.sln *.csproj ; do __epm_restore_by $i done }