epm-restore

#!/bin/sh
#
# Copyright (C) 2020  Etersoft
# Copyright (C) 2020  Vitaly Lipatov <lav@etersoft.ru>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#

load_helper epm-sh-altlinux
load_helper epm-assure

# FIXME: python modules are packaged into python packages, but we have only module names and rpm package names instead of python packages
# enable python3dest(PEP-503 normalized name) provides
# https://bugzilla.altlinux.org/show_bug.cgi?id=39003
__epm_filter_pip_to_rpm()
{
    tr "A-Z" "a-z" | sed -e "s|_|-|g" -e "s|^python[-_]||" \
        -e "s|bs4|beautifulsoup4|" \
        -e "s|pillow|Pillow|" \
        -e "s|sqlalchemy|SQLAlchemy|" \
        -e "s|flask-SQLAlchemy|flask_sqlalchemy|" \
        -e "s|redis|redis-py|" \
        -e "s|pyjwt|jwt|" \
        -e "s|pymonetdb|monetdb|" \
        -e "s|pyyaml|yaml|" \
        -e "s|flask-migrate|Flask-Migrate|" \
        -e "s|twisted|twisted-core|" \
        -e "s|pymacaroons|pymacaroons-pynacl|" \
        -e "s|pygments|Pygments|" \
        -e "s|memcached|memcache|" \
        -e "s|pyopenssl|OpenSSL|"
}

# TODO: remove me
fill_sign()
{
    local sign="$1"
    echo "$2" | grep -E -- "$sign[[:space:]]*[0-9.]+?" | sed -E -e "s|.*$sign[[:space:]]*([0-9.]+?).*|\1|"
}


# macro pkg caseline
__epm_pi_sign_to_rpm()
{
    local t="$1"
    local l="$2"
    local equal="$3"
    [ -n "$equal" ] || equal=">="

    local pi=''
    local sign ll
    for sign in "<=" "<" ">=" ">" "==" "!="; do
        ll=$(fill_sign "$sign" "$l")
        [ -n "$ll" ] || continue
        [ "$sign" = "==" ] && sign="$equal"
        [ "$sign" = "!=" ] && sign=">="
        [ -n "$pi" ] && pi="$pi
"
        pi="$pi$t $sign $ll"
    done
    [ -n "$pi" ] || pi="$t"
    echo "$pi"
}

__epm_get_array_name()
{
    echo "$*" | grep "=" | head -n1 | sed -e 's| *=.*||'
}

__epm_lineprint_python_array()
{
    local a="$*"
    [ -n "$a" ] || return
    local name="$(__epm_get_array_name "$a")"
    (echo "$a" | sed -E -e 's@(\]|\)).*@\1@' ; echo "print('\n'.join($name))" ) | ( a= python - || a= python3 )
}

# translate pip requirement lines to rpm notation
# (compare signs and package names)
__epm_restore_convert_to_rpm_notation()
{
    local equal="$1"
    local l
    while read l ; do
        if echo "$l" | grep -q "; *python_version *< *['\"]3" ; then
            [ -n "$verbose" ] && warning "    $t is python2 only requirement, skipped"
            continue
        fi
        # drop various "python_version > '3.5'"
        l="$(echo "$l" | sed -e "s| *;.*||")"
        if echo "$l" | grep -qE "^ *#" || [ -z "$l" ] ; then
            continue
        fi
        local t="$(echo "$l" | sed -E -e "s|[[:space:]]*[<>!=]+.*||" -e "s| *#.*||" | __epm_filter_pip_to_rpm)"
        [ -n "$t" ] || continue
        # until new section
        if echo "$l" | grep -qE "^\[" ; then
            break
        fi
        # if dependency_links URLs, use egg name
        if echo "$l" | grep -qE "://" ; then
            if echo "$l" | grep -q "#egg=" ; then
                t="$(echo "$l" | sed -e "s|.*#egg=||" -e "s|\[.*||" | __epm_filter_pip_to_rpm)"
            else
                warning "    skipping URL $l ..."
                continue
            fi
        fi

        __epm_pi_sign_to_rpm "$t" "$l" "$equal"
    done
}

__epm_restore_pip()
{
    local req_file="$1"
    local reqmacro
    local ilist

    if [ -n "$dryrun" ] ; then
        reqmacro="%py3_use"
        basename "$req_file" | egrep -q "(dev|test)" && reqmacro="%py3_buildrequires"
        echo
        echo "# generated by epm --restore --dry-run from $(basename $(dirname $(realpath $req_file)))/$req_file"
        cat $req_file | __epm_restore_convert_to_rpm_notation | sed -e "s|^|$reqmacro |"
        return
    else
        info "Install requirements from $req_file ..."
        ilist="$(cat $req_file | __epm_restore_convert_to_rpm_notation | cut -d' ' -f 1 | sed -e "s|^|python3-module-|")"
    fi

    ilist="$(estrlist list $ilist)"
    docmd epm install $ilist
}

__eresection()
{
    rhas "$1" "[[:space:]]*$2[[:space:]]*=[[:space:]]*[\[(]"
}

__epm_restore_setup_py()
{
    local req_file="$1"
    if [ -z "$dryrun" ] ; then
        info "Install requirements from $req_file ..."
    fi

    local ar=''
    local ilist=''
    local reqmacro
    local section=''
    while read l ; do
        if rhas "$l" "^ *#" ; then
            continue
        fi
        # start of section
        if __eresection "$l" "REQUIREMENTS" ; then
            reqmacro="%py3_use"
            section="$l"
        fi
        if __eresection "$l" "install_requires" ; then
            reqmacro="%py3_use"
            section="$l"
        fi
        if __eresection "$l" "setup_requires" ; then
            reqmacro="%py3_buildrequires"
            section="$l"
        fi
        if __eresection "$l" "tests_require" ; then
            reqmacro="%py3_buildrequires"
            section="$l"
        fi
        if [ -n "$section" ] ; then
            ar="$ar
$l"
        fi

        # not end of section
        if [ -z "$section" ] || ! rhas "$l" "(\]|\)),*" ; then
            continue
        fi

        if [ -n "$dryrun" ] ; then
            echo
            echo "# generated by epm --restore --dry-run from $(basename $(dirname $(realpath $req_file)))/$req_file $(__epm_get_array_name "$section")"
            __epm_lineprint_python_array "$ar" | __epm_restore_convert_to_rpm_notation ">=" | sed -e "s|^|$reqmacro |"
        else
            ilist="$ilist $(__epm_lineprint_python_array "$ar" | __epm_restore_convert_to_rpm_notation ">=" | cut -d' ' -f 1 | sed -e "s|^|python3-module-|")"
        fi
        section=''
        ar=''
    done < $req_file

    if [ -n "$dryrun" ] ; then
        return
    fi

    ilist="$(estrlist list $ilist)"
    docmd epm install $ilist
}

__epm_print_npm_list()
{
    local reqmacro="$1"
    local req_file="$2"
    local l
    while read l ; do
        # "tap": "^14.10.7"
        echo "$l" | grep -q '"\(.*\)": "\(.*\)"' || continue
        local name="$(echo "$l" | sed -e 's|.*"\(.*\)": ".*|\1|')"
        [ -z "$name" ] && continue
        local ver="$(echo "$l" | sed -e 's|.*"\(.*\)": "\(.*\)".*|\2|')" #'
        [ -z "$name" ] && continue

        if [ -n "$dryrun" ] ; then
            local pi=''
            local sign
            if echo "$ver" | grep -q "^\^" ; then
                sign=">="
            else
                sign="="
            fi
            ll=$(echo "$ver" | sed -e 's|^[^~]||')
            pi="$pi$reqmacro node-$name $sign $ll"
#         [ -n "$pi" ] && pi="$pi
#"
#           [ -n "$pi" ] || pi="$pi$reqmacro: node-$t"
            echo "$pi"
            continue
        else
            local pi="node-$name"
            #echo "    $l -> $name -> $pi"
        fi
        [ -n "$name" ] || continue
        ilist="$ilist $pi"
    done < $req_file

    [ -n "$dryrun" ] || echo "$ilist"
}

__epm_restore_npm()
{
    local req_file="$1"

    epm assure jq

    if [ -n "$dryrun" ] ; then
        local lt=$(mktemp)
        a= jq .dependencies <$req_file >$lt
        echo
        echo "# generated by epm --restore --dry-run from $(basename $(dirname $(realpath $req_file)))/$req_file"
        __epm_print_npm_list "Requires:" $lt

        echo
        echo "# devDependencies generated by epm --restore --dry-run from $(basename $(dirname $(realpath $req_file)))/$req_file"
        a= jq .devDependencies <$req_file >$lt
        __epm_print_npm_list "BuildRequires:" $lt
        rm -f $lt
        return
    fi

    info "Install requirements from $req_file ..."
    local lt=$(mktemp)
    a= jq .dependencies <$req_file >$lt
    ilist="$(__epm_print_npm_list "" $lt)"
    a= jq .devDependencies <$req_file >$lt
    ilist="$ilist $(__epm_print_npm_list "" $lt)"
    rm -f $lt
    docmd epm install $ilist
}

__epm_restore_by()
{
    local req_file="$1"
    [ -s "$req_file" ] || return

    if file $req_file | grep -q "ELF [3264]*-bit LSB executable" ; then
        assure_exists ldd-requires
        showcmd ldd-requires $req_file
        local TOINSTALL="$(a= ldd-requires $req_file | grep "^apt-get install" | sed -e "s|^apt-get install ||")"
        if [ -n "$dryrun" ] ; then
            estrlist list $TOINSTALL
            return
        fi
        [ -n "$TOINSTALL" ] || { info "There are no missed packages is found for $req_file binary." ; return ; }
        docmd epm install $TOINSTALL
        return
    fi

    case $(basename $req_file) in
        requirements.txt|dev-requirements.txt|requirements-dev.txt|requirements_dev.txt|requirements_test.txt|requirements-test.txt|test-requirements.txt|requires.txt)
            [ -s "$req_file" ] && __epm_restore_pip "$req_file"
            ;;
        setup.py|python_dependencies.py)
            [ -s "$req_file" ] && __epm_restore_setup_py "$req_file"
            ;;
        package.json)
            [ -s "$req_file" ] && __epm_restore_npm "$req_file"
            ;;
        Gemfile|package.json)
            info "$req_file support is not implemented yet"
            ;;
    esac
}

epm_restore()
{
    req_file="$pkg_filenames"
    if [ -n "$pkg_urls" ] && echo "$pkg_urls" | grep -qE "^https?://" ; then
        req_file="$(basename "$pkg_urls")"
        #assure eget
        [ -r "$req_file" ] && fatal "File $req_file is already exists in $(pwd)"
        info "Downloading '$req_file' from '$pkg_urls' ..."
        eget "$pkg_urls"
        [ -s "$req_file" ] || fatal "Can't download $req_file from '$pkg_urls'"
    fi

    if [ -n "$req_file" ] ; then
        __epm_restore_by $req_file
        return
    fi

# TODO: nowhere works: python3 setup.py --requires

    # if run with empty args
    for i in requirements.txt requirements_dev.txt requirements-dev.txt dev-requirements.txt requirements-test.txt requirements_test.txt test-requirements.txt Gemfile requires.txt package.json setup.py python_dependencies.py; do
        __epm_restore_by $i
    done

}