rpmgs

#!/bin/bash
# 2005-2006, 2009, 2014, 2015, 2016, 2017 (c) Etersoft www.etersoft.ru
# 2005-2016 Author: Vitaly Lipatov <lav@etersoft.ru>
# Public domain
#
# GS - get source
#
# Скачивает исходники, автоматически выправляя ситуацию с gz/bz2/tgz/zip (tar для git, tar.bz2 для src.rpm)
# Параметры:
# - название спек-файла
# -a - get all source
# check for the same file with other compression

# load common functions, compatible with local and installed script
. `dirname $0`/../share/eterbuild/functions/common
load_mod rpm tarball web buildsrpm

WEXT=""
GETSOURCE=""
LOADALL=''

#############################
Usage="Usage: $name [-f] [spec] [new_version]"
function mygetopts()
{
name=${0##*/}
Descr="$name (Get Source) - get sources by spec / repository"

phelp()
{
	echog "$Descr"
	echog "$Usage"
	echog "You can run 'rpmgs 1.2' for set new version 1.2 and download it"
	echog "Use HEAD instead version for get latest git commit"
	echo
	echog "Options:"
#	echog "   -a  get all source (not only Source|Source0)"
	echog "   -f  force download and commit tarball(s) (remove source file before download)"
}

while getopts :hf opt; do
    case $opt in
    h) phelp; exit 0;;
#    a) LOADALL=1 ;;
    f) FORCEDOWNLOAD=-f ;;
    +?) echog "$name: options should not be preceded by a '+'." 1>&2; exit 2;;
    ?)  echog "$name: $OPTARG: bad option.  Use -h for help." 1>&2 ; exit 2;;
    esac
done
 
# remove args that were options
[ "$OPTIND" -gt 0 ] && shift $(($OPTIND - 1))

LISTRPMARGS=$@

}

repack_tarball()
{
	# TODO: move it into repack
	[ "$(realpath "$1")" = "$(realpath "$2")" ] && return
	docmd erc -f repack "$1" "$2"
	# remove original
	rm -f "$1"
}

check_tarball()
{
	[ -s "$1" ] || return
	erc -q check "$1"
}

# Args: URL TARGET
download_to()
{
	local URL="$1"
	local TARGET="$2"
	pushd $(dirname $TARGET)
		local DF="$(basename "$URL")"
		download_url "$URL" && repack_tarball "$DF" "$TARGET"
		local RET=$?
		# TODO: repack need remove origin
		#rm -fv "$DF"
		[ "$RET" = "0" ] || rm -fv "$DF" "$TARGET"
	popd
	return $RET
}

# Args: URL target_file
download_any_tarball()
{
	local GETSOURCE="$1"
	local TARGET="$2"
	local FORMATS="tar.xz tar.bz2 tar.gz zip tgz 7z tbz2 tbz rar tar"
	local BASESOURCE="$GETSOURCE"

	local ORIGEXT=$(get_ext "$BASESOURCE")
	[ -n "$ORIGEXT" ] || fatal "Error with $GETSOURCE. Have no idea how to load files without extension"

	# first try download with original extension (exclude for tar)
	if [ "$ORIGEXT" != "tar" ] ; then
		FORMATS="$ORIGEXT $(estrlist exclude "$ORIGEXT" "$FORMATS")"
	fi

	BASESOURCE=$(dirname "$BASESOURCE")/$(basename "$BASESOURCE" .$ORIGEXT)

	local ext
	# try download by exts list
	for ext in $FORMATS ; do
		[ -z "$FORCEDOWNLOAD" ] && check_tarball "$TARGET" && { echo "$TARGET already exists, continue... " ; continue; }
		download_to "$BASESOURCE.$ext" "$TARGET" || continue
		return
	done
	fatal "Cannot retrieve $GETSOURCE"
}

__replace_first_line()
{
	# https://unix.stackexchange.com/questions/250603/replace-only-on-the-first-matching-line-with-sed
	sed -e "1,/$1/s/$1/$2/"
}

# param: spec name number (f.i., url for Source-url)
function source_ext()
{
	local GETSOURCEEXT=
	# %define SourceUrl ftp://updates.etersoft.ru/pub/Etersoft/WINE@Etersoft/last/sources/tarball/%name-%version.tar.gz
	#GETSOURCEURL=$(eval_spec $1 | grep -i "^%define ${2}Url${3} " | head -n 1 | sed -e "s/ *\$//g" | sed -e "s/^%define[ \t].*[ \t]//g")
	local SN="$3"
	#[ "$SN" = "0" ] && SN="$SN\?"
	if grep -q "^# $SN-$2:" "$1" ; then
		local TMPSPEC=$1.tmpurl
		local NEWSOURCE=$(grep --text "^# $SN-$2:" "$1" | sed -e "s/.*$SN-$2:[ \t]*//g" | tail -n1)
		test -n "$NEWSOURCE" || fatal "Can't extract URL from $SN-$2"
		# Fake replace for correct subst variables
		NEWSOURCE="$(echo "$NEWSOURCE" | sed -e 's|\&|\\&|g')"
		# TODO: move to separate function and rewrite
		# TODO: use special field before %build
		cat $1 | __replace_first_line "^Summary:.*" "" | sed -e "s|^\(# $SN-$2:.*\)|Summary: $NEWSOURCE\n\1|" > $TMPSPEC
		# TODO: replace only first entry
		#cat $1 | sed -e "s|^Summary:.*|Summary: $NEWSOURCE|1" > $TMPSPEC
		GETSOURCEEXT=$(eval_spec "$TMPSPEC" | get_var "Summary")
		#rhas "$GETSOURCEEXT" "%" && fatal "some macro unexpanded in URL. Check $TMPSPEC"
		rm -f "$TMPSPEC"
	fi
	
	echo "$GETSOURCEEXT"
	test -n "$GETSOURCEEXT"
}

# Get real URL from comment Source-xxx

# Source-svn: http://svn.wikimedia.org/svnroot/mediawiki/trunk/extensions/Collection/
function get_source_svn()
{
	GETSOURCESVN=$(source_ext "$1" svn "$2")
}


# Source-git: http://git.altlinux.org/people/lav/packages/rpm-build-fonts.git
function get_source_git()
{
	GETSOURCEGIT=$(source_ext "$1" git "$2")
}

# Source-url: ftp://updates.etersoft.ru/pub/Etersoft/WINE@Etersoft/last/sources/tarball/%name-%version.tar.gz
function get_source_url()
{
	GETSOURCEURL=$(source_ext "$1" url "$2")
	#rhas "$GETSOURCEURL" "%" && fatal "some macro unexpanded in URL. Check $TMPSPEC"
}

# Source-script: .gear/update.sh
function get_source_script()
{
	GETSOURCESCRIPT=$(source_ext "$1" script "$2")
}

function print_error()
{
	echog "Can't find any spec file. It is possible you run this script not in git dir."
	echog "If you use old style build, run rpmgs with spec name as arg."
	echog "If you wish download src.rpm, use rpmgp script."
	exit 1
}

# tarball dirname [options]
gear_update_from_tarball()
{
	local CREATEFLAG=-f
	local TARBALL="$1"
	local CURNAME="$2"
	shift 2
	assert_var TARBALL CURNAME
	[ -d "$CURNAME" ] || CREATEFLAG=-c
	# TODO: check tarball ext. for unsupported arch and realize it here or in gear-update
	echo "Commit tarball '$TARBALL' to git subdir '$CURNAME'..."
	if ! docmd gear-update $CREATEFLAG $@ "$TARBALL" "$CURNAME" ; then
		if gear-update $CREATEFLAG $@ "$TARBALL" "$CURNAME" 2>&1 | grep -q "More than one subdirectory specified" ; then
			info "Try unpack as is"
			CREATEFLAG="$CREATEFLAG -a"
			docmd gear-update $CREATEFLAG $@ "$TARBALL" "$CURNAME" && return
		fi
	else
		return 0
	fi
	fatal "can't import tarball '$TARBALL'"
}

# if we have named rule
is_predownloaded_rule()
{
	cat $(get_root_git_dir)/.gear/rules | grep -q "^tar:.*\.gear/$1"
	#fatal "missed tar:.gear/gear-sources in $(get_root_git_dir)/.gear/rules"
}


# Args: tarball dir
# Uses: RPMSOURCEDIR FORCEDOWNLOAD
commit_tarball_to_dir()
{
	local TARBALL="$1"
	local EXTTARBALL="$2"
	local CURNAME="$3"

	if [ "$EXTTARBALL" = "copy" ] ; then
		#local OLDFILE=$(echo "$CURNAME")
		cp -v "$TARBALL" "$CURNAME"
		docmd git add "$CURNAME"
		return 0
	fi

	if [ -f "$TARBALL" ] ; then
		gear_update_from_tarball "$TARBALL" "$CURNAME" $FORCEDOWNLOAD || { warning "Error with update tarball in repo" ; return 1 ; }
		# force commit ever files from .gitignore
		docmd git add -f "$CURNAME"
		# TODO: add GETSOURCEURL url
		docmd git commit -m "just import $(basename $TARBALL) with rpmgs script"
		rm -fv "$TARBALL"
	else
		echo "Skip missed $TARBALL tarball committing"
		return 1
	fi
	return 0
}

# TODO: rewrite for any tarball commit
# Arg: tarball
commit_tarball()
{
	local TARBALL="$1"
	local EXTTARBALL="$(get_ext $TARBALL)"
	local CURNAME
	CURNAME=$(get_tardir_from_rules "$EXTTARBALL" $(basename "$TARBALL"))
	if [ -z "$CURNAME" ] ; then
		info "Can't get dir (no $EXTTARBALL: line in rules file), just commit $(basename "$TARBALL") file"
		EXTTARBALL="copy"
		CURNAME=$(get_tardir_from_rules "$EXTTARBALL" $(basename "$TARBALL"))
		[ -n "$CURNAME" ] || fatal "There is no correct '$EXTTARBALL:' line nor 'copy:' in gear rules file for $(basename "$TARBALL"), needed for commit tarball"
	fi
	# FIXME:
	# use real path for download
	#is_gear_sources && CURNAME=
	#is_gear_sources && CURNAME=$(get_root_git_dir)/$BASENAME
	#fatal "FIXME: fail with is_gear_sources in commit_tarball"

	# hack: try detect dir for unpacking
	#test -d "$CURNAME" || CURNAME=$(get_root_git_dir)/$(get_tarballname "$spec")
	#test -d "$CURNAME" || CURNAME=$(get_root_git_dir)/$BASENAME

	commit_tarball_to_dir "$TARBALL" "$EXTTARBALL" "$CURNAME"
}

# update .gear/@name@-postsubmodules if needed
# uses: BASENAME VERSION
update_post_git_submodules()
{
	# TODO: check for rules
	local PSM=$(get_root_git_dir)/.gear/$BASENAME-postsubmodules
	[ -d $PSM ] || return

	info "Detected post git submodules hook"

	cat $(get_root_git_dir)/.gear/rules | grep -q "tar:.*-postsubmodules" || fatal "missed tar:.gear/$BASENAME-postmodules in $(get_root_git_dir)/.gear/rules"
	[ -s $(get_root_git_dir)/.gitmodules ] || fatal "there is no .gitmodules file in repo root"

	cd $(get_root_git_dir) || fatal
	docmd git submodule init
	docmd git submodule sync
	docmd git submodule update || fatal

	rm -rfv $PSM/*
	# echo * do not catch .names really
	#cp -a $(echo * | grep -v -E "(.gear|.git)") $PSM/ || fatal #"
	#find $PSM -name ".git" -type f -delete -print
	#find $PSM -name ".gitmodules" -type f -delete -print
	local i
	for i in $(grep "path = " .gitmodules | sed -e "s|.*path = \(.*\)|\1|") ; do #"
		cp -al --parents $i $PSM/ || fatal
	done
	docmd git add -f $PSM
	# TODO: put short commit id in a description
	# git rev-parse --short HEAD
	docmd git commit $PSM -m "update source prepared with submodules for version $VERSION"
}

source_postupdate_hook()
{
	local RGD=$(get_root_git_dir)
	if [ -s $RGD/.gear/source-postupdate-hook ] ; then
    		info "Detected .gear/source-postupdate-hook, run it ..."
		sh $RGD/.gear/source-postupdate-hook $VERSION
	fi
}

# param: development, production
update_predownloaded()
{
	local MODE="$1"
	local SDNAME=predownloaded-$MODE
	is_predownloaded_rule $SDNAME || return
	PSM=$(get_root_git_dir)/.gear/$SDNAME

	local RGD=$(get_root_git_dir)

	# hack for subdir??
	CURNAME=$BASENAME

	# TODO: add support for git
	rm -rf $PSM
	if [ -n "$GETSOURCEGIT" ] ; then
		# cp -a exclude .dir
		mkdir -p $PSM || fatal
		cp -a $RGD/* $PSM || fatal
	else
		cp -a $RGD/$CURNAME $PSM || fatal
	fi

	cd $PSM || fatal

	if [ -s $RGD/.gear/predownloaded-preinstall-hook ] ; then
    		info "Detected .gear/predownloaded-preinstall-hook, run it ..."
		sh $RGD/.gear/predownloaded-preinstall-hook $MODE $VERSION
                COMMITMSG="update predownloaded-$MODE with a hook script"
	fi

	#### npm only part
	if [ -s "./package.json" ] ; then

		# CHECKME: drop postinstall due run dev scripts during install --production
		# replace with fake commands?

		local COMMITMSG=''
                local PRODUCTION=''

		[ "$MODE" = "production" ] && PRODUCTION='--production'

      		info "Detected npm install hook, run npm install $PRODUCTION ..."
		a= npm install $PRODUCTION || fatal
		COMMITMSG="update node_modules with npm install $PRODUCTION for $VERSION (see $SDNAME in .gear/rules)"

		if [ -s $RGD/.gear/predownloaded-postinstall-hook ] ; then
    		        info "Detected .gear/predownloaded-postinstall-hook, run it ..."
			sh $RGD/.gear/predownloaded-postinstall-hook $MODE $VERSION
		fi

                # prune removes modules not listed in package.json
		# a= npm prune $PRODUCTION
		a= npm dedup

		# drop all exclude node_modules
		# TODO: correct .* removing
		rm -rf $(ls -1 | grep -v node_modules) .*

		#epm assure jq || fatal
		#(cd node_modules && rm -rf $(jq -r -c '.devDependencies | keys[]' ../package.json))
        fi
        ### end of npm part

	cd - >/dev/null

	docmd git add -f $PSM
	docmd git commit $PSM -m "$COMMITMSG"

}


parse_cmd_pre_spec "$@"
mygetopts $LISTARGS

test -z "$VERBOSE" || echo "'$LISTNAMES' @ '$LISTRPMARGS'"

if [ -n "$LISTRPMARGS" ] ; then
	if [ -z "${LISTRPMARGS/*spec/}" ] ; then
		fatal "run with incorrect filename $LISTRPMARGS"
	fi
	if [ ! -f "$LISTNAMES" ] ; then
		fatal "set version permitted only for one file"
	fi
	if [ "${LISTRPMARGS/ /}" != "$LISTRPMARGS" ] ; then
		fatal "you run rpmgs with more than one version"
	fi
	GSSETVERSION=$LISTRPMARGS
fi

test -z "$LISTNAMES" && print_error

[ -z "$GSSETRELEASE" ] || GSSKIPADDCHANGELOG=1

for spec in $LISTNAMES
do
	if [ -n "${spec/*spec/}" ] ; then
		print_error
	fi

	set_specdir $spec

	# Set version if needed
	if [ -n "$GSSETVERSION" ] && [ "$GSSETVERSION" != "HEAD" ] ; then
		CURVER=$(get_version $spec)
		set_version $spec ${GSSETVERSION/-*/}
		if [ "$CURVER" != "$GSSETVERSION" ] ; then
			set_release $spec $GSSETRELEASE
			if [ -n "$GSSETRELEASE" ] ; then
				echo "Set new $GSSETVERSION-$GSSETRELEASE version for $spec"
			else
				echo "Set new $GSSETVERSION version for $spec"
			fi
		else
			echo "Version $GSSETVERSION already was set"
			GSSKIPADDCHANGELOG=1
		fi
	fi

	LOADALL=1
	if [ -n "$LOADALL" ] ; then
		SOURCELIST=$(grep "^Source[0-9]*:" $spec | sed -e "s|:.*||g")
	else
		SOURCELIST=$(grep "^Source0\?:" $spec | sed -e "s|:.*||g")
	fi

	DOWNLOADSOME=''
	for SN in $SOURCELIST
	do
		GETSOURCE=$(eval_spec $spec | get_var "$SN")
		[ -n "$GETSOURCE" ] || fatal "Problem with empty $SN"

		echo "Updating $SN: $GETSOURCE ..."

		# for get RPMSOURCEDIR and BASENAME/VERSION/RELEASE
		build_rpms_name $spec

		mkdir -p $RPMSOURCEDIR/ || fatal "Can't create/chdir..."
		FTB=$RPMSOURCEDIR/$(basename "$GETSOURCE")

		# TODO: do not use RPMSOURCEDIR for temp. tarballs
		[ -n "$FORCEDOWNLOAD" ] && rm -f "$FTB"
		#[ -f "$RPMSOURCEDIR/$FTB" ] && { echog "Tarball $FTB already exists in $RPMSOURCEDIR dir, skipping." ; continue ; }

		# Test for eterbuild extensions (will set GETSOURCEURL or GETSOURCESVN)
		get_source_url $spec $SN || get_source_git $spec $SN || get_source_svn $spec $SN || get_source_script $spec $SN
		#[ "$SN" = "Source1" ] && exit
		#if ! rhas "$GETSOURCE" ".tar$" ; then
		#	warning "It is recommended to use .tar tarballs for sources ($FTB now)"
		#fi

		if [ -n "${GETSOURCESVN}" ] ; then
			is_gear $SPECDIR || fatal "Source-svn works only with gear repo"

		elif [ -n "${GETSOURCEGIT}" ] ; then
			is_gear $SPECDIR || fatal "Source-git works only with gear repo"

		elif [ -n "${GETSOURCESCRIPT}" ] ; then
			is_gear $SPECDIR || fatal "Source-script works only with gear repo"
			#[ -x "$GETSOURCESCRIPT" ] || fatal "Can't find executable $GETSOURCESCRIPT"
			bash -x $(get_root_git_dir)/$GETSOURCESCRIPT $GSSETVERSION "$FTB" || fatal "fatal with $GETSOURCESCRIPT"
			DOWNLOADSOME=1

		# TODO: rewrite code to use original file format and temp. download dir
		elif [ -n "${GETSOURCEURL}" ] ; then
			if [ -z "$FORCEDOWNLOAD" ] && check_tarball "$FTB" ; then
				echo "$FTB already exists, skipping... "
			else
				echog "Try to load ${GETSOURCEURL} for $spec"
				download_to "$GETSOURCEURL" "$FTB" || fatal "Can't download $GETSOURCEURL"
				DOWNLOADSOME=1
			fi
		else
			if ! rhas "$GETSOURCE" "ps?://" ; then
				if [ "$SN" != "Source" ] && [ "$SN" != "Source0" ] ; then
					info "No URL. Skipping $GETSOURCE downloading"
					continue
				else
					# only for main source
					warning "$SN $GETSOURCE has no URL"
					continue
				fi
			fi
			download_any_tarball "$GETSOURCE" "$FTB"
			DOWNLOADSOME=1
		# TODO: fix download single packed file
		#else
		#		download_to "$GETSOURCE" "$FTB"
		fi

		if [ -z "$GSSETVERSION" ] && [ -z "$FORCEDOWNLOAD" ]; then
			echog "Skip $FTB committing (run without new version or without -f)"
			continue
		fi

		if [ -n "${GETSOURCESVN}" ] ; then
			# clone svn repo to current dir
			# FIXME: need to clone in git root dir
			docmd git svn clone $GETSOURCESVN $(get_root_git_dir)
			echo "Run svn rebase from $GETSOURCESVN"
			docmd git svn rebase
			DOWNLOADSOME=1
		elif [ -n "${GETSOURCEGIT}" ] ; then
                        if [ -s $(get_root_git_dir)/.gear/upstream/remotes ] ; then
                            docmd gear-remotes-restore
                        fi
			echog "Try to fetch ${GETSOURCEGIT} for $spec"
			#TODO check upstream
			#TODO error if incompatible
			docmd git remote add upstream $GETSOURCEGIT
			docmd git fetch --tags upstream || fatal
			#TODO check tag: v%version or %version
			[ -n "$GSSETVERSION" ] || fatal "Sorry, I don't know new version"
			if [ "$GSSETVERSION" = "HEAD" ] ; then
				docmd git merge upstream/master
			else
				# some heuristics
				rc=1
				alternate_tag1="$(echo "v$GSSETVERSION" | sed -e "s|\.|-|g")"
				alternate_tag2="$(echo "REL_$GSSETVERSION" | sed -e "s|\.|_|g")"
				alternate_tag3="$(echo "version_$GSSETVERSION")"
				for i in  v$GSSETVERSION $alternate_tag1 $GSSETVERSION $alternate_tag2 $alternate_tag3 ; do
					git rev-parse $i >/dev/null 2>/dev/null && docmd git merge $i && rc=0 && break
				done
                                if [ "$rc" = "1" ] ; then
                                    i="$(git tag | grep "$GSSETVERSION\$")"
   				    git rev-parse $i >/dev/null 2>/dev/null && docmd git merge $i && rc=0
                                fi
				[ "$rc" = "0" ] || fatal "Can't find tag for $GSSETVERSION version"
			fi

			# TODO: it is more clean detect that dir
			if [ -d "$(get_root_git_dir)/.gear/tags" ] ; then
				docmd gear-update-tag -a
				cd $(get_root_git_dir)/.gear || fatal
				docmd git add tags/* -f
				docmd git commit -m "update .gear/tags"
				cd - >/dev/null
			fi

			update_post_git_submodules
                        source_postupdate_hook
			# TODO: make plugins
			update_predownloaded development
			update_predownloaded production

			#docmd gammit
			#docmd git commit -m "merge $FTB with rpmgs script"
			DOWNLOADSOME=1
		elif is_gear ; then
			commit_tarball "$FTB"
                        source_postupdate_hook
			# TODO: make plugins
			update_predownloaded development
			update_predownloaded production
			DOWNLOADSOME=1
		fi

		echog "DONE with $FTB"

	done

	[ -n "$DOWNLOADSOME" ] || fatal "No upstream code is updated for new version $GSSETVERSION."

	if [ -z "$GSSKIPADDCHANGELOG" ] ; then
		# Write changelog if all done
		CURVER=$(get_version $spec)
		CURREL=$(get_release $spec)
		EGEARME=""
		is_gear && EGEARME=" with rpmgs script"
		add_changelog_helper "- new version ($CURVER)$EGEARME" $spec || echog "Changelog entry for $CURVER-$CURREL already exists"
	fi

done

exit 0