multigit/mgit

#!/bin/bash
shopt -s nullglob
IFS=$'\n\b'

# die hard, see https://github.com/capr/die
say()   { echo "$@" >&2; }
die()   { echo -n "ABORT: " >&2; echo "$@" >&2; exit 1; }
debug() { [ "$DEBUG" ] && echo "$@" >&2; }
run()   { debug -n "EXEC: $@ "; "$@"; local ret=$?; debug "[$ret]"; return $ret; }
must()  { debug -n "MUST: $@ "; "$@"; local ret=$?; debug "[$ret]"; [ $ret == 0 ] || die "$@ [$ret]"; }
dry()   { if [ "$DRY" ]; then [ "$VERBOSE" ] && say "$@"; else "$@"; fi; }

usage() {
	say
	say " multigit 4.1b - git wrapper for working with overlaid repos."
	say " Cosmin Apreutesei | public domain | https://github.com/capr/multigit"
	say
	say " USAGE: mgit [OPTIONS...] COMMAND ..."
	say
	say "   ls                           list cloned repos"
	say "   ls-all                       list all known repos"
	say "   ls-uncloned                  list all known but not cloned repos"
	say
	say "   ls-modified|st[atus]         list modified files across all repos"
	say "   ls-unpushed                  list repos that are ahead of origin"
	say "   ls-untracked                 list files untracked by any repo"
	say "   ls-double-tracked            list files tracked by multiple repos"
	say "   ls-tracked                   list files and which repos are tracking them"
	say "   which FILENAME               list which repo(s) are tracking a file"
	say
	say "   init REPO                              create a local repo"
	say "   clone [REMOTE/]REPO|URL[=VERSION] ...  clone one ore more repos"
	say "   clone-all                              clone all known uncloned repos"
	say "   clone-release REL|RELFILE              clone/checkout all repos from a release (file)"
	say "   remove REPO ...                        remove repos from disk (!)"
	say "   convert [NAME]                         convert current git repo to mgit"
	say
	say "   baseurl [REMOTE [URL|-]]     get/set/delete the baseurl of a remote"
	say "   origin [REPO [REMOTE|URL|-]] get/set/delete the known origin of a repo"
	say
	say "   [-] REPO1,...                       start a shell for using git on a repo"
	say "   [-] REPO1,...|--all command ...     execute any git command on a repo"
	say "   [-] REPO1,...|--all exec ...        execute a shell command in a repo context"
	say "   [-] REPO1,...|--all ver[sion] [tag] show repo version or tag (as enum or list)"
	say "   [-] REPO1,...|--all make-symlinks   make symbolic links in .mgit/REPO"
	say "   [-] REPO1,...|--all make-hardlinks  make hard links in .mgit/REPO"
	say
	say "   release [REL]                show a release or list releases"
	say "   release REL update [tag]     create/update a release based on HEADs"
	say "   release REL clone            clone/checkout all repos from a release"
	say "   release REL remove           remove a release file"
	say
	say "   bash                         drop to bash (even on Windows)"
	say "   [help|--help]                show this screen"
	say
	say " OPTIONS:"
	say
	say "   -v                           verbose"
	say "   --debug                      print commands"
	say "   --dry                        don't actually remove stuff"
	say "   --yes                        choose yes when asked to remove stuff"
	say "   -SS                          reuse SSH connections (Linux only)"
	say
	# append any plugin help files
	for f in .mgit/*.help; do
		cat "$f"
	done
	exit
}

check_root() { [ -d .mgit ] || die "'.mgit' dir not found."; }

list_known() {
	check_root
	(cd .mgit && \
	for f in *.origin; do
		echo "${f%.origin}"
	done)
}

list_cloned() {
	check_root
	(cd .mgit && \
	for f in *; do
		[ -d "$f/.git" ] && echo "$f"
	done)
}

list_uncloned() {
	check_root
	(cd .mgit && \
	for f in *.origin; do
		f="${f%.origin}"
		[ ! -d "$f/.git" ] && echo $f
	done)
}

list_modified() {
	check_root
	"$0" --all "$@" status -s
}

list_unpushed() {
	check_root
	for repo in `list_cloned`; do
		[ "$(GIT_DIR=".mgit/$repo/.git" \
				git rev-list HEAD...origin/master --count 2>/dev/null)" \
			!= "0" ] && echo "$repo"
	done
}

tracked_files() {
	check_root
	([ -d .git ] && git ls-files
	for repo in `list_cloned`; do
		GIT_DIR=".mgit/$repo/.git" git ls-files
	done) | sort | uniq $1
}

which_tracks() {
	[ "$1" ] || die "Filename expected."
	list_tracked | while IFS=" " read repo file; do
		[ "$file" = "$1" ] && echo "$repo"
	done
}

existing_files() {
	find * -type f | grep -v "^$1$" | sort
}

list_tracked() {
	check_root
	local MGIT_DIR="$PWD"
	for repo in `list_cloned`; do
		(cd "$PWD0"; GIT_DIR="$MGIT_DIR/.mgit/$repo/.git" git ls-files) | while read f; do
			printf "%-16s %s\n" "$repo" "$f"
		done
	done
}

list_sub() { # list_sub A B removes from list A the lines found in list B.
	# NOTE: using a temp file because bash on Windows can't do <(tracked_files)
	"$2" | sort > $$.tmp
	if [ "$OSTYPE" != "msys" ]; then
		# NOTE: not using grep because it's buggy on OSX (doesn't match all).
		"$1" $$.tmp | comm -23 - $$.tmp
	else
		# NOTE: using grep because Git for Windows doesn't have comm,
		# and because if MSYS is in PATH, comm from MSYS can't be used with
		# Git's bash (it crashes).
		"$1" $$.tmp | grep -v -F -x -f $$.tmp
	fi
	rm $$.tmp
}

list_untracked() {
	check_root
	list_sub existing_files tracked_files
}

list_double_tracked() {
	check_root
	# NOTE: using a temp file because bash on Windows can't do <(tracked_files -d)
	tracked_files -d > $$.tmp
	[ "$(cat $$.tmp)" ] && \
		PWD0="$PWD" list_tracked | grep -F -w -f $$.tmp
	rm $$.tmp
}

clone_all() {
	for repo in `list_uncloned`; do
		"$0" clone "$repo"
	done
}

check_repo_name() {
	[ "$1" ] || die "Repo name expected."
	local name="$1"
	name="${name//[^\.\-_0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ]/}"
	[ "$name" = "$1" ] || die "Invalid name: '$1'. Only letters, numbers, '.', '-' and '_' allowed."
	name="${name//^[\.\-]/}"
	[ "$name" = "$1" ] || die "Invalid name: '$1'. Must not start with '.' or '-'."
}

init() {
	check_repo_name "$1"
	local name="$1"

	[ -d ".mgit/$name" ] && die "Already cloned: '$name'."

	must mkdir -p ".mgit/$name"
	export GIT_DIR=".mgit/$name/.git"
	must git init -q $MULTIGIT_INIT_OPTS >&2
	must git config core.worktree ../../..

	must git config core.excludesfile ".mgit/$name.exclude"
	must make_exclude_file "$name"
}

make_exclude_file() {
	[ "$no_exclude_file" ] && return
	[ -f ".mgit/$1.exclude" ] && return
	# make a default "exclude-all" exclude file
	echo '*' > ".mgit/$1.exclude"
}

clone_one() {
	local arg="$1"
	local name
	local origin
	local rorigin
	local url
	local ver

	arg="${arg//[[:blank:]]/}" # spaces not allowed
	[ "$arg" = "$1" ] || die "$name is an invalid name"

	# extract `=version` if any
	ver="${arg##*=}"
	[ "$ver" = "$arg" ] && ver=""
	arg="${arg%=*}"

	# check if the arg is a full url or just `[origin/]name`
	if [ "${arg#*:}" != "$arg" ]; then
		url="$arg"
		origin="$arg"
		name="${url##*/}"
		name="${name%.git}"
	else
		name="${arg##*/}"
		origin="${arg%/*}"
		[ "$origin" = "$arg" ] && origin=""
	fi

	# check that arg is not `/` or `origin/`
	[ "$name" ] || die "$name is an invalid name"

	export GIT_DIR=".mgit/$name/.git"

	# if if repo is already cloned do a checkout instead.
	[ -d ".mgit/$name" ] && {
		local ver0="$(git_ver_for "$name")"
		[ "$ver" -a "$ver" = "$ver0" ] && {
			[ "$VERBOSE" ] && say "SKIP:  $name already at $ver"
			exit 0
		}
		say "PULL:  $name $ver (was: $ver0)"
		must git fetch -q $MULTIGIT_FETCH_OPTS >&2
		if [ "$ver" ]; then
			must git -c advice.detachedHead=false checkout -q "$ver" >&2
		else
			must git checkout -q -B master origin/master >&2
		fi
		exit 0
	}

	# check for a registered origin
	[ -f ".mgit/$name.origin" ] && \
		rorigin=$(cat .mgit/$name.origin)

	# decide the origin
	if [ "$origin" ]; then
		[ "$rorigin" -a "$origin" != "$rorigin" ] && \
			say "NOTE:  $name using different origin: '$origin' (was '$rorigin')."
	else
		origin="$rorigin"
		[ "$origin" ] || die "$name has no origin file"
	fi

	# find the origin url
	if [ ! "$url" ]; then
		if [ -f ".mgit/$origin.baseurl" ]; then
			local baseurl=$(cat .mgit/$origin.baseurl)
			url="$baseurl$name"
		else
			# assume the origin on file is a full url: check if it is
			if [ "${origin#*:}" != "$origin" ]; then
				url="$origin"
			else
				say "ABORT: $name has unknown origin: '$origin'."
				say "HINT:  To register '$origin' to be used as an origin, type, eg.:"
				say "HINT:    "$(basename "$0")" baseurl $origin https://github.com/$origin/"
				exit 1
			fi
		fi
	fi

	# finally, clone the repo
	(
		say "CLONE: $name $ver"
		no_exclude_file=1 must init "$name"
		must git remote add origin "$url"
		must git fetch -q $MULTIGIT_FETCH_OPTS >&2
	) || {
		# cleanup on failed fetch because git doesn't.
		rm -rf ".mgit/$name/"
		rm -f ".mgit/$name.exclude"
		die "$name clone failed"
	}
	if run git rev-parse -q --verify origin/master >/dev/null; then
		run git branch -q --track master origin/master >&2
		must git -c advice.detachedHead=false checkout -q $ver >&2
	else # new repo
		must git checkout -q -b master
		git config branch.master.remote origin
		git config branch.master.merge refs/heads/master
	fi
	# make an exclude file if one wasn't checked out already.
	must make_exclude_file "$name"

	# (re)register the repo's origin.
	if [ "$origin" != "$rorigin" ]; then
		if [ "$rorigin" ]; then
			say "NOTE:  $name has new origin: '$origin' (was: '$rorigin')"
		else
			[ "$VERBOSE" ] && {
				say "NOTE:  $name has origin: '$origin'"
				say "       (url: $url)"
			}
		fi
		must mkdir -p .mgit
		echo "$origin" > ".mgit/$name.origin" \
			|| die "Could not create origin file .mgit/$name.origin".
	fi
}

clone() {
	[ "$1" ] || die "Repo name expected."
	if [ $# = 1 ]; then
		clone_one "$1"
	else
		while [ $# != 0 ]; do
	   	"$0" clone "$1"
	   	shift
	   done
	fi
}

remove_one() {

	[ -d ".mgit/$1/" ] || die "$1 is not a cloned repo"

	# don't remove from a subshell
	[ "$MULTIGIT_REPO" = "$1" ] && die "Refusing to remove '$1' from a subshell."

	# get tracked files for this repo
	files="$(GIT_DIR=".mgit/$1/.git" git ls-files)" || {
		say "ABORT: Could not get the list of files for '$1'."
		say "HINT:  If you know that there are no checked out files,"
		say "HINT:  feel free to \`rm -rf .mgit/$1/ .mgit/$1.exclude\`."
		exit 1
	}

	# ask for confirmation if there are files to delete
	[ "$files" -a -z "$YES" ] && {
		local n=$(echo "$files" | wc -l)
		say "Remove ALL $((n)) files of '$1'? You can't undo this [yes/N]"
		read yes
		[ "$yes" = "yes" ] || { say "Canceled."; exit 1; }
	}

	# remove files
	for file in $files; do
		dry rm "$file"
	done

	# remove empty directories
	for file in $files; do
		echo "$(dirname "$file")"
	done | uniq | while read dir; do
		[ "$dir" != "." ] && dry /bin/rmdir -p "$dir" 2>/dev/null
	done

	# remove the git dir
	dry rm -rf ".mgit/$1/"
	dry rm  -f ".mgit/$1.exclude"

	say "REMOVED: $1"
}

remove() {
	[ "$1" ] || die "Repo name expected."
	if [ $# = 1 ]; then
		remove_one "$1"
	else
		while true; do
			case "$1" in
				--dry)    export DRY=1; shift ;;
				--yes)    export YES=1; shift ;;
				*)        break ;;
			esac
		done
		while [ $# != 0 ]; do
			"$0" remove "$1"
	   	shift
	   done
	fi
}

convert() {
	local name="$1"
	[ "$name" ] || name="$(basename "$PWD")"
	check_repo_name "$name"
	[ -d ".mgit/$name" ] && {
		[ -d "$PWD0/.git" ] && die "$name already taken"
		[ "$VERBOSE" ] && say "SKIP:  $name already in multigit"
		exit 0
	}
	[ -d "$PWD0/.git" ] || die "No .git dir in current dir"
	must mkdir -p ".mgit/$name"
	must mv "$PWD0/.git" ".mgit/$name/.git"
	export GIT_DIR=".mgit/$name/.git"
	must git config core.worktree ../../..
	must git config core.excludesfile ".mgit/$name.exclude"
	must make_exclude_file "$name"
	say "CONVERTED: $name"
}

baseurl() {
	[ "$1" ] || {
		for f in .mgit/*.baseurl; do
			f="${f#.mgit/}"
			f="${f%.baseurl}"
			printf "%-20s %s\n" "$f" "$(baseurl "$f")"
		done
		return
	}
	local origin="$1"
	local url="$2"
	# spaces not allowed
	origin="${origin//[[:blank:]]/}"
	url="${url//[[:blank:]]/}"
	[ -z "$1" -o "$1" != "$origin" ] && die "Invalid origin name '$1'."
	[ "$2" -a "$2" != "$url" ] && die "Invalid baseurl '$2'."

	if [ "X$url" = "X-" ]; then
		rm ".mgit/$origin.baseurl"
	elif [ "$url" ]; then
		[ "${url##*/}" != "" ] && die "A base URL must end with a '/'."
		mkdir -p .mgit
		echo "$url" > ".mgit/$origin.baseurl"
	else
		cat ".mgit/$origin.baseurl"
	fi
}

origin() {
	[ "$1" ] || {
		for f in .mgit/*.origin; do
			f="${f#.mgit/}"
			f="${f%.origin}"
			printf "%-20s %s\n" "$f" "$(origin "$f")"
		done
		return
	}
	local repo="$1"
	local origin="$2"

	# spaces not allowed
	repo="${repo//[[:blank:]]/}"
	origin="${origin//[[:blank:]]/}"
	[ -z "$1" -o "$1" != "$repo" ] && die "Invalid repo name '$1'."
	[ "$2" -a "$2" != "$origin" ] && die "Invalid origin '$2'."

	if [ "X$origin" = "X-" ]; then
		rm ".mgit/$repo.origin"
	elif [ "$origin" ]; then
		mkdir -p .mgit
		echo "$origin" > ".mgit/$repo.origin"
	else
		cat ".mgit/$repo.origin"
	fi
}

list_releases() {
	for f in .mgit/*.release; do
		f="${f#.mgit/}"
		f="${f%.release}"
		echo "$f"
	done
}

show_release() {
	[ -f ".mgit/$1.release" ] || die "Unknown release '$1'."
	cat ".mgit/$1.release"
}

update_release() {
	local rel="$1"
	[ -f "$rel" ] || rel=".mgit/$rel.release"
	if [ -f "$rel" ]; then
		local s="$(cat "$rel")"
		echo "$s" | (IFS=" "; while read repo ver0; do
			local name="${repo##*/}"
			local ver="$(git_ver_for "$name" $2)"
			[ "$ver0" = "*" ] && ver="*" # not touching those
			printf "%-20s %s\n" "$repo" "$ver"
			[ "$ver" != "$ver0" ] && printf "%-20s %s -> %s\n" "$repo" "$ver0" "$ver" >&2
		done) > "$rel"
	else
		"$0" --all version $2 > "$rel"
	fi
}

remove_release() {
	rm ".mgit/$1.release"
}

list_release_repos() {
	cat "$RELEASE" | (IFS=" "; while read repo ver; do
		local name="${repo##*/}"
		echo "$name"
	done)
}

clone_release() {
	local rel="$1"
	[ "${rel##*.}" = "release" -a -f "$rel" ] || rel=".mgit/$rel.release"
	[ -f "$rel" ] || die "Release not found: '$1'."

	local s="$(cat "$rel")" # load it in memory to make sure we have it till the end.

	# step 1: remove any repos that are not part of the release.
	RELEASE="$rel" list_sub list_cloned list_release_repos | while read repo; do
		"$0" --yes remove "$repo"
	done

	# step 2: clone/checkout repos present in the release.
	echo "$s" | (
		IFS=" "
		local fail=0
		while read repo ver; do
			# Version "*" in a release file specifies that the repo is already cloned
			# and checked out: this is the repo that contains the release file itself.
			# Even if cloned manually beforehand, the repo that contains the release file
			# must still be listed in the release file to avoid removing it in step 1.
			if [ "$ver" != "*" ]; then
				"$0" clone "$repo=$ver" || fail=1
			fi
		done
		exit $fail
	)
}

release() {
	local rel="$1"
	local cmd="$2"
	rel="${rel//[[:blank:]]/}" # spaces not allowed
	[ "$rel" -a "$1" != "$rel" ] && die "Invalid release name '$1'."
	[ "$rel" ] || { list_releases; return; }
	shift 2
	case "$cmd" in
		"")       show_release "$rel" "$@" ;;
		update)   update_release "$rel" "$@" ;;
		clone)    clone_release "$rel" "$@" ;;
		remove)   remove_release "$rel" "$@" ;;
		*)        die "Invalid release command '$cmd'."
	esac
}

git_shell() {
	cd "$PWD0" || return
	say "Entering subshell: git commands will affect the repo '$MULTIGIT_REPO'."
	say "Type \`exit' to exit subshell."
	git status -s
	say
	if [ "$OSTYPE" = "msys" ]; then
		export PROMPT="[$MULTIGIT_REPO] \$P\$G"
		"$COMSPEC" /k
	else
		export PS1="[$MULTIGIT_REPO] \u@\h:\w\$ "
		"$SHELL" --norc -i
	fi
}

git_ver_for() {
	if [ "$2" = "tag" ]; then
		GIT_DIR=".mgit/$1/.git" git describe --tags --abbrev=0 2>/dev/null $3
	else
		GIT_DIR=".mgit/$1/.git" git describe --tags --long --always $3
	fi
}

git_ver() {
	printf "$MULTIGIT_REPO=$(git_ver_for "$MULTIGIT_REPO" $1) "
}

git_version() {
	printf "%-20s %s" "$MULTIGIT_REPO"
	git_ver_for "$MULTIGIT_REPO" $1
}

git_remove_links() {
	[ "$OSTYPE" = "msys" ] && die "Not for Windows."
	([ "$MULTIGIT_REPO" ] && cd ".mgit/$MULTIGIT_REPO" || exit 1
	find . ! -path './.git/*' ! -path './.git' ! -path '.' -exec rm -rf {} \; 2>/dev/null)
}
git_make_hardlinks() {
	git_remove_links
	git ls-files | while read f; do
		mkdir -p "$(dirname ".mgit/$MULTIGIT_REPO/$f")"
		ln -f "$f" ".mgit/$MULTIGIT_REPO/$f"
	done
}
git_make_symlinks() {
	git_remove_links
	git ls-files | while read f; do
		mkdir -p "$(dirname ".mgit/$MULTIGIT_REPO/$f")"
		ln -sf "$PWD/$f" ".mgit/$MULTIGIT_REPO/$f"
	done
}

git_cmd_one() {
	local repo="$1"
	local cmd="$2"
	shift 2
	[ "$VERBOSE" ] && say "@$repo:"
	export GIT_DIR="$PWD/.mgit/$repo/.git"
	export MULTIGIT_REPO="$repo"
	[ -d "$GIT_DIR" ] || die "Unknown repo: '$repo'."
	case "$cmd" in
		exec)           cd "$PWD0" && "$@" ;;
		ver)            git_ver "$@" ;;
		version)        git_version "$@" ;;
		make-symlinks)  git_make_symlinks "$@" ;;
		make-hardlinks) git_make_hardlinks "$@" ;;
		"")             git_shell "$@" ;;
		*)
			# look for and execute a git plugin command
			if [ -f ".mgit/git-$cmd.sh" ]; then
				PWD0="$PWD0" ".mgit/git-$cmd.sh" "$@"
			else
				(cd "$PWD0" && git "$cmd" "$@")
			fi
		;;
	esac
}

git_cmd() {
	local repos="$1"; shift
	if [ "$repos" = "--all" ]; then
		[ "$1" ] || die "Refusing to start a subshell for each repo."
		for repo in `list_cloned`; do
			git_cmd_one "$repo" "$@"
		done
	else
		(IFS=","; for repo in $repos; do
			git_cmd_one "$repo" "$@"
		done)
	fi
	[ "$1" = "ver" ] && say
}

cd_root() {
	local pwd1
	while [ "$PWD" != "$pwd1" ]; do
		[ -d .mgit ] && return
		pwd1="$PWD"
		cd .. || die "Could not cd to '$PWD/..'."
	done
	cd "$PWD0" # root dir not found, go back to initial dir
}

PWD0="$PWD"
cd_root

while true; do
	case "$1" in
		-v)       shift; export VERBOSE=1;;
		--debug)  shift; export DEBUG=1;;
		--dry)    shift; export DRY=1;;
		--yes)    shift; export YES=1;;
		-SS)      shift; export GIT_SSH_COMMAND="ssh -o ControlMaster=auto -o ControlPersist=10 -o ControlPath=~/.ssh/sock-%r@%h-%p";;
		*)        break;;
	esac
done
cmd="$1"; shift
case "$cmd" in
	"")           usage ;;
	help)         usage ;;
	--help)       usage ;;
	ls)           list_cloned ;;
	ls-all)       list_known ;;
	ls-uncloned)  list_uncloned ;;
	ls-modified)  list_modified "$@" ;;
	st)           list_modified "$@" ;;
	status)       list_modified "$@" ;;
	which)	      which_tracks "$@" ;;
	ls-unpushed)  list_unpushed ;;
	ls-untracked) list_untracked ;;
	ls-double-tracked) list_double_tracked ;;
	ls-tracked)   list_tracked ;;
	init)         init "$@" ;;
	clone)        clone "$@" ;;
	clone-all)    clone_all "$@" ;;
	clone-release) clone_release "$@" ;;
	remove)       remove "$@" ;;
	convert)      convert "$@" ;;
	baseurl)      baseurl "$@" ;;
	origin)       origin "$@" ;;
	release)      release "$@" ;;
	bash)         exec bash "$@" ;;
	-)            git_cmd "$@" ;;
	*)
		# look for and execute a plugin command
		if [ -f ".mgit/$cmd.sh" ]; then
			".mgit/$cmd.sh" "$@"
		else
			git_cmd "$cmd" "$@"
		fi
		;;
esac