git-clone-all

#!/usr/bin/env bash

SCRIPT_PATH="$(realpath "$0")"
SCRIPT_NAME="$(basename "$SCRIPT_PATH")"

# {{{ color variables

export C_RED='\033[0;31m'
export C_BLUE='\033[0;34m'
export C_CYAN='\033[0;36m'
export C_BLACK='\033[0;30m'
export C_GREEN='\033[0;32m'
export C_WHITE='\033[0;37m'
export C_PURPLE='\033[0;35m'
export C_YELLOW='\033[0;33m'
export C_DARK_GRAY='\033[0;30m'
export C_LIGHT_RED='\033[0;31m'
export C_LIGHT_GRAY='\033[0;37m'
export C_LIGHT_BLUE='\033[0;34m'
export C_LIGHT_CYAN='\033[0;36m'
export C_LIGHT_GREEN='\033[0;32m'
export C_BROWN_ORANGE='\033[0;33m'
export C_LIGHT_PURPLE='\033[0;35m'

export C_RED_BOLD='\033[1;31m'
export C_BLUE_BOLD='\033[1;34m'
export C_CYAN_BOLD='\033[1;36m'
export C_BLACK_BOLD='\033[1;30m'
export C_GREEN_BOLD='\033[1;32m'
export C_WHITE_BOLD='\033[1;37m'
export C_PURPLE_BOLD='\033[1;35m'
export C_YELLOE_BOLD='\033[1;33m'
export C_DARK_GRAE_BOLD='\033[1;30m'
export C_LIGHT_REE_BOLD='\033[1;31m'
export C_LIGHT_GRAY_BOLD='\033[1;37m'
export C_LIGHT_BLUE_BOLD='\033[1;34m'
export C_LIGHT_CYAE_BOLD='\033[1;36m'
export C_LIGHT_GREEE_BOLD='\033[1;32m'
export C_BROWN_ORANGE_BOLD='\033[1;33m'
export C_LIGHT_PURPLE_BOLD='\033[1;35m'

export C_RED_UNDERLINED='\033[4;31m'
export C_BLUE_UNDERLINED='\033[4;34m'
export C_CYAN_UNDERLINED='\033[4;36m'
export C_BLACK_UNDERLINED='\033[4;30m'
export C_GREEN_UNDERLINED='\033[4;32m'
export C_WHITE_UNDERLINED='\033[4;37m'
export C_PURPLE_UNDERLINED='\033[4;35m'
export C_YELLOW_UNDERLINED='\033[4;33m'
export C_DARK_GRAY_UNDERLINED='\033[4;30m'
export C_LIGHT_RED_UNDERLINED='\033[4;31m'
export C_LIGHT_GRAY_UNDERLINED='\033[4;37m'
export C_LIGHT_BLUE_UNDERLINED='\033[4;34m'
export C_LIGHT_CYAN_UNDERLINED='\033[4;36m'
export C_LIGHT_GREEN_UNDERLINED='\033[4;32m'
export C_BROWN_ORANGE_UNDERLINED='\033[4;33m'
export C_LIGHT_PURPLE_UNDERLINED='\033[4;35m'

export C_RED_BLINKING='\033[5;31m'
export C_BLUE_BLINKING='\033[5;34m'
export C_CYAN_BLINKING='\033[5;36m'
export C_BLACK_BLINKING='\033[5;30m'
export C_GREEN_BLINKING='\033[5;32m'
export C_WHITE_BLINKING='\033[5;37m'
export C_PURPLE_BLINKING='\033[5;35m'
export C_YELLOW_BLINKING='\033[5;33m'
export C_DARK_GRAY_BLINKING='\033[5;30m'
export C_LIGHT_RED_BLINKING='\033[5;31m'
export C_LIGHT_GRAY_BLINKING='\033[5;37m'
export C_LIGHT_BLUE_BLINKING='\033[5;34m'
export C_LIGHT_CYAN_BLINKING='\033[5;36m'
export C_LIGHT_GREEN_BLINKING='\033[5;32m'
export C_BROWN_ORANGE_BLINKING='\033[5;33m'
export C_LIGHT_PURPLE_BLINKING='\033[5;35m'

export C_ARG="$C_BLUE"
export C_ERROR="$C_RED"
export C_TEXT="$C_WHITE"
export C_DIR="$C_PURPLE"
export C_NUMBER="$C_GREEN"
export C_REPO="$C_LIGHT_PURPLE"
export C_TEXT_HIGHLIGHT="$C_YELLOW"

# }}}
# {{{ argument variable definitions

DEFAULT_JOBS=8
DEFAULT_LIMIT=25
DEFAULT_DIR="$(pwd)"

DIR=""
DEPTH=0
JOBS=-1
LIMIT=-1
OWNER=""

# }}}
# {{{ functions

echo_usage() {
	local -r SCRIPT_NAME="$1"

	if [[ -z "$SCRIPT_NAME" ]]; then
		echo -e "${C_ERROR}* Script name not provided or empty.${NC}"
		exit 1
	fi

	echo -e "${C_TEXT}* Usage $SCRIPT_NAME${NC} ${C_ARG}[...args]${NC}"
	echo ""
	echo -e "${C_TEXT}For help, run:${NC} ${C_TEXT_HIGHLIGHT}./$SCRIPT_NAME}{NC} ${C_ARG}--help${NC}"
}

echo_description() {
	echo -e "${C_TEXT}  This script can be useful for quickly cloning a large number of${NC}"
	echo -e "${C_TEXT}  repositories from GitHub, especially for organizations with a large${NC}"
	echo -e "${C_TEXT}  number of repositories.${NC}"
	echo ""
	echo -e "${C_TEXT}  It can also be used for backing up all of a user's or organization's${NC}"
	echo -e "${C_TEXT}  repositories to a local machine.${NC}"
	echo ""
	echo -e "${C_TEXT}  By utilizing parallel jobs, the cloning process can be significantly${NC}"
	echo -e "${C_TEXT}  faster compared to cloning each echo repository one by one.${NC}"
}

echo_flags() {
	echo -e " ${C_TEXT_HIGHLIGHT}Flags:${NC}"
	echo ""
	echo -e "  ${C_ARG}--help${NC}${C_TEXT}: Show this help text${NC}"
	echo ""
	echo -e "  ${C_ARG}--depth${NC}${C_TEXT}: The depth at which to clone the remote repository. Defaults${NC}"
	echo -e "           ${C_TEXT}to -1, which is the full history. Any other value will limit${NC}"
	echo -e "           ${C_TEXT}the clone to the specified number of commits.${NC}"
	echo ""
	echo -e "  ${C_ARG}--dir${NC}${C_TEXT}:   Sets the directory where the repositories will be cloned.${NC}"
	echo ""
	echo -e "  ${C_ARG}--owner${NC}${C_TEXT}: Sets the username or organization name to clone all${NC}"
	echo -e "           ${C_TEXT}repositories for.${NC}"
	echo -e "           ${C_TEXT}This argument is required and must be followed by the desired${NC}"
	echo -e "           ${C_TEXT}username or organization name.${NC}"
	echo -e "           ${C_TEXT}Only repositories owned by the specified user or organization${NC}"
	echo -e "           ${C_TEXT}will be echo tloned.${NC}"
	echo ""
	echo -e "  ${C_ARG}--limit${NC}${C_TEXT}: Sets the maximum number of repositories that will be cloned.${NC}"
	echo -e "           ${C_TEXT}This argument is optional and must be followed by a numerical${NC}"
	echo -e "           ${C_TEXT}value. If not specified, all repositories owned by the${NC}"
	echo -e "           ${C_TEXT}specified user or organization will be cloned.${NC}"
	echo ""
	echo -e "  ${C_ARG}--jobs${NC}${C_TEXT}: Sets the number of clone processes to run in parallel.${NC}"
	echo -e "          ${C_TEXT}This argument is optional and must be followed by a numerical${NC}"
	echo -e "          ${C_TEXT}value. If not specified, the default value of 1 will be used.${NC}"
	echo -e "          ${C_TEXT}Increasing the number of jobs can speed up the cloning process,${NC}"
	echo -e "          ${C_TEXT}but may also put more strain on system resources.${NC}"
	echo -e "          ${C_TEXT}It is recommended to use a value that is appropriate for your${NC}"
	echo -e "          ${C_TEXT}system capabilities.${NC}"
}

echo_examples() {
	local -r SCRIPT_NAME="$1"

	if [[ -z "$SCRIPT_NAME" ]]; then
		echo -e "${C_ERROR}Script name not provided or empty.${NC}"
		exit 1
	fi

	echo -e " ${C_TEXT_HIGHLIGHT}Examples:${NC}"
	echo ""
	echo -e "  ${C_TEXT}- To clone a maximum of${NC} ${C_NUMBER}500${NC} ${C_TEXT}repositories owned by the user 'me'${NC}"
	echo -e "    ${C_TEXT}using${NC} ${C_NUMBER}16${NC} ${C_TEXT}parallel jobs into the current directory, run:${NC}"
	echo -e "    ${C_TEXT}\$${NC} ${C_TEXT_HIGHLIGHT}./$SCRIPT_NAME${NC} ${C_ARG}--owner${NC} ${C_TEXT}me${NC} ${C_ARG}--jobs${NC} ${C_TEXT}16${NC} ${C_ARG}--limit${NC} ${C_NUMBER}500${NC} ${C_ARG}--dir${NC} ${C_TEXT}.${NC}"

	echo ""
	echo -e "  ${C_TEXT}- To clone all repositories owned by the organization microsoft"
	echo -e "    ${C_TEXT}using${NC} ${C_NUMBER}20${NC} ${C_TEXT}parallel jobs and a max of${NC} ${C_NUMBER}500${NC} ${C_TEXT}repositories, into the${NC}"
	echo -e "    ${C_TEXT}directory at${NC} ${C_TEXT_HIGHLIGHT}./microsoft${NC}${C_TEXT}, you can run:${NC}"
	echo -e "    ${C_TEXT}\$${NC} ${C_TEXT_HIGHLIGHT}./$SCRIPT_NAME${NC} ${C_ARG}--owner${NC} microsoft ${C_ARG}--jobs${NC} ${C_NUMBER}20${NC} ${C_ARG}--limit${NC} ${C_NUMBER}500${NC} ${C_ARG}--dir${NC} ${C_TEXT_HIGHLIGHT}./microsoft"
}

echo_help() {
	local -r SCRIPT_NAME="$1"

	if [[ -z "$SCRIPT_NAME" ]]; then
		echo -e "${C_ERROR}Script name not provided or empty.${NC}"
		exit 1
	fi

	echo_description
	echo ""
	echo ""
	echo_flags
	echo ""
	echo ""
	echo_examples "$SCRIPT_NAME"
}

# }}}
# {{{ environment validation

FOUND_ERRORS=0

GH_PATH="$(which gh)"
JQ_PATH="$(which jq)"
PARALLEL_PATH="$(which parallel)"

if [[ -z "$GH_PATH" ]]; then
	echo -e "${C_ERROR}Error: The GitHub CLI (gh) is required but was not found.${NC}"

  FOUND_ERRORS=1
fi

if [[ -z "$PARALLEL_PATH" ]]; then
	echo -e "${C_ERROR}Error: The parallel utility is required but was not found.${NC}"

  FOUND_ERRORS=1
fi

# TODO: Recommend actual install commands based on distro
if [[ -z "$JQ_PATH" ]]; then
	echo -e "${C_ERROR}Error: The JQ utility is required but was not found.${NC}"

  FOUND_ERRORS=1
fi

if [[ $FOUND_ERRORS -eq 1 ]]; then
  echo ""
  echo -e "${C_HIGHLIGHT}See --help for usage instructions.${NC}"

  exit 1
fi

# }}}
# {{{ parse arguments

for ((i = 1; i <= $#; i++)); do
	case ${!i} in
	--help)
		echo_help "$SCRIPT_NAME"
		exit 0
		;;
	--jobs)
		next_index=$((i + 1))
		JOBS=${!next_index}
		;;
	--limit)
		next_index=$((i + 1))
		LIMIT=${!next_index}
		;;
	--owner)
		next_index=$((i + 1))
		OWNER=${!next_index}
		;;
	--depth)
		next_index=$((i + 1))
		DEPTH=${!next_index}
		;;
	--dir)
		next_index=$((i + 1))
		DIR=${!next_index}
		;;
	esac
done

# }}}
# {{{ set default values

if [[ $JOBS -eq -1 ]]; then
  JOBS=$DEFAULT_JOBS
fi

if [[ $LIMIT -eq -1 ]]; then
  LIMIT=$DEFAULT_LIMIT
fi

if [[ -z "$DIR" ]]; then
  DIR="$DEFAULT_DIR"
fi

# }}}
# {{{ validate final argument values

FOUND_ERRORS=0

if [[ "$LIMIT" -lt 1 ]]; then
  echo -e "${C_ERROR}Error: --limit must be greater than 1${NC}"

  FOUND_ERRORS=1
fi

if [[ "$JOBS" -lt 1 ]]; then
	echo -e "${C_ERROR}Error: --jobs must be at least 1${NC}"

  FOUND_ERRORS=1
fi

if [[ "$DEPTH" -lt 0 ]]; then
	echo -e "${C_ERROR}Error: --depth must be at least 0 if provided${NC}"

  FOUND_ERRORS=1
fi

if [[ -z "$OWNER" ]]; then
	echo -e "${C_ERROR}Error: --owner not specified${NC}"

  FOUND_ERRORS=1
fi

if [[ ! -d "$DIR" ]]; then
	echo -e "${C_ERROR}Error: Output directory${NC} ${C_DIR}$DIR${NC} ${C_ERROR}does not exist.${NC}"

  FOUND_ERRORS=1
fi

if [[ $FOUND_ERRORS -eq 1 ]]; then
  echo ""
  echo -e "${C_TEXT_HIGHLIGHT}See --help for usage instructions.${NC}"

  exit 1
fi

# }}}

echo -e "${C_TEXT}* Output directory:${NC} ${C_DIR}$DIR${NC}"
echo -e "${C_TEXT}* Cloning at most${NC} ${C_NUMBER}$LIMIT${NC} repositories.${NC}"
echo -e "${C_TEXT}* Will clone${NC} ${C_NUMBER}$JOBS${NC} repositories concurrently.${NC}"

if [[ $DEPTH -eq 0 ]]; then
  echo -e "${C_TEXT}* Will clone${NC} ${C_TEXT_HIGHLIGHT}full${NC} commit histories.${NC}"
else
  echo -e "${C_TEXT}* Will clone only the last${NC} ${C_NUMBER}$DEPTH${NC} commits.${NC}"
fi

echo ""
echo -e "${C_TEXT}* Fetching repositories for${NC} ${C_TEXT_HIGHLIGHT}$OWNER${NC}${C_TEXT}...${NC}"

GH_REPOS_LIST_JSON="$(gh repo list "$OWNER" --json name --limit "$LIMIT")"
mapfile -t GH_REPOS_LIST < <(echo "$GH_REPOS_LIST_JSON" | jq -r '.[].name')
GH_REPOS_LIST_COUNT="${#GH_REPOS_LIST[@]}"

if [[ -z "$GH_REPOS_LIST_COUNT" ]]; then
  echo -e "${C_ERROR}Error: Found no repositories for${NC} ${C_TEXT_HIGHLIGHT}$OWNER${NC}${C_ERROR}.${NC}"
  exit 1
fi

NEW_GH_REPOS_LIST=()
SKIP_COUNT=0

for GH_REPO in "${GH_REPOS_LIST[@]}"; do
	if [[ ! -d "$DIR/$GH_REPO" ]]; then
		NEW_GH_REPOS_LIST+=("$GH_REPO")
  else
    if [[ $SKIP_COUNT -eq 0 ]]; then
      echo ""
    fi

    ((SKIP_COUNT++))

    echo -e "${C_TEXT}- Found existing repository${NC} ${C_REPO}$GH_REPO${NC}${C_TEXT}${NC}"
	fi
done

EXISTING_REPO_COUNT=$((${#GH_REPOS_LIST[@]} - ${#NEW_GH_REPOS_LIST[@]}))
NEW_REPO_COUNT=${#NEW_GH_REPOS_LIST[@]}

if [[ $NEW_REPO_COUNT -eq 0 ]]; then
  echo ""
  echo -e "${C_TEXT}* All repositories already present in${NC} ${C_TEXT_HIGHLIGHT}$DIR${NC}${C_TEXT}, nothing to do${NC}"
  exit 0
fi

echo ""
echo -e "* Found ${C_NUMBER}$GH_REPOS_LIST_COUNT${NC} repositories."
echo -e "* ${C_NUMBER}$NEW_REPO_COUNT${NC} new repositories will be cloned in ${C_TEXT_HIGHLIGHT}$DIR/${NC}"
echo -e "* ${C_NUMBER}$EXISTING_REPO_COUNT${NC} existing repositories will be skipped."
echo ""

REPOS=("${NEW_GH_REPOS_LIST[@]}")

# export variables to be used in the clone_repos function
export ARG_OWNER=$OWNER
export ARG_DIR=$DIR
export ARG_DEPTH=$DEPTH
export C_ERROR
export C_TEXT
export C_TEXT_HIGHLIGHT
export C_NUMBER
export C_REPO
export C_ARG
export C_DIR

clone_repo() {
  REPO="$1"
  CWD="$(pwd)"

  echo -e "${C_TEXT}- Cloning${NC} ${C_REPO}$ARG_OWNER/$REPO${NC}${C_TEXT}...${NC}"

  cd "$ARG_DIR" || exit 1

  if [[ "$ARG_DEPTH" -gt 0 ]]; then
   git clone --depth "$ARG_DEPTH" git@github.com:"$ARG_OWNER"/"$REPO" "$REPO" > /dev/null 2>&1

   cd "$REPO" || exit
   git fetch --all > /dev/null 2>&1
   cd ..
  else
   git clone git@github.com:"$ARG_OWNER"/"$REPO" "$REPO" > /dev/null 2>&1

   cd "$REPO" || exit
   git fetch --all > /dev/null 2>&1
   cd ..
  fi

  cd "$CWD" || exit 1
}


export -f clone_repo

parallel -j "$JOBS" clone_repo ::: "${REPOS[@]}"

# Cleanup variables
unset ARG_OWNER
unset ARG_DIR
unset ARG_DEPTH

unset C_ERROR
unset C_TEXT
unset C_TEXT_HIGHLIGHT
unset C_NUMBER
unset C_REPO
unset C_ARG
unset C_DIR

# {{{ unset color variables

unset C_RED
unset C_BLUE
unset C_CYAN
unset C_BLACK
unset C_GREEN
unset C_WHITE
unset C_PURPLE
unset C_YELLOW
unset C_DARK_GRAY
unset C_LIGHT_RED
unset C_LIGHT_GRAY
unset C_LIGHT_BLUE
unset C_LIGHT_CYAN
unset C_LIGHT_GREEN
unset C_BROWN_ORANGE
unset C_LIGHT_PURPLE

unset C_RED_BOLD
unset C_BLUE_BOLD
unset C_CYAN_BOLD
unset C_BLACK_BOLD
unset C_GREEN_BOLD
unset C_WHITE_BOLD
unset C_PURPLE_BOLD
unset C_YELLOE_BOLD
unset C_DARK_GRAE_BOLD
unset C_LIGHT_REE_BOLD
unset C_LIGHT_GRAY_BOLD
unset C_LIGHT_BLUE_BOLD
unset C_LIGHT_CYAE_BOLD
unset C_LIGHT_GREEE_BOLD
unset C_BROWN_ORANGE_BOLD
unset C_LIGHT_PURPLE_BOLD

unset C_RED_UNDERLINED
unset C_BLUE_UNDERLINED
unset C_CYAN_UNDERLINED
unset C_BLACK_UNDERLINED
unset C_GREEN_UNDERLINED
unset C_WHITE_UNDERLINED
unset C_PURPLE_UNDERLINED
unset C_YELLOW_UNDERLINED
unset C_DARK_GRAY_UNDERLINED
unset C_LIGHT_RED_UNDERLINED
unset C_LIGHT_GRAY_UNDERLINED
unset C_LIGHT_BLUE_UNDERLINED
unset C_LIGHT_CYAN_UNDERLINED
unset C_LIGHT_GREEN_UNDERLINED
unset C_BROWN_ORANGE_UNDERLINED
unset C_LIGHT_PURPLE_UNDERLINED

unset C_RED_BLINKING
unset C_BLUE_BLINKING
unset C_CYAN_BLINKING
unset C_BLACK_BLINKING
unset C_GREEN_BLINKING
unset C_WHITE_BLINKING
unset C_PURPLE_BLINKING
unset C_YELLOW_BLINKING
unset C_DARK_GRAY_BLINKING
unset C_LIGHT_RED_BLINKING
unset C_LIGHT_GRAY_BLINKING
unset C_LIGHT_BLUE_BLINKING
unset C_LIGHT_CYAN_BLINKING
unset C_LIGHT_GREEN_BLINKING
unset C_BROWN_ORANGE_BLINKING
unset C_LIGHT_PURPLE_BLINKING

unset C_ARG
unset C_ERROR
unset C_TEXT
unset C_DIR
unset C_NUMBER
unset C_REPO
unset C_TEXT_HIGHLIGHT

unset NC

# }}}