Merge remote-tracking branch 'upstream/master'

Author: ipitio

README.md

@@ -58,7 +58,7 @@ git clone --depth=1 -b master --single-branch [https|ssh]://<PAT>@github.com/pkg
 - `-m 0` ensures only the public packages of the owners you've added are updated (default)
   - You'll need the proper permissions to update private packages
 - `-d 0` allows everything to be updated in one go
-  - A graceful restart is initiated every 4.5 hours by default
+  - A graceful restart is initiated every 4 hours by default
 
 ```bash
 echo "[Unit]
@@ -74,7 +74,7 @@ ExecStart=/usr/bin/sh -c '                   \\
   GITHUB_TOKEN=<PAT>                        ;\\
   GITHUB_OWNER=<username>                   ;\\
   GITHUB_REPO=backage                       ;\\
-  GITHUB_BRANCH=master                      ;\\
+  GITHUB_BRANCH=                      ;\\
   BKG_PATH=\$GITHUB_REPO/\$GITHUB_BRANCH    ;\\
   mkdir -p /opt/\$BKG_PATH                  ;\\
   docker run -v /opt/\$BKG_PATH:/app         \\

src/bkg.sh

@@ -141,8 +141,8 @@ main() {
                 mv "$connections".bak "$connections"
 
                 echo "$(
+					missed_owners
                     cat "$BKG_OWNERS"
-                    find "$BKG_INDEX_DIR" -mindepth 1 -maxdepth 1 -type d -exec basename {} \; 2>/dev/null | sort -u | awk '{print "0/"$1}'
                 )" >"$BKG_OWNERS"
 
                 : >all_owners_in_db
@@ -155,7 +155,7 @@ main() {
                 mv owners.tmp "$BKG_OWNERS"
                 grep -vFxf all_owners_in_db "$connections" >"$temp_connections"
 
-                if [[ "$pkg_left" == "0" || "${db_size_curr::-5}" == "${db_size_prev::-5}" ]]; then
+                if [[ "$pkg_left" == "0" || "${db_size_curr::-4}" == "${db_size_prev::-4}" ]]; then
                     set_BKG BKG_BATCH_FIRST_STARTED "$today"
                     rm -f packages_to_update
                     \cp packages_all packages_to_update
@@ -194,15 +194,17 @@ main() {
         else
             save_owner "$GITHUB_OWNER"
             get_membership "$GITHUB_OWNER" >"$connections"
-            [ ! -s "$connections" ] || env_parallel --lb save_owner <"$connections"
+            if [ -s "$connections" ]; then
+				env_parallel --lb save_owner <"$connections" || while read -r connection; do save_owner "$connection"; done <"$connections"
+			fi
         fi
 
         rm -f "$connections"
         rm -f "$temp_connections"
         BKG_BATCH_FIRST_STARTED=$(get_BKG BKG_BATCH_FIRST_STARTED)
         [ -d "$BKG_INDEX_DIR" ] || mkdir "$BKG_INDEX_DIR"
 
-        if [ "$GITHUB_OWNER" = "ipitio" ]; then
+        if [[ "$GITHUB_OWNER" = "ipitio" && "$(git branch --show-current)" = "master" ]]; then
             get_BKG_set BKG_OWNERS_QUEUE | env_parallel --lb update_owner
         else # typically fewer owners
             run_parallel update_owner "$(get_BKG_set BKG_OWNERS_QUEUE)"

src/lib/owner.sh

@@ -124,31 +124,16 @@ update_owner() {
         find "$BKG_INDEX_DIR/$owner" -type f -name '*.json' ! -name '.*' -print0 | xargs -0 jq -cs '[.] | add' >"$BKG_INDEX_DIR/$owner/.json.tmp"
         jq -cs '{ ("package"): . }' "$BKG_INDEX_DIR/$owner/.json.tmp" >"$BKG_INDEX_DIR/$owner/.json"
         ytoxt "$BKG_INDEX_DIR/$owner/.json"
-
-		# if .package exists, split it into array
-		if jq -e '.package' "$BKG_INDEX_DIR/$owner/.json" &>/dev/null; then
-			jq -c '.package[]' "$BKG_INDEX_DIR/$owner/.json" >"$BKG_INDEX_DIR/$owner/.json.tmp"
-        	mv -f "$BKG_INDEX_DIR/$owner/.json.tmp" "$BKG_INDEX_DIR/$owner/.json"
-		fi
+		jq -c '.package[]' "$BKG_INDEX_DIR/$owner/.json" >"$BKG_INDEX_DIR/$owner/.json.tmp" 2>/dev/null
+        mv -f "$BKG_INDEX_DIR/$owner/.json.tmp" "$BKG_INDEX_DIR/$owner/.json" 2>/dev/null
 
         echo "Creating $owner repo arrays..."
-		# shellcheck disable=SC2086,SC2016
-		parallel '
-			repo={}
-			repo_dir="$BKG_INDEX_DIR/'"$owner"'/$repo"
-			src="$BKG_INDEX_DIR/'"$owner"'/.json"
-			tmp="$repo_dir/.json.tmp"
-			dst="$repo_dir/.json"
-
-			jq -c --arg repo "$repo" '"'"'[.[] | select(.repo == $repo)]'"'"' "$src" >"$tmp" &&
-			jq -cs '"'"'{ ("package"): . }'"'"' "$tmp" >"$dst" &&
-			ytoxt "$dst"
-			if jq -e '"'"'.package'"'"' "$dst" &>/dev/null; then
-				jq -c '"'"'.package[]'"'"' "$dst" >"$tmp"
-				mv -f "$tmp" "$dst"
-			fi
-		' ::: $owner_repos
-	fi
+        parallel "jq -c --arg repo {} '[.[] | select(.repo == \$repo)]' \"$BKG_INDEX_DIR/$owner/.json\" > \"$BKG_INDEX_DIR/$owner/{}/.json.tmp\"" <<<"$owner_repos"
+        xargs -I {} bash -c "jq -cs '{ (\"package\"): . }' \"$BKG_INDEX_DIR/$owner/{}/.json.tmp\" > \"$BKG_INDEX_DIR/$owner/{}/.json\"" <<<"$owner_repos"
+        xargs -I {} bash -c "ytoxt \"$BKG_INDEX_DIR/$owner/{}/.json\"" <<<"$owner_repos"
+		xargs -I {} bash -c "jq -c '.package[]' \"$BKG_INDEX_DIR/$owner/{}/.json\" > \"$BKG_INDEX_DIR/$owner/{}/.json.tmp\"" 2>/dev/null <<<"$owner_repos"
+        xargs -I {} mv -f "$BKG_INDEX_DIR/$owner/{}/.json.tmp" "$BKG_INDEX_DIR/$owner/{}/.json" 2>/dev/null <<<"$owner_repos"
+    fi
 
     sed -i '/^\(.*\/\)*'"$owner"'$/d' "$BKG_OWNERS"
     echo "Updated $owner"

src/lib/package.sh

@@ -121,7 +121,7 @@ update_package() {
         sqlite3 "$BKG_INDEX_DB" "select id from '$table_version_name' where date >= '$BKG_BATCH_FIRST_STARTED';" | sort -u >"${table_version_name}"_already_updated
         local break_now=false
 
-        for page in $(seq 0 5); do
+        for page in $(seq 0 1); do
             ((page > 0)) || continue
             local pages_left=0
             page_version "$page"