From 55f9d8f519802d8288f4fc6c1d00eddb9411477f Mon Sep 17 00:00:00 2001
From: z4yx <z4yx@users.noreply.github.com>
Date: Sun, 5 Apr 2020 21:24:24 +0800
Subject: [PATCH] migrating k8s,ELK,mongodb,mysql

---
 ELK.sh        | 36 ++++--------------------------
 gitlab-ce.sh  |  2 +-
 kubernetes.sh | 32 +++------------------------
 mongodb.sh    | 61 ++++++++++-----------------------------------------
 mysql.sh      | 61 ++++-----------------------------------------------
 5 files changed, 24 insertions(+), 168 deletions(-)

diff --git a/ELK.sh b/ELK.sh
index f4bc3d7..89aa3b3 100755
--- a/ELK.sh
+++ b/ELK.sh
@@ -4,6 +4,7 @@ set -o pipefail
 
 _here=`dirname $(realpath $0)`
 apt_sync="${_here}/apt-sync.py" 
+yum_sync="${_here}/yum-sync.py"
 
 BASE_PATH="${TUNASYNC_WORKING_DIR}"
 BASE_URL=${TUNASYNC_UPSTREAM_URL:-"https://packages.elastic.co"}
@@ -17,8 +18,6 @@ declare -A REPO_VERSIONS=(
 	["kibana"]="4.5 4.6"
 )
 
-mkdir -p ${YUM_PATH} ${APT_PATH}
-
 # =================== APT repos ===============================
 if [[ ! -z ${DRY_RUN:-} ]]; then
 	export APT_DRY_RUN=1
@@ -37,35 +36,8 @@ done
 
 # ================ YUM/DNF repos ===============================
 
-cache_dir="/tmp/yum-elk-cache/"
-cfg="/tmp/yum-elk.conf"
-cat <<EOF > ${cfg}
-[main]
-keepcache=0
-
-EOF
-
 for repo in "${!REPO_VERSIONS[@]}"; do
-for version in ${REPO_VERSIONS[$repo]}; do
-cat <<EOF >> ${cfg}
-[${repo}-${version}]
-name=${repo} ${version} packages
-baseurl=${BASE_URL}/${repo}/${version}/centos
-repo_gpgcheck=0
-gpgcheck=0
-enabled=1
-sslverify=0
-
-EOF
+	versions=${REPO_VERSIONS[$repo]}
+	components=${versions// /,}
+	"$yum_sync" "${BASE_URL}/${repo}/@{comp}/centos/" 7 "$components" x86_64 "${repo}-@{comp}" "$YUM_PATH"
 done
-done
-
-if [[ -z ${DRY_RUN:-} ]]; then
-	reposync -c $cfg -d -p ${YUM_PATH} -e ${cache_dir}
-	for repo in "${!REPO_VERSIONS[@]}"; do
-		for version in ${REPO_VERSIONS[$repo]}; do
-			createrepo --update -v -c ${cache_dir} -o ${YUM_PATH}/${repo}-${version}/ ${YUM_PATH}/${repo}-${version}/
-		done
-	done
-fi
-rm $cfg
diff --git a/gitlab-ce.sh b/gitlab-ce.sh
index 9d139c5..4ab2f63 100755
--- a/gitlab-ce.sh
+++ b/gitlab-ce.sh
@@ -14,7 +14,7 @@ YUM_PATH="${BASE_PATH}/yum"
 UBUNTU_PATH="${BASE_PATH}/ubuntu/"
 DEBIAN_PATH="${BASE_PATH}/debian/"
 
-"$yum_sync" "${UPSTREAM}/el/@{os_ver}/x86_64/" 6-8 "gitlab" x86_64 "el@{os_ver}" "$YUM_PATH"
+"$yum_sync" "${UPSTREAM}/el/@{os_ver}/@{arch}/" 6-8 "gitlab" x86_64 "el@{os_ver}" "$YUM_PATH"
 echo "YUM finished"
 
 "$apt_sync" "${UPSTREAM}/ubuntu" @ubuntu-lts main amd64,i386 "$UBUNTU_PATH"
diff --git a/kubernetes.sh b/kubernetes.sh
index 72dee73..85ffaed 100755
--- a/kubernetes.sh
+++ b/kubernetes.sh
@@ -5,6 +5,7 @@ set -o pipefail
 
 _here=`dirname $(realpath $0)`
 apt_sync="${_here}/apt-sync.py"
+yum_sync="${_here}/yum-sync.py"
 
 BASE_PATH="${TUNASYNC_WORKING_DIR}"
 BASE_URL=${TUNASYNC_UPSTREAM_URL:-"https://packages.cloud.google.com"}
@@ -14,38 +15,11 @@ APT_PATH="${BASE_PATH}/apt"
 
 EL_VERSIONS=(kubernetes-el7-aarch64 kubernetes-el7-armhfp kubernetes-el7-x86_64)
 
-mkdir -p ${YUM_PATH} ${APT_PATH}
-
-
 # =================== APT repos ===============================
 "$apt_sync" "${BASE_URL}/apt" "kubernetes-@{debian-current},kubernetes-@{ubuntu-lts}" main amd64,i386,armhf,arm64 "$APT_PATH"
 echo "APT finished"
 
 # =================== YUM/DNF repos ==========================
 
-base_url="${BASE_URL}/yum/repos"
-cache_dir="/tmp/yum-k8s-cache/"
-cfg="/tmp/yum-k8s.conf"
-
-
-if [[ -z ${DRY_RUN:-} ]]; then
-
-	for elver in ${EL_VERSIONS[@]}; do
-
-		echo "=== Syncing $elver"
-		cat << EOF > $cfg
-[main]
-keepcache=0
-[${elver}]
-name=${elver}
-baseurl=${base_url}/${elver}/
-enabled=1
-EOF
-		arch=(${elver//-/ })
-		arch=${arch[-1]}
-		reposync -a "$arch" -c "$cfg" -d -p "${YUM_PATH}" -e "$cache_dir"
-		[[ -d "${YUM_PATH}/${elver}" ]] || mkdir "${YUM_PATH}/${elver}"
-		createrepo --update -v -c "$cache_dir" -o "${YUM_PATH}/${elver}/" "${YUM_PATH}/${elver}/"
-	done
-fi
-
+"$yum_sync" "${BASE_URL}/yum/repos/@{os_ver}/" 7 kubernetes x86_64,armhfp,aarch64 "@{comp}-el@{os_ver}-@{arch}" "$YUM_PATH"
+echo "YUM finished"
diff --git a/mongodb.sh b/mongodb.sh
index 6a67fe4..49cdc3a 100755
--- a/mongodb.sh
+++ b/mongodb.sh
@@ -3,66 +3,29 @@ set -e
 
 _here=`dirname $(realpath $0)`
 apt_sync="${_here}/apt-sync.py" 
+yum_sync="${_here}/yum-sync.py"
 
 BASE_PATH="${TUNASYNC_WORKING_DIR}"
 BASE_URL=${TUNASYNC_UPSTREAM_URL:-"http://repo.mongodb.org"}
 
-YUM_PATH="${BASE_PATH}/yum"
-APT_PATH="${BASE_PATH}/apt"
-
-RHEL_VERSIONS=("6" "7" "8")
 MONGO_VERSIONS=("4.2" "4.0" "3.6")
 STABLE_VERSION="4.2"
 
+YUM_PATH="${BASE_PATH}/yum"
+APT_PATH="${BASE_PATH}/apt"
 UBUNTU_PATH="${APT_PATH}/ubuntu"
 DEBIAN_PATH="${APT_PATH}/debian"
 
-mkdir -p $UBUNTU_PATH $DEBIAN_PATH $YUM_PATH
-
-cache_dir="/tmp/yum-mongodb-cache/"
-cfg="/tmp/mongodb-yum.conf"
-cat <<EOF > ${cfg}
-[main]
-keepcache=0
-
-EOF
-
-for mgver in ${MONGO_VERSIONS[@]}; do
-	for elver in ${RHEL_VERSIONS[@]}; do
-		# Check if mongo/os version combination exists
-		wget -q --spider "https://repo.mongodb.org/yum/redhat/$elver/mongodb-org/$mgver/" \
-		&& cat <<EOF >> ${cfg}
-[el$elver-${mgver}]
-name=el$elver-${mgver}
-baseurl=https://repo.mongodb.org/yum/redhat/$elver/mongodb-org/${mgver}/x86_64/
-repo_gpgcheck=0
-gpgcheck=0
-enabled=1
-sslverify=0
-
-EOF
-	done
+components=$(printf ",%s" "${MONGO_VERSIONS[@]}")
+components=${components:1}
+"$yum_sync" "${BASE_URL}/yum/redhat/@{os_ver}/mongodb-org/@{comp}/@{arch}/" 6-8 "$components" x86_64 "el@{os_ver}-@{comp}" "$YUM_PATH"
+pushd "${YUM_PATH}"
+for stable in el*-${STABLE_VERSION}; do
+	# e.g. "el8" -> "el8-4.2"
+	ln -fsn $stable ${stable%-$STABLE_VERSION}
 done
-
-if [[ -z ${DRY_RUN:-} ]]; then
-	reposync -c $cfg -d -p ${YUM_PATH} -e $cache_dir
-	for mgver in ${MONGO_VERSIONS[@]}; do
-		for elver in ${RHEL_VERSIONS[@]}; do
-			[[ -e "${YUM_PATH}/el$elver-$mgver/" ]] && createrepo --update -v -c $cache_dir -o "${YUM_PATH}/el$elver-$mgver/" "${YUM_PATH}/el$elver-$mgver/"
-		done
-	done
-fi
-
-for elver in ${RHEL_VERSIONS[@]}; do
-	[[ -e "${YUM_PATH}/el$elver-${STABLE_VERSION}" ]] && (cd "${YUM_PATH}" && ln -fsn "el$elver-${STABLE_VERSION}" el$elver)
-done
-
-rm $cfg
-
-
-if [[ ! -z ${DRY_RUN:-} ]]; then
-	export APT_DRY_RUN=1
-fi
+popd
+echo "YUM finished"
 
 base_url="http://repo.mongodb.org"
 for mgver in ${MONGO_VERSIONS[@]}; do
diff --git a/mysql.sh b/mysql.sh
index dafc3f5..38705c2 100755
--- a/mysql.sh
+++ b/mysql.sh
@@ -5,6 +5,7 @@ set -o pipefail
 
 _here=`dirname $(realpath $0)`
 apt_sync="${_here}/apt-sync.py" 
+yum_sync="${_here}/yum-sync.py"
 
 BASE_PATH="${TUNASYNC_WORKING_DIR}"
 BASE_URL="${TUNASYNC_UPSTREAM_URL:-"https://repo.mysql.com"}"
@@ -32,63 +33,9 @@ echo "Ubuntu finished"
 echo "Debian finished"
 
 # =================== YUM/DNF repos ==========================
-
-cache_dir="/tmp/yum-mysql-cache/"
-cfg="/tmp/yum-mysql.conf"
-cat <<EOF > ${cfg}
-[main]
-keepcache=0
-
-EOF
-
-for elver in "6" "7" "8"; do
-cat << EOF >> $cfg
-[mysql-connectors-community-el${elver}]
-name=MySQL Connectors Community
-baseurl=http://repo.mysql.com/yum/mysql-connectors-community/el/$elver/x86_64/
-enabled=1
-
-[mysql-tools-community-el${elver}]
-name=MySQL Tools Community
-baseurl=http://repo.mysql.com/yum/mysql-tools-community/el/$elver/x86_64/
-enabled=1
-
-[mysql80-community-el${elver}]
-name=MySQL 8.0 Community Server
-baseurl=http://repo.mysql.com/yum/mysql-8.0-community/el/$elver/x86_64/
-enabled=1
-
-EOF
-done
-
-for elver in "6" "7"; do
-cat << EOF >> $cfg
-[mysql56-community-el${elver}]
-name=MySQL 5.6 Community Server
-baseurl=http://repo.mysql.com/yum/mysql-5.6-community/el/$elver/x86_64/
-enabled=1
-
-[mysql57-community-el${elver}]
-name=MySQL 5.7 Community Server
-baseurl=http://repo.mysql.com/yum/mysql-5.7-community/el/$elver/x86_64/
-enabled=1
-EOF
-done
-
-if [[ -z ${DRY_RUN:-} ]]; then
-	reposync -c $cfg -d -p ${YUM_PATH} -e $cache_dir
-	for repo in "mysql56-community" "mysql57-community"; do
-		for elver in "6" "7"; do
-			createrepo --update -v -c $cache_dir -o ${YUM_PATH}/${repo}-el${elver}/ ${YUM_PATH}/${repo}-el${elver}/
-		done
-	done
-	for repo in "mysql-connectors-community" "mysql-tools-community" "mysql80-community"; do
-		for elver in "6" "7" "8"; do
-			createrepo --update -v -c $cache_dir -o ${YUM_PATH}/${repo}-el${elver}/ ${YUM_PATH}/${repo}-el${elver}/
-		done
-	done
-fi
-rm $cfg
+COMPONENTS="mysql-connectors-community,mysql-tools-community,mysql-8.0-community,mysql-5.6-community,mysql-5.7-community"
+"$yum_sync" "${BASE_URL}/yum/@{comp}/el/@{os_ver}/@{arch}/" 6-8 "$COMPONENTS" x86_64 "@{comp}-el@{os_ver}" "$YUM_PATH"
+echo "YUM finished"
 
 # --------- dev.mysql.com --------