ELK stack syncing script

ELK.sh

@@ -0,0 +1,72 @@
+#!/bin/bash
+set -e
+set -o pipefail
+
+_here=`dirname $(realpath $0)`
+. ${_here}/helpers/apt-download
+
+[ -z "${LOADED_APT_DOWNLOAD}" ] && (echo "failed to load apt-download"; exit 1)
+
+BASE_PATH="${TUNASYNC_WORKING_DIR}"
+BASE_URL=${TUNASYNC_UPSTREAM_URL:-"https://packages.elastic.co"}
+
+YUM_PATH="${BASE_PATH}/yum"
+APT_PATH="${BASE_PATH}/apt"
+
+declare -A REPO_VERSIONS=(
+	["elasticsearch"]="2.3"
+	["logstash"]="2.3 5.0"
+	["kibana"]="4.5"
+)
+
+mkdir -p ${YUM_PATH} ${APT_PATH}
+
+# =================== APT repos ===============================
+# export APT_DRY_RUN=1
+ 
+for repo in "${!REPO_VERSIONS[@]}"; do
+	# magic here, don't quote ${REPO_VERSIONS[$repo][@]}
+	# so that bash loops over space-sperated tokens
+	for version in ${REPO_VERSIONS[$repo]}; do
+		echo $repo-$version
+		apt_url="${BASE_URL}/${repo}/${version}/debian"
+		dest_path="${APT_PATH}/${repo}/${version}"
+		[[ ! -d ${dest_path} ]] && mkdir -p ${dest_path}
+		apt-download-binary ${apt_url} "stable" "main" "amd64" "${dest_path}" || true
+		apt-download-binary ${apt_url} "stable" "main" "i386" "${dest_path}" || true
+	done
+done
+
+# ================ YUM/DNF repos ===============================
+
+cache_dir="/tmp/yum-elk-cache/"
+cfg="/tmp/yum-elk.conf"
+cat <<EOF > ${cfg}
+[main]
+keepcache=0
+
+EOF
+
+for repo in "${!REPO_VERSIONS[@]}"; do
+for version in ${REPO_VERSIONS[$repo]}; do
+yum_url=
+dest_path="${APT_PATH}/${repo}/${version}"
+cat <<EOF >> ${cfg}
+[${repo}-${version}]
+name=${repo} ${version} packages
+baseurl=${BASE_URL}/${repo}/${version}/centos
+repo_gpgcheck=0
+gpgcheck=0
+enabled=1
+sslverify=0
+
+EOF
+done
+done
+
+reposync -c $cfg -d -p ${YUM_PATH} -e ${cache_dir}
+for repo in "${!REPO_VERSIONS[@]}"; do
+	for version in ${REPO_VERSIONS[$repo]}; do
+		createrepo --update -v -c ${cache_dir} -o ${YUM_PATH}/${repo}-${version}/ ${YUM_PATH}/${repo}-${version}/
+	done
+done

helpers/apt-download

@@ -17,11 +17,11 @@ function check-and-download() {
 
 
 function apt-download-binary() {
-	base_url=$1
+	local base_url=$1
-	dist=$2
+	local dist=$2
-	repo=$3
+	local repo=$3
-	arch=$4
+	local arch=$4
-	dest_base_dir=$5
+	local dest_base_dir=$5
 	if [ -z $dest_base_dir ]; then
 		echo "Destination directory is empty, cannot continue"
 		return 1
@@ -36,7 +36,10 @@ function apt-download-binary() {
 	
 	check-and-download "${base_url}/dists/${dist}/Contents-${arch}.gz" "${dist_tmp_dir}/Contents-${arch}.gz" || true
 	check-and-download "${base_url}/dists/${dist}/InRelease"  "${dist_tmp_dir}/InRelease" || true
-	check-and-download "${base_url}/dists/${dist}/Release" "${dist_tmp_dir}/Release"
+	check-and-download "${base_url}/dists/${dist}/Release" "${dist_tmp_dir}/Release" || {
+		echo "Invalid Repository"
+		return 1
+	}
 	check-and-download "${base_url}/dists/${dist}/Release.gpg" "${dist_tmp_dir}/Release.gpg" || true
 	
 	# Load Package Index URLs from Release file
@@ -133,10 +136,14 @@ function apt-download-binary() {
 			fi
 		fi
 		while [ $downloaded != true ]; do
-			echo "downloading ${pkg_url}"
+			echo "downloading ${pkg_url} to ${dest_filename}"
-			wget -q -O ${dest_filename} ${pkg_url} && {
+			if [[ -z ${APT_DRY_RUN} ]]; then
-				echo "${pkg_checksum}  ${dest_filename}" | ${checksum_cmd} -c - && downloaded=true # two space for md5sum/sha1sum/sha256sum check format
+				wget -q -O ${dest_filename} ${pkg_url} && {
-			}
+					echo "${pkg_checksum}  ${dest_filename}" | ${checksum_cmd} -c - && downloaded=true # two space for md5sum/sha1sum/sha256sum check format
+				}
+			else
+				downloaded=true
+			fi
 		done
 	done