shell scripts from greyhoundforty

about

Landing page and home to various shell scripts that I am working on. This site was generated using Jekyll and is hosted using the Keybase Filesystem.

Other places you can find me:

twitter
github
tech blog

backupScript.sh

Backup script using rsnapshot and s3cmd

Overview

This script is used to automate the process of installing and configuring s3cmd and rsnapshot for local “hot” backups and off-site “cold” linux backups. The script also sets up a cron job to compress the current rsnapshot backup directory every night at 10pm and push to a bucket in IBM Cloud Object Storage (S3).

The script was tested on Centos 6/7, Ubuntu 14/16, RHEL 6/7, and Debian 8.

Script Code

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
#!/usr/bin/env bash

# Author: Ryan TIffany
# Email: rtiffany@us.ibm.com

# Script Variables
DIALOG='\033[0;36m' 
WARNING='\033[0;31m'
LINKY='\033[0;41m'
NC='\033[0m'
today=$(date "+%F")
hst=$(hostname -s)

# Short description 
overview() { 
  echo -e "${DIALOG}This script will install s3cmd and rsnapshot on your server and help with some basic backup configurations.${NC}"
  echo -e "${DIALOG}By default rsnapshot is configured to only backup this system, but can be configured to backup remote systems as well.${NC}\n"
}

overview 

check_your_privilege () {
    SUDO=''
    if [[ "$(id -u)" != 0 ]]; then
        SUDO='sudo'
        echo -e "${WARNING}Note: This setup script requires root permissions. You will be prompted for your sudo password.${NC}"
    fi
}

check_your_privilege

set_install_variables()	{

  # OS specific variables.
  if [ -e /etc/redhat-release ] ; then
    echo -e "${DIALOG}Red Hat based distribution detected...${NC}"
    OS_VENDOR=$(awk '{print $1}' /etc/redhat-release | tr '[:lower:]' '[:upper:]')
    if [ "${OS_VENDOR}" = "RED" ]; then
          OS_VENDOR="REDHAT"
    elif [ "${OS_VENDOR}" = "CENTOS" ]; then
          OS_VENDOR="CENTOS"
    fi
    OS_INSTALL_TOOL="/usr/bin/yum -y install"
    MAJOR_VERSION=$(rpm -qa \*-release | grep -Ei "oracle|redhat|centos" | cut -d"-" -f3)
      if [ "${OS_VENDOR}" = "REDHAT" ] && [ "${MAJOR_VERSION}" = "6" ]; then
        $SUDO rpm -Uvh http://download.fedoraproject.org/pub/epel/6/x86_64/epel-release-6-8.noarch.rpm >/dev/null
      elif [ "${OS_VENDOR}" = "CENTOS" ] && [ "${MAJOR_VERSION}" = "6" ]; then
        $SUDO rpm -Uvh http://download.fedoraproject.org/pub/epel/6/x86_64/epel-release-6-8.noarch.rpm >/dev/null 
      elif [ "${OS_VENDOR}" = "REDHAT" ] && [ "${MAJOR_VERSION}" = "7" ]; then
        $SUDO yum install epel-release -y >/dev/null 
      elif [ "${OS_VENDOR}" = "CENTOS" ] && [ "${MAJOR_VERSION}" = "7" ]; then
        $SUDO yum install epel-release -y >/dev/null 
      fi
  elif [ -e /usr/bin/lsb_release ] ; then
    echo -e "${DIALOG}Debian based distribution detected...${NC}"
    OS_VENDOR=$(lsb_release -si | tr '[:lower:]' '[:upper:]')
    #OS_VERSION_MAJOR=$(lsb_release -sr | cut -d. -f1)
    #OS_VERSION_MINOR=$(lsb_release -sr | cut -d. -f2)
    OS_INSTALL_TOOL="apt-get -y install"
  fi
}

install_tools() {
	$SUDO $OS_INSTALL_TOOL s3cmd rsync rsnapshot wget >/dev/null 
}

configure_rsnapshot()
{
  sudo mv /etc/rsnapshot.conf{,.bak}
  sudo wget -O /etc/rsnapshot.conf https://gist.githubusercontent.com/greyhoundforty/6b6975d973f5550fce69b71ed8485d34/raw/4091bb17f03dc9b0a3f745b348e686db14b4027e/rsnapshotv2.conf
  echo
  echo
  echo -n -e "${DIALOG}Please supply the directory you would like to use to store your backups. Use the full path with a trailing slash (example: /backups/)${NC}  "
  read -r RSNAPSHOT_BACKUP_DIR
  echo -e "\n${DIALOG}Set rsnapshot backup directory to ${RSNAPSHOT_BACKUP_DIR} ${NC}  "
  
  sudo sed -i "s|BACKUP_DIR|$RSNAPSHOT_BACKUP_DIR|" /etc/rsnapshot.conf
  echo -e "${DIALOG}Testing rsnapshot configuration.${NC}\n"
  rsnapshot configtest
}

configure_s3cmd() { 

	wget -O "$HOME/.s3cfg" https://gist.githubusercontent.com/greyhoundforty/676814921b8f4367fba7604e622d10f3/raw/f6ce1f2248c415cefac4eec4f1c112ad4a03a0d1/s3cfg
	echo 
	echo 
	echo -n -e "${DIALOG}Please supply your Cloud Object Storage (S3) Access Key:${NC}  "
	read -r -s COS_ACCESS_KEY
	sed -i "s|cos_access_key|$COS_ACCESS_KEY|" "$HOME"/.s3cfg
  echo ""
	echo -n -e "${DIALOG}Please supply your Cloud Object Storage (S3) Secret Key:${NC}  "
	read -r -s COS_SECRET_KEY
	sed -i "s|cos_secret_key|$COS_SECRET_KEY|" "$HOME/.s3cfg"
	echo 
	echo -n -e "${DIALOG}Please supply your Cloud Object Storage (S3) Endpoint:${NC}  "
	read -r ENDPOINT 
  sed -i "s|cos_endpoint|$ENDPOINT|g" "$HOME/.s3cfg"
  echo ""
  echo -e "${DIALOG}We will now test our config file by creating a test bucket based on your systems hostname.${NC}"
  $(which s3cmd) mb s3://"$hst"
  if [ $(s3cmd ls | egrep "$hst" | wc -l) = "1" ];then
    echo -e "${DIALOG}s3cmd configuration test passed. Now Removing test bucket.${NC}"
  $(which s3cmd) rb s3://"$hst" 
  else
    echo -e "${WARNING}Error: Bucket creation did not succeed, double check your HOME/.s3cfg configuration file.${NC}"
  fi
  
}

# Set a basic daily cron to compress our rsnapshot backup directory and send it to s3cmd 
cos_backup_schedule() { 
echo -e "${DIALOG}Setting Daily cron to send backups to Cloud Object Storage${NC}"
cat <<EOF > dailybackup
00 5 * * * $(which tar) -czf ${today}.backup.tar.gz ${RSNAPSHOT_BACKUP_DIR}
EOF

sudo mv dailybackup /etc/cron.d/
}

# Echo out some post install information
post_install() {
  echo ""
  echo "-------------------------------------------------------------------------------"
  echo -e "${DIALOG}Installation and configuration of rsnapshot and s3cmd has completed.${NC}\n"
  echo -e "${DIALOG}Important file locations:${NC}"
  echo -e "Rsnapshot Configuration File - ${LINKY}/etc/rsnapshot.conf${NC}"
  echo -e "Rsnapshot Cronjob File - ${LINKY}/etc/rsnapshot.conf${NC}"
  echo -e "Rsnapshot Configuration File - ${LINKY}/etc/cron.d/rsnapshot${NC}\n"
  echo -e "${DIALOG}Please note that by default this script only configures rsnapshot to backup this system.${NC}" 
  echo -e "${DIALOG}If you would like to add remote systems for rsnapshot to also backup, you will need to edit the ${LINKY}/etc/rsnapshot.conf file.${NC}\n"
  echo -e "${DIALOG}The following guide should assist in setting up remote hosts in rsnapshot: ${LINKY}https://github.com/greyhoundforty/COSTooling/blob/master/rsnapshot.md${NC}\n"
  echo -e "${DIALOG}This script also installed a basic daily cron job to compress the RSNAPSHOT_BACKUP_DIR, timestamp the backup with todays date and send it to Cloud Object Storage."
  echo -e "${DIALOG}To update when this process runs please edit the file: ${LINKY}/etc/cron.d/dailybackup${NC}"
  echo "-------------------------------------------------------------------------------"
}


set_install_variables
install_tools
configure_rsnapshot
configure_s3cmd
cos_backup_schedule
post_install

Gist Link

minioCluster.sh

Distributed minio cluster on SoftLayer

Overview

This script can be used to set up a distributed minio cluster on SoftLayer VSIs. The script is best used as a post provisioning script and takes advantage of the SoftLayer userdata service.

Script Code

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
#!/bin/sh

## Disable cloud-init warning on Ubuntu 16 
touch /root/.cloud-warnings.skip

hst=$(hostname -s)

mkfs.btrfs /dev/xvd{c,e,f,g}
mkdir -p /storage
mount /dev/xvdc /storage

## meta data bit 
mkdir /meta && mount /dev/xvdh1 /meta
cat /meta/meta.js | cut -c 3- | rev | cut -c 3- | rev > run-userdata
awk '{gsub(/\\n/,"\n")}1' run-userdata > userdata
sed -i 's@\\\\@\x0@g' userdata
sed -i 's@\\@@g' userdata
sed -i 's@\x0@\\@g' userdata
ACCESS_KEY=$(cat userdata | grep accessKey | awk '{print $2}' | cut -d '"' -f 2)
SECRET_KEY=$(cat userdata | grep secretKey | awk '{print $2}' | cut -d '"' -f 2)

wget https://dl.minio.io/server/minio/release/linux-amd64/minio
chmod +x minio 
mv minio /usr/local/bin/

mkdir /root/.minio
wget -O /root/.minio/config.json https://greyhoundforty.keybase.pub/scripts/minio_config.json
sed -i "s/exampleAccessKey/$ACCESS_KEY/g" /root/.minio/config.json
sed -i "s/exampleSecretKey/$SECRET_KEY/g" /root/.minio/config.json

privateIP=$(ip addr show eth0|grep inet | grep -v inet6 | awk '{print $2}' | cut -d '/' -f 1)

wget -O /etc/default/minio https://greyhoundforty.keybase.pub/scripts/default_minio

# sed -i "s/exampleIP/$privateIP/g" /etc/default/minio 

wget -O /etc/systemd/system/minio.service  https://raw.githubusercontent.com/minio/minio-service/master/linux-systemd/minio.service
sed -i "s/minio-user/root/g" /etc/systemd/system/minio.service

systemctl enable minio.service
sleep 60 && reboot