python中fab模块的应用实例

import getpass
import os

from collections import OrderedDict
from fabric.api import *

def helloworld():
   print "helloworld!"

def hello(name , age):
   print "hello ! I am %s , my age=%s" % (name , age)

def ls():
   local("ls /usr/local")
   output = local('echo hello' , capture=True)
   print output

def setRoles():
   env.roledefs = {
      'hadoop1' : ['[email protected]'] ,
      'hadoop2' : ['[email protected]'] ,
      'hadoop3' : ['[email protected]'] ,
      'hadoop4' : ['[email protected]']            
   }
   env.passwords = {
      '[email protected]:22' : 'A8285310' ,
      '[email protected]:22' : 'A8285310' ,
      '[email protected]:22' : 'A2885310' ,
      '[email protected]:22' : 'A8285310' 
   }

@roles('hadoop2')
def setBaseEnv():

   hadoop1 = str(env.roledefs["hadoop1"]).split('@' , 1)[1]
   hadoop2 = str(env.roledefs["hadoop2"]).split('@' , 1)[1]
   hadoop3 = str(env.roledefs["hadoop3"]).split('@' , 1)[1]
   hadoop4 = str(env.roledefs["hadoop4"]).split('@' , 1)[1]
   #set etc hosts
   with cd('/etc'):
              run('echo -e "{0} hadoop1.cbas\n{1} hadoop2.cbas\n{2} hadoop3.cbas\n{3} hadoop4.cbas" >> hosts'.format(hadoop1.split("'" ,1)[0] , hadoop2.split("'" ,1)[0] , hadoop3.split("'" ,1)[0] , hadoop4.split("'" ,1)[0]))
      run("cat hosts")
      run("echo -----------------------------------------------------------------------------------------------------------")

   #set language
   run("echo LANG=en_US.UTF-8 > /etc/sysconfig/i18n")
   run("source /etc/sysconfig/i18n")
   run("echo $LANG")
   run("echo -----------------------------------------------------------------------------------------------------------")    
   
   #close the iptables and selinux forever
   run("chkconfig iptables off")
   run("chkconfig --list iptables")
   run("sed -i 's/enforcing/disabled/g' /etc/selinux/config")
   run("cat /etc/selinux/config")
   run("echo -----------------------------------------------------------------------------------------------------------")

   #set localtime
   run("\cp /usr/share/zoneinfo/Asia/Shanghai /etc/localtime")
   run("cat /etc/localtime")
   run("echo -----------------------------------------------------------------------------------------------------------")
   
   #set file limit
   run('echo -e "*    soft    nofile    65536\n*    hard    nofile    65536" >> /etc/security/limits.conf')
   run("cat /etc/security/limits.conf")
   run("sed -i '/1024/s/^/#/' /etc/security/limits.d/90-nproc.conf")
   run("echo -----------------------------------------------------------------------------------------------------------")
   
   #set ntpd and httpd  start with the system
   run("chkconfig ntpd on")
   run("chkconfig httpd on")
   run("chkconfig --list ntpd")
   run("chkconfig --list httpd")
   run("echo -----------------------------------------------------------------------------------------------------------")

   #mkdir
   run("mkdir -p /data/apps")
   run('mkdir -p /software')
   run("echo -----------------------------------------------------------------------------------------------------------")
   
   #move centos repo
   run("mv /etc/yum.repos.d/*.repo /etc")
   run("echo -----------------------------------------------------------------------------------------------------------")

   #reboot
   run("reboot")

@roles('hadoop1')
def installBasePak():

   #check selinux
   run("/usr/sbin/sestatus -v")

   #close THP
   run("echo never > /sys/kernel/mm/redhat_transparent_hugepage/defrag")
   run("echo never > /sys/kernel/mm/redhat_transparent_hugepage/enabled")
   run("echo never > /sys/kernel/mm/transparent_hugepage/defrag")
   run("echo never > /sys/kernel/mm/transparent_hugepage/enabled")
   run("echo -----------------------------------------------------------------------------------------------------------")

   with cd('/software'):
      run('unzip basePackage.zip')   
      #install base package
      run("rpm -Uvh nc-1.84-24.el6.x86_64.rpm")
      run("rpm -Uvh snappy-1.1.0-1.el6.x86_64.rpm")
      run("rpm -Uvh snappy-devel-1.1.0-1.el6.x86_64.rpm")
      run("rpm -Uvh mpfr-2.4.1-6.el6.x86_64.rpm")
      run("rpm -Uvh ppl-0.10.2-11.el6.x86_64.rpm")
      run("rpm -Uvh cloog-ppl-0.15.7-1.2.el6.x86_64.rpm")
      run("rpm -Uvh cpp-4.4.7-18.el6.x86_64.rpm")
      run("rpm -Uvh gcc-4.4.7-18.el6.x86_64.rpm")
      run("rpm -Uvh libstdc++-devel-4.4.7-18.el6.x86_64.rpm")
      run("rpm -Uvh gcc-c++-4.4.7-18.el6.x86_64.rpm")
      run("rpm -Uvh zlib-devel-1.2.3-29.el6.x86_64.rpm")
      run("rpm -Uvh python-devel-2.6.6-66.el6_8.x86_64.rpm")
      run("rpm -Uvh perl-5.10.1-144.el6.x86_64.rpm")
      run("rpm -Uvh perl-DBI-1.609-4.el6.x86_64.rpm")
      run("rpm -Uvh perl-DBD-MySQL-4.013-3.el6.x86_64.rpm")
      run("rpm -Uvh mysql-5.1.73-8.el6_8.x86_64.rpm")
      run("rpm -Uvh mysql-server-5.1.73-8.el6_8.x86_64.rpm")
      run("echo -----------------------------------------------------------------------------------------------------------")

@roles('hadoop1')
def sendPak():
   with cd('/software'):
      run('scp basePackage.zip mysql.zip redis.zip [email protected]:/software')
      run('scp basePackage.zip redis.zip nginx.zip [email protected]:/software')
      run('scp basePackage.zip redis.zip nginx.zip ruby.zip ftp.zip hue.tar [email protected]:/software')

@roles('hadoop2')
def test():
   with cd('/software/ruby-2.4.1/ext/openssl'):
                run('ruby extconf.rb --with-openssl-include=/software/openssl1f/include --with-openssl-lib=/software/openssl1f/lib')
                run('rm -rf Makefile')
                run('cp /software/Makefile_openssl /software/ruby-2.4.1/ext/openssl/Makefile')
                run('make')
                run('make install')
        with cd('/software'):
                run('gem install redis-4.0.1.gem')
                run('gem list')
@roles('hadoop1')
def buildHdpMirrorCenter():
   with cd('/software'):
      run('tar -xf AMBARI-2.4.1.0.tar')
      run('tar -xf HDP-2.5.0.0.tar')
      run('tar -xf HDP-UTILS-1.1.0.21.tar')
      run('\cp AMBARI-2.4.1.0/centos6/2.4.1.0-22/ambari.repo  /etc/yum.repos.d')
      run('\cp HDP/centos6/hdp.repo /etc/yum.repos.d')
      run('ln -s /software/AMBARI-2.4.1.0  /var/www/html/AMBARI-2.4.1.0')
      run('ln -s /software/HDP  /var/www/html/HDP')
      run('ln -s /software/HDP-UTILS-1.1.0.21  /var/www/html/HDP-UTILS-1.1.0.21')
      run('service httpd restart')
      run('yum clean all')
      run('yum makecache')
      run('yum repolist')    
      run('curl http://hadoop1.cbas/HDP')
      run('curl http://hadoop1.cbas/HDP-UTILS-1.1.0.21')
      run('curl http://hadoop1.cbas/AMBARI-2.4.1.0')

@roles('hadoop1')
def deployAmbariServer():
   run('yum install -y ambari-server')

@roles('hadoop4')
def deployHue():
   with cd('/software'):
      run('mkdir /hue')
      run('mv hue.tar /hue')
      run('groupadd hue')
      run('useradd -g hue hue')
   with cd('/hue'):
      run('tar -xf hue.tar')
      run('chown hue:hue /hue -R')

@roles('hadoop1' , 'hadoop2')
def deployJDK():
   with cd('/software'):
      run('tar -xf jdk-8u144-linux-x64.tar.gz')
      run('mv jdk1.8.0_144 jdk1.8')
   with cd('/etc'):
      run('echo -e "JAVA_HOME=/software/jdk1.8\nCLASSPATH=.:/software/jdk1.8/lib:/software/jdk1.8/jre/lib\nPATH=/software/jdk1.8/bin:$PATH:/software/jdk1.8/jre/bin\nexport JAVA_HOME CLASSPATH PATH" >> profile')
      run('source profile')
      run('java -version')

@roles('hadoop1' , 'hadoop2')
def deployNginx():
   with cd('/software'):
      run('service httpd stop')
      run('chkconfig httpd off')
      run('unzip nginx.zip')
      run('tar -xf nginx-1.8.1.tar.gz')
      run('tar -xf openssl-1.0.2j.tar.gz') 
      run('tar -xf pcre-8.21.tar.gz') 
      run('tar -xf zlib-1.2.7.tar.gz')
      run('unzip nginx-http-concat-master.zip') 
      run('unzip nginx_upstream_check_module-master.zip')
   with cd('/software/nginx-1.8.1'):
      run('./configure --prefix=/software/nginx --with-pcre=/software/pcre-8.21 --with-zlib=/software/zlib-1.2.7 --add-module=/software/nginx-http-concat-master --add-module=/software/nginx_upstream_check_module-master --with-http_ssl_module  --with-openssl=/software/openssl-1.0.2j --with-http_realip_module')
      run('make')
      run('make install')
      run('/software/nginx/sbin/nginx -t')
      run('/software/nginx/sbin/nginx')

@roles('hadoop1' , 'hadoop2')
def deployRedis():
   with cd('/software'):
      run('unzip redis.zip')
      run('tar -xf redis-3.2.8.tar.gz')
   with cd('/software/redis-3.2.8'):
      run('make')
      run('make install PREFIX=/software/redis')
      run('mkdir /software/redis-cluster')
      run('mkdir /software/redis-cluster/redis6371')
      run('mkdir /software/redis-cluster/redis6372')
      run('cp /software/redis/bin/* /software/redis-cluster/redis6371')
      run('cp /software/redis/bin/* /software/redis-cluster/redis6372')
      run('cp /software/redis6371.conf /software/redis-cluster/redis6371')
      run('cp /software/redis6372.conf /software/redis-cluster/redis6372')
      run('cp /software/redis-3.2.8/src/redis-trib.rb /software/redis-cluster')
   run('/software/redis-cluster/redis6371/redis-server /software/redis-cluster/redis6371/redis6371.conf')
   run('/software/redis-cluster/redis6372/redis-server /software/redis-cluster/redis6372/redis6372.conf')
   run('netstat -ntlp | grep redis')

@roles('hadoop2')
def deployRuby():
   with cd('/software'):
      run('mkdir ruby')
      run('unzip ruby.zip')
      run('tar -xf ruby-2.4.1.tar.gz')
      run('tar -xf openssl-1.0.1f.tar.gz')
   with cd('/software/ruby-2.4.1'):
      run('./configure --prefix=/software/ruby')
      run('make')
      run('make install')
      run('ln -s /software/ruby/bin/ruby /usr/bin/ruby')
      run('ln -s /software/ruby/bin/gem /usr/bin/gem')
   with cd('/software/openssl-1.0.1f'):
      run('./config -fPIC --prefix=/software/openssl1f enable-shared')
      run('./config -t')
      run('make depend')
      run('make install')
   with cd('/software/ruby-2.4.1/ext/zlib'):
      run('ruby extconf.rb')
      run("sed -i '16a top_srcdir = /software/ruby-2.4.1' Makefile")
      run('make')
      run('make install')
   with cd('/software/ruby-2.4.1/ext/openssl'):
      run('ruby extconf.rb --with-openssl-include=/software/openssl1f/include --with-openssl-lib=/software/openssl1f/lib')
      run("sed -i '16a top_srcdir = /software/ruby-2.4.1' Makefile")
      run('make')
      run('make install')
   with cd('/software'):
      run('gem install redis-4.0.1.gem')
      run('gem list')
   with cd('/software/redis-cluster'):
      run('./redis-trib.rb create --replicas 1 hadoop2.cbas:6371 hadoop2.cbas:6372 hadoop3.cbas:6371 hadoop3.cbas:6372 hadoop4.cbas:6371 hadoop4.cbas:6372')

@roles('hadoop4')
def deployFtp():
   with cd('/software'):
      run('unzip ftp.zip')
      run('rpm -ivh vsftpd-2.2.2-24.el6.x86_64.rpm')
      run('rpm -ivh easy-rsa-3.0.3-1.el6.noarch.rpm')
      run('mkdir -p /ftp')
      run('useradd -M -D /ftp -s /sbin/nologin ftptest')
   with cd('/etc/vsftpd'):
      run('rm -rf vsftpd.conf')
      run('cp /software/vsftpd.conf ./')

@roles('hadoop1')
def deployPgsql():
   with cd('/software'):
      run('unzip pgsql.zip')
      run('rpm -ivh postgresql-libs-8.4.20-7.el6.x86_64.rpm')
      run('rpm -ivh postgresql-8.4.20-7.el6.x86_64.rpm')
      run('rpm -ivh postgresql-server-8.4.20-7.el6.x86_64.rpm')

def deployMysql(masterOrslave , password):
   global MYSQLCMD
        MYSQLCMD = "grant replication slave on *.* to 'root'@'hadoop2.cbas' identified by '%s'" % (password)
   if (masterOrslave == 'master' or masterOrslave == 'slave') and not password is None:
      with cd('/software'):
         run('rpm -ivh mysql-connector-java-5.1.37-1.noarch.rpm')
         run('unzip mysql.zip')
         run('tar -xf mysql56.tar')
            run('useradd mysql')
         run('mkdir -p /data/mysql')
         run('rm -rf /etc/my.cnf')
         run('rm -rf /etc/init.d/mysqld')
         run('rm -rf /usr/bin/mysql')
         run('/software/mysql/scripts/mysql_install_db --basedir=/software/mysql --datadir=/data/mysql')
         run('cp /software/mysql/support-files/mysql.server /etc/rc.d/init.d/mysqld')
                        run('chkconfig --add mysqld')
         run('chkconfig mysqld on')
                with cd('/data'):
                        run('touch mysqld.log')
         run('chown mysql:mysql /data -R')
      if masterOrslave == 'master':
         with cd('/software/mysql'):
            run('echo -e "basedir = /software/mysql\ndatadir = /data/mysql\nport = 3306\nsocket = /tmp/mysql.sock\npid-file=/data/mysql.pid\nbind-address=0.0.0.0\nmax_connections=3000\ndefault-storage-engine=innoDB\nbinlog_format=mixed\nserver-id=1\nlog-bin=master-bin\nlog-bin-index=master-bin.index\n[mysqld_safe]\nlog-error = /data/mysqld.log" >> my.cnf')
         with cd('/software/mysql/bin'):
                           run('ln -s /software/mysql/bin/mysql /usr/bin/mysql')
                           run('service mysqld start')
                           run("./mysqladmin -u root password '%s'" % (password))
            run(" ./mysql -uroot -p{0} -e {1}{2};{3} ".format(password , '"' , MYSQLCMD , '"'))
            run('service mysqld restart')
      elif masterOrslave == 'slave':
         with cd('/software/mysql'):
                                run('echo -e "basedir = /software/mysql\ndatadir = /data/mysql\nport = 3306\nsocket = /tmp/mysql.sock\npid-file=/data/mysql.pid\nbind-address=0.0.0.0\nmax_connections=3000\ndefault-storage-engine=innoDB\nbinlog_format=mixed\nserver-id=2\nrelay-log=slave-relay-bin\nrelay-log-index=slave-relay-bin.index\n[mysqld_safe]\nlog-error = /data/mysqld.log" >> my.cnf')
         with cd('/software/mysql/bin'):
            run('ln -s /software/mysql/bin/mysql /usr/bin/mysql')
            run('service mysqld start')
            run("./mysqladmin -u root password '%s'" % (password))
   else:
      run('echo usage:fab -H [email protected](the host which you want to deploy mysql master or slave) setRoles deployMysql:masterOrslave=master|slave , password=Mysqlpassword