mysql_backup_extract.py

#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
Desc: decrypt the db backup file,  then to ungzip it , finally, to be a  origin state.
Date: 2016-08-10
Email: zhanggong1@jd.com
Version: v0.1
"""
import commands
import sys
import os
import getopt
import errno
import shutil
import glob
import subprocess
class CBackupTool(object):
#for mysql
CBACKUP_TOOL_DEFAULT_MYSQL_VERSION = "5.7"
MYSQL_VERSION_ALLOW_LIST = ["5.6", "5.7", "8.0", "10.2"]
XTRABACKUP = 'XtraBackUp'
INNOBACKUPEX = 'InnoBackupEx'
#tool action list
CBACKUP_TOOL_SNAPSHOT_EXTRACT_ACTION = "snapshot"
CBACKUP_TOOL_BINLOG_EXTRACT_ACTION = "binlog"
#snapshot flag string
CBACKUP_TOOL_SNAPSHOT_FLAG = ".xbstream"
CBACKUP_TOOL_SNAPSHOT_RESULT_DIR_NAME = "tmp_snapshot"
#shell const
CBACKUP_TOOL_SHELL_SUCCESS_STATUS = 0
#sys.exit: status code
CBACKUP_TOOL_SUCCESS = 0
CBACKUP_TOOL_FAILED = 1
CBACKUP_TOOL_PARAMETER_NOT_ENOUGH = 2
CBACKUP_TOOL_WRONG_PARAMETER = 3
#check xbstream return code
CBACKUP_XB_NOT_EXIST = -1
CBACKUP_XB_GO_WRONG = -2
CBACKUP_XB_OK = 0
#check qpress return code
qpress_path = '/usr/bin'
CBACKUP_QP_NOT_EXIST = -1
CBACKUP_QP_GO_WRONG = -2
CBACKUP_QP_FILE_NOT_EXIST = -3
CBACKUP_QP_OK = 0
@classmethod
def usage(cls):
help_msg = '''Attention: the result of the command is to be stored in current dir.
Prerequisites: openssl, gzip, tee, python[suggestion:>=2.7], percona xtrabackup  
Usage: 
python mysql_backup_extract.py  -v 5.6 -f ./backup.xbstream.gz.enc
python mysql_backup_extract.py  -f ./mysql-bin.000551.gz.enc
-h show usage
-k decrypt key, default value is none
-f a backup file path.
-v mysql version, the default version is: 5.6 , you just only allow to select one from list : ''' + ' , '.join(cls.MYSQL_VERSION_ALLOW_LIST)
version = '''
-- 5.6 apply to MySQL5.6
-- 5.7 apply to MySQL5.7, Percona5.7
-- 8.0 apply to MySQL8.0
-- 10.2 apply to MariaDB10.2
'''
print help_msg + version
@classmethod
def rds_snapshot_extract_cmd(cls, decrypt_key, infile, out_dir, flag):
if flag == cls.INNOBACKUPEX:
return "openssl enc -d -aes-256-cbc -salt -pass pass:'%s'  -in '%s' | gzip -d -c | xbstream -x -v -C  '%s'" /
% (decrypt_key, infile, out_dir)
elif flag == cls.XTRABACKUP:
return '''xbstream -x < {infile} -C {out_dir}; /
xtrabackup --decrypt=AES256 --encrypt-key="{decrypt_key}" --decompress --remove-original /
--target-dir={out_dir}'''.format(infile=infile, out_dir=out_dir, decrypt_key=decrypt_key)
@classmethod
def rds_binlog_extract_cmd(cls, decrypt_key, infile, outfile):
return "openssl enc -d -aes-256-cbc -salt -pass pass:'%s'  -in '%s' | gzip -d | tee '%s'" % (decrypt_key, infile, outfile)
@classmethod
def cmd_dispatch(cls, argv):
try:
opts, args = getopt.getopt(argv, "hk:f:v:")
except getopt.GetoptError:
cls.usage()
sys.exit(cls.CBACKUP_TOOL_WRONG_PARAMETER)
decrypt_key = ""
backup_file = ""
mysql_version = cls.CBACKUP_TOOL_DEFAULT_MYSQL_VERSION
for opt, arg in opts:
if opt == '-k':
decrypt_key = arg
elif opt == '-f':
backup_file = arg
elif opt == '-v':
mysql_version = arg
elif opt == '-h':
cls.usage()
sys.exit(cls.CBACKUP_TOOL_SUCCESS)
if len(backup_file) <= 0 or len(mysql_version) <= 0:
print "except -h option, the other options ,  you must provide it!!!"
cls.usage()
sys.exit(cls.CBACKUP_TOOL_PARAMETER_NOT_ENOUGH)
if not ((mysql_version.strip()) in cls.MYSQL_VERSION_ALLOW_LIST):
print "illegal mysql version, just only allow to select one from the list: " + ' , '.join(cls.MYSQL_VERSION_ALLOW_LIST)
sys.exit(cls.CBACKUP_TOOL_WRONG_PARAMETER)
if not os.path.isfile(backup_file):
print "dear, please provide a real backup file path!"
sys.exit(cls.CBACKUP_TOOL_WRONG_PARAMETER)
#check whether it is a snapshot or binlog backup file
do_action = ""
path, name = os.path.split(backup_file)
if cls.CBACKUP_TOOL_SNAPSHOT_FLAG in name:
if name.split('.')[-1] == cls.CBACKUP_TOOL_SNAPSHOT_FLAG.split('.')[-1]:
flag = cls.XTRABACKUP
if not decrypt_key:
decrypt_key = "qKevZky3OrhGnLFqLMD25PJF77OsCJb6"
else:
flag = cls.INNOBACKUPEX
if not decrypt_key:
decrypt_key = "default_aes_cbc_key"
do_action = cls.CBACKUP_TOOL_SNAPSHOT_EXTRACT_ACTION
else:
if not decrypt_key:
decrypt_key = "default_aes_cbc_key"
do_action = cls.CBACKUP_TOOL_BINLOG_EXTRACT_ACTION
#start to dispatch command.
if do_action == cls.CBACKUP_TOOL_SNAPSHOT_EXTRACT_ACTION:
cls._unpack_snapshot_to_current_dir(decrypt_key, backup_file, mysql_version, flag)
else:
cls._unpack_binlog_to_current_dir(decrypt_key, backup_file)
#cp -f /tmp/rds_backup_tools/user.* $dst_dir/mysql/
@classmethod
def replace_mysql_user_table(cls, snapshot_out_dir, mysql_version):
script_file_path = os.path.realpath(__file__)
script_file_parent_dir = os.path.dirname(script_file_path)
if len(snapshot_out_dir) > 0:
if os.path.exists(snapshot_out_dir):
src_file_pattern_path = script_file_parent_dir + os.path.sep + mysql_version + os.path.sep + "user.*"
dst_dir = snapshot_out_dir + os.path.sep + "mysql"
for src_file in glob.glob(src_file_pattern_path):
shutil.copy(src_file, dst_dir)
@classmethod
def remove_snapshot_out_dir(cls, snapshot_out_dir):
if len(snapshot_out_dir) > 0 and os.path.exists(snapshot_out_dir):
shutil.rmtree(snapshot_out_dir, ignore_errors=True)
##check if xbstream exists.
@classmethod
def check_xbstream(cls):
try:
subprocess.call(["xbstream", "-?"], stdout=open(os.devnull, "w"))
except OSError as e:
if e.errno == os.errno.ENOENT:
#xbstream is not exists.
return cls.CBACKUP_XB_NOT_EXIST
else:
#xbstream went wrong.
return cls.CBACKUP_XB_GO_WRONG
return cls.CBACKUP_XB_OK
@classmethod
def create_snapshot_output_dir(cls, out_dir):
if len(out_dir) > 0 and not os.path.exists(out_dir):
try:
os.makedirs(out_dir)
except OSError as e:
if e.errno == errno.EEXIST and os.path.isdir(out_dir):
return True
return False
return True
@classmethod
def _check_qpress(cls):
try:
qpress_file = os.popen('whereis qpress').read().strip().split(':')[1].strip()
if not os.path.isfile(qpress_file) or not os.access(qpress_file, os.X_OK):
return cls.CBACKUP_QP_NOT_EXIST
return cls.CBACKUP_QP_OK
except Exception:
return cls.CBACKUP_QP_NOT_EXIST
@classmethod
def check_install_qpress(cls):
qpress_rc = cls._check_qpress()
if qpress_rc != cls.CBACKUP_XB_OK:
script_dir = os.path.split(os.path.realpath(__file__))[0]
qpress_file = "%s/qpress" % script_dir
if not os.path.isfile(qpress_file):
return cls.CBACKUP_QP_FILE_NOT_EXIST
else:
try:
shutil.copy(qpress_file, cls.qpress_path)
subprocess.call(["chmod", "777", qpress_file], stdout=open(os.devnull, "w"))
qpress_rc = cls._check_qpress()
if qpress_rc != cls.CBACKUP_XB_OK:
return cls.CBACKUP_QP_GO_WRONG
else:
return cls.CBACKUP_QP_OK
except Exception:
return cls.CBACKUP_QP_GO_WRONG
else:
return cls.CBACKUP_QP_OK
@classmethod
def _unpack_snapshot_to_current_dir(cls, decrypt_key, backup_file, mysql_version, flag):
if len(decrypt_key) > 0 and len(backup_file) > 0:
if os.path.isfile(backup_file):
absolute_backup_file_path = os.path.abspath(backup_file)
snapshot_out_dir = os.getcwd() + os.path.sep + cls.CBACKUP_TOOL_SNAPSHOT_RESULT_DIR_NAME
#clean garbage
cls.remove_snapshot_out_dir(snapshot_out_dir)
#check if xbstream exists.
xb_rc = cls.check_xbstream()
if xb_rc != cls.CBACKUP_XB_OK:
print "database xtrabackup not exist, or not in PATH env variable."
sys.exit(cls.CBACKUP_TOOL_FAILED)
#create snapshot extract result dir
if not cls.create_snapshot_output_dir(snapshot_out_dir):
print "sorry, create snapshot extract result dir failed, please create it by yourself"
sys.exit(cls.CBACKUP_TOOL_FAILED)
#check qpress tool if xtrabackup
if flag in ("XtraBackUp"):
qpress_rc = cls.check_install_qpress()
if qpress_rc != cls.CBACKUP_XB_OK:
print "qpress not exist, or not in PATH env variable, please install qpress by yourself"
sys.exit(cls.CBACKUP_TOOL_FAILED)
decrypt_cmd_str = cls.rds_snapshot_extract_cmd(decrypt_key, absolute_backup_file_path,
snapshot_out_dir, flag)
#print decrypt_cmd_str
print "please wait..."
rc = commands.getstatusoutput(decrypt_cmd_str)
if rc[0] == cls.CBACKUP_TOOL_SHELL_SUCCESS_STATUS:
cls.replace_mysql_user_table(snapshot_out_dir, mysql_version)
print "Congratulations, both decrypt, unzip and xbstream are successfull, now you can start to restore your DB!"
sys.exit(cls.CBACKUP_TOOL_SUCCESS)
else:
print "decrypt ,gzip or xbstream failed, please check key and other arguments!"
sys.exit(cls.CBACKUP_TOOL_FAILED)
else:
print "decrypt key and backup file must be provided"
sys.exit(cls.CBACKUP_TOOL_WRONG_PARAMETER)
@classmethod
def _unpack_binlog_to_current_dir(cls, decrypt_key, backup_file):
if len(decrypt_key) > 0 and len(backup_file) > 0:
if os.path.isfile(backup_file):
absolute_backup_file_path = os.path.abspath(backup_file)
backup_file_full_name = os.path.basename(absolute_backup_file_path)
out_file_name = backup_file_full_name.split('.')[0] + "." + backup_file_full_name.split('.')[1]
decrypt_cmd_str = cls.rds_binlog_extract_cmd(decrypt_key, absolute_backup_file_path, out_file_name)
#print decrypt_cmd_str
print "please wait..."
rc = commands.getstatusoutput(decrypt_cmd_str)
if rc[0] == cls.CBACKUP_TOOL_SHELL_SUCCESS_STATUS:
print "Congratulations , both decrypt and unzip are successfull, now you can start to restore your DB!"
sys.exit(cls.CBACKUP_TOOL_SUCCESS)
else:
print "decrypt or gzip failed, please check key and other arguments!"
sys.exit(cls.CBACKUP_TOOL_FAILED)
else:
print "decrypt key and backup file must be provided"
sys.exit(cls.CBACKUP_TOOL_WRONG_PARAMETER)
if __name__ == "__main__":
if len(sys.argv) == 1:
CBackupTool.usage()
sys.exit(CBackupTool.CBACKUP_TOOL_PARAMETER_NOT_ENOUGH)
CBackupTool.cmd_dispatch(sys.argv[1:])