backup/seal
1
0
Fork 0
mirror of https://github.com/hequan2017/seal.git synced 2026-05-04 13:42:02 +08:00
Code Issues Projects Releases Wiki Activity

v0.4.1 增加sql审核 app

Browse Source
This commit is contained in:
何全
2019-06-13 15:50:13 +08:00
parent 09352ac1c7
commit a3b2b62b55
10 changed files with 279 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
README.md
View File

@@ -36,8 +36,10 @@
* 二期: k8s管理平台 (开发中)
* node/service/pod 列表 (已完成)
* pod webssh (已完成)
* 三期: mysql sql语句审核(开发中)
* 引擎 goInception
## DEMO
> http://129.28.156.219:8001

3
assets/tests.py
View File

@@ -1,3 +0,0 @@
from django.test import TestCase
# Create your tests here.

5
k8s/apps.py Normal file
View File

@@ -0,0 +1,5 @@
from django.apps import AppConfig
class K8sConfig(AppConfig):
name = 'k8s'

2
requirements.txt
View File

@@ -68,6 +68,7 @@ parso==0.2.1
pathtools==0.1.2
pexpect==4.6.0
pickleshare==0.7.4
prettytable==0.7.2
prometheus-client==0.2.0
promise==2.2.1
prompt-toolkit==1.0.15
@@ -77,6 +78,7 @@ pyasn1-modules==0.2.5
pycparser==2.19
Pygments==2.2.0
PyHamcrest==1.9.0
PyMySQL==0.9.3
PyNaCl==1.3.0
python-crontab==2.3.6
python-dateutil==2.8.0

2
seal/settings.py
View File

@@ -39,6 +39,8 @@ INSTALLED_APPS = [
'django.contrib.staticfiles',
'system.apps.SystemConfig',
'assets.apps.AssetsConfig',
'k8s.apps.K8sConfig',
'sqlaudit.apps.SqlauditConfig',
'bootstrap4',
'django_celery_results',
'django_celery_beat',

5
sqlaudit/apps.py Normal file
View File

@@ -0,0 +1,5 @@
from django.apps import AppConfig
class SqlauditConfig(AppConfig):
name = 'sqlaudit'

154
sqlaudit/config/config.toml Normal file
View File

@@ -0,0 +1,154 @@
# IP地址
host = "0.0.0.0"
# 端口
port = 4000
# TiDB数据库目录
path = "/tmp/tidb"
[log]
# 日志级别: debug, info, warn, error, fatal.
level = "info"
# 日志格式, one of json, text, console.
format = "text"
# 禁用时间戳输出
disable-timestamp = false
# 日志文件
[log.file]
# 日志文件名
filename = ""
# 日志文件的最大上限(MB)
max-size = 300
# Max日志文件的保存天数，默认值 `0`，即不清理
max-days = 0
# 要保留的最大旧日志文件数，默认值 `0`，即不清理
max-backups = 0
# 日志轮询，默认值 `true`，即开启
log-rotate = true
[inc]
backup_host="127.0.0.1"
backup_port=3306
backup_user="root"
backup_password="123456"
enable_nullable = true
enable_drop_table = false
check_timestamp_count = true
check_table_comment = false
check_column_comment = false
# 审核列类型变更
check_column_type_change = true
# 安全更新是否开启(mysql自身的功能).
# -1 表示不做操作,基于远端数据库 [默认值]
# 0 表示关闭安全更新
# 1 表示开启安全更新
sql_safe_updates = -1
support_charset = "utf8,utf8mb4"
lang = "en-US"
# 全量日志
general_log = false
[osc]
# 用来设置在arkit返回结果集中，对于原来OSC在执行过程的标准输出信息是不是要打印到结果集对应的错误信息列中，
# 如果设置为1，就不打印，如果设置为0，就打印。而如果出现了错误，则都会打印。默认值：OFF
osc_print_none = false
# 对应参数pt-online-schema-change中的参数--print。默认值：OFF
osc_print_sql = false
# 全局的OSC开关，默认是打开的，如果想要关闭则设置为OFF，这样就会直接修改。默认值：OFF
osc_on = false
# 这个参数实际上是一个OSC开关，如果设置为0，则全部ALTER语句都使用OSC方式，
# 如果设置为非0，则当这个表占用空间大小大于这个值时才使用OSC方式。
# 单位为M，这个表大小的计算方式是通过语句
# select (DATA_LENGTH + INDEX_LENGTH)/1024/1024 from information_schema.tables
# where table_schema = 'dbname' and table_name = 'tablename' 来实现的。默认值：16
# [0-1048576]
osc_min_table_size = 16
# 对应参数pt-online-schema-change中的参数alter-foreign-keys-method，具体意义可以参考OSC官方手册。默认值：none
# [auto | none | rebuild_constraints | drop_swap]
osc_alter_foreign_keys_method = "none"
# 对应参数pt-online-schema-change中的参数recursion_method，具体意义可以参考OSC官方手册。默认值：processlist
# [processlist | hosts | none]
osc_recursion_method = "processlist"
# 对应参数pt-online-schema-change中的参数--max-lag。默认值：3
osc_max_lag = 3
# 对应参数pt-online-schema-change中的参数--[no]check-alter。默认值：ON
osc_check_alter = true
# 对应参数pt-online-schema-change中的参数--[no]check-replication-filters。默认值：ON
osc_check_replication_filters = true
# 对应参数pt-online-schema-change中的参数--[no]drop-old-table。默认值：ON
osc_drop_old_table = true
# 对应参数pt-online-schema-change中的参数--[no]drop-new-table。默认值：ON
osc_drop_new_table = true
# 对应参数pt-online-schema-change中的参数--max-load中的thread_running部分。默认值：80
osc_max_thread_running = 80
# 对应参数pt-online-schema-change中的参数--max-load中的thread_connected部分。默认值：1000
osc_max_thread_connected = 1000
# 对应参数pt-online-schema-change中的参数--critical-load中的thread_running部分。默认值：80
osc_critical_thread_running = 80
# 对应参数pt-online-schema-change中的参数--critical-load中的thread_connected部分。默认值：1000
osc_critical_thread_connected = 1000
# 对应参数pt-online-schema-change中的参数--chunk-time。默认值：1
osc_chunk_time = 1.0
# 对应参数pt-online-schema-change中的参数--chunk-size-limit。默认值：4
osc_chunk_size_limit = 4
# 对应参数pt-online-schema-change中的参数--chunk-size。默认值：1000
osc_chunk_size = 1000
# 对应参数pt-online-schema-change中的参数--check-interval，意义是Sleep time between checks for --max-lag。默认值：5
osc_check_interval = 5
osc_bin_dir = "/usr/local/bin"
[ghost]
ghost_allow_on_master = true
ghost_assume_rbr = true
ghost_chunk_size = 1000
ghost_concurrent_rowcount = true
ghost_cut_over = "atomic"
ghost_cut_over_lock_timeout_seconds = 3
ghost_default_retries = 60
ghost_heartbeat_interval_millis = 500
ghost_max_lag_millis = 1500
ghost_approve_renamed_columns = true
ghost_exponential_backoff_max_interval = 64
ghost_dml_batch_size = 10
ghost_ok_to_drop_table = true
ghost_skip_foreign_key_checks = true

81
sqlaudit/config/install.md Normal file
View File

@@ -0,0 +1,81 @@
# mysql 审核引擎 goInception 的基本使用
## 官网地址
> https://github.com/hanchuanchuan/goInception
## 安装
```
git clone https://github.com/hanchuanchuan/goInception.git
cd goInception
```
## 修改配置
* 开启备份
```
vim config/config.toml
[inc]
backup_host="127.0.0.1"
backup_port=3306
backup_user="root"
backup_password="123456"
```
## 启动
```
make parser
go build -o goInception tidb-server/main.go
./goInception -config=config/config.toml
```
> pip install pymysql prettytable
## 代码
```
import pymysql
import prettytable as pt
tb = pt.PrettyTable()
sql = '''/*--user=root;--password=123456;--host=192.168.100.90;--check=0;--port=3306;--execute=1;--backup=1;*/
inception_magic_start;
use go;
create table t1(id int primary key,c1 int,c2 int );
insert into t1(id,c1,c2) values(1,1,1);
inception_magic_commit;'''
conn = pymysql.connect(host='127.0.0.1', user='', passwd='',
db='', port=4000, charset="utf8mb4")
cur = conn.cursor()
ret = cur.execute(sql)
result = cur.fetchall()
cur.close()
conn.close()
tb.field_names = [i[0] for i in cur.description]
for row in result:
tb.add_row(row)
print(tb)
```
## 结果
```
+----------+----------+-------------+----------------------+---------------+----------------------------------------------------+---------------+------------------------+------------------------+--------------+---------+-------------+
| order_id | stage | error_level | stage_status | error_message | sql | affected_rows | sequence | backup_dbname | execute_time | sqlsha1 | backup_time |
+----------+----------+-------------+----------------------+---------------+----------------------------------------------------+---------------+------------------------+------------------------+--------------+---------+-------------+
| 1 | EXECUTED | 0 | Execute Successfully | None | use go | 0 | 1560411582_21_00000000 | None | 0.000 | None | 0 |
| 2 | EXECUTED | 0 | Execute Successfully | None | create table t1(id int primary key,c1 int,c2 int ) | 0 | 1560411582_21_00000001 | 192_168_100_90_3306_go | 0.006 | None | 0 |
| | | | Backup Successfully | | | | | | | | |
| 3 | EXECUTED | 0 | Execute Successfully | None | insert into t1(id,c1,c2) values(1,1,1) | 1 | 1560411582_21_00000002 | 192_168_100_90_3306_go | 0.002 | None | 0.004 |
| | | | Backup Successfully | | | | | | | | |
+----------+----------+-------------+----------------------+---------------+----------------------------------------------------+---------------+------------------------+------------------------+--------------+---------+-------------+
```

0
sqlaudit/migrations/__init__.py Normal file
View File

26
sqlaudit/tests.py Normal file
View File

@@ -0,0 +1,26 @@
#!/usr/bin/env python
# -*- coding:utf-8 -*-
import pymysql
import prettytable as pt
tb = pt.PrettyTable()
sql = '''/*--user=root;--password=123456;--host=192.168.100.90;--check=0;--port=3306;--execute=1;--backup=1;*/
inception_magic_start;
use go;
create table t1(id int primary key,c1 int,c2 int );
insert into t1(id,c1,c2) values(1,1,1);
inception_magic_commit;'''
conn = pymysql.connect(host='127.0.0.1', user='', passwd='',
db='', port=4000, charset="utf8mb4")
cur = conn.cursor()
ret = cur.execute(sql)
result = cur.fetchall()
cur.close()
conn.close()
tb.field_names = [i[0] for i in cur.description]
for row in result:
tb.add_row(row)
print(tb)