diff --git a/README.md b/README.md index 1145973..de0cd9b 100644 --- a/README.md +++ b/README.md @@ -36,8 +36,10 @@ * 二期: k8s管理平台 (开发中) * node/service/pod 列表 (已完成) * pod webssh (已完成) - - +* 三期: mysql sql语句审核(开发中) + * 引擎 goInception + + ## DEMO > http://129.28.156.219:8001 diff --git a/assets/tests.py b/assets/tests.py index 7ce503c..e69de29 100644 --- a/assets/tests.py +++ b/assets/tests.py @@ -1,3 +0,0 @@ -from django.test import TestCase - -# Create your tests here. diff --git a/k8s/apps.py b/k8s/apps.py new file mode 100644 index 0000000..76d60eb --- /dev/null +++ b/k8s/apps.py @@ -0,0 +1,5 @@ +from django.apps import AppConfig + + +class K8sConfig(AppConfig): + name = 'k8s' diff --git a/requirements.txt b/requirements.txt index d5eb9f5..67bab76 100644 --- a/requirements.txt +++ b/requirements.txt @@ -68,6 +68,7 @@ parso==0.2.1 pathtools==0.1.2 pexpect==4.6.0 pickleshare==0.7.4 +prettytable==0.7.2 prometheus-client==0.2.0 promise==2.2.1 prompt-toolkit==1.0.15 @@ -77,6 +78,7 @@ pyasn1-modules==0.2.5 pycparser==2.19 Pygments==2.2.0 PyHamcrest==1.9.0 +PyMySQL==0.9.3 PyNaCl==1.3.0 python-crontab==2.3.6 python-dateutil==2.8.0 diff --git a/seal/settings.py b/seal/settings.py index d31c358..564cfcb 100644 --- a/seal/settings.py +++ b/seal/settings.py @@ -39,6 +39,8 @@ INSTALLED_APPS = [ 'django.contrib.staticfiles', 'system.apps.SystemConfig', 'assets.apps.AssetsConfig', + 'k8s.apps.K8sConfig', + 'sqlaudit.apps.SqlauditConfig', 'bootstrap4', 'django_celery_results', 'django_celery_beat', diff --git a/sqlaudit/apps.py b/sqlaudit/apps.py new file mode 100644 index 0000000..9090f14 --- /dev/null +++ b/sqlaudit/apps.py @@ -0,0 +1,5 @@ +from django.apps import AppConfig + + +class SqlauditConfig(AppConfig): + name = 'sqlaudit' diff --git a/sqlaudit/config/config.toml b/sqlaudit/config/config.toml new file mode 100644 index 0000000..a9bbed4 --- /dev/null +++ b/sqlaudit/config/config.toml @@ -0,0 +1,154 @@ + +# IP地址 +host = "0.0.0.0" + +# 端口 +port = 4000 + +# TiDB数据库目录 +path = "/tmp/tidb" + +[log] +# 日志级别: debug, info, warn, error, fatal. +level = "info" + +# 日志格式, one of json, text, console. +format = "text" + +# 禁用时间戳输出 +disable-timestamp = false + +# 日志文件 +[log.file] +# 日志文件名 +filename = "" + +# 日志文件的最大上限(MB) +max-size = 300 + +# Max日志文件的保存天数,默认值 `0`,即不清理 +max-days = 0 + +# 要保留的最大旧日志文件数,默认值 `0`,即不清理 +max-backups = 0 + +# 日志轮询,默认值 `true`,即开启 +log-rotate = true + +[inc] + +backup_host="127.0.0.1" +backup_port=3306 +backup_user="root" +backup_password="123456" + + +enable_nullable = true +enable_drop_table = false + +check_timestamp_count = true +check_table_comment = false +check_column_comment = false + +# 审核列类型变更 +check_column_type_change = true + +# 安全更新是否开启(mysql自身的功能). +# -1 表示不做操作,基于远端数据库 [默认值] +# 0 表示关闭安全更新 +# 1 表示开启安全更新 +sql_safe_updates = -1 + +support_charset = "utf8,utf8mb4" + +lang = "en-US" + +# 全量日志 +general_log = false + +[osc] + +# 用来设置在arkit返回结果集中,对于原来OSC在执行过程的标准输出信息是不是要打印到结果集对应的错误信息列中, +# 如果设置为1,就不打印,如果设置为0,就打印。而如果出现了错误,则都会打印。默认值:OFF +osc_print_none = false + +# 对应参数pt-online-schema-change中的参数--print。默认值:OFF +osc_print_sql = false + +# 全局的OSC开关,默认是打开的,如果想要关闭则设置为OFF,这样就会直接修改。默认值:OFF +osc_on = false + +# 这个参数实际上是一个OSC开关,如果设置为0,则全部ALTER语句都使用OSC方式, +# 如果设置为非0,则当这个表占用空间大小大于这个值时才使用OSC方式。 +# 单位为M,这个表大小的计算方式是通过语句 +# select (DATA_LENGTH + INDEX_LENGTH)/1024/1024 from information_schema.tables +# where table_schema = 'dbname' and table_name = 'tablename' 来实现的。默认值:16 +# [0-1048576] +osc_min_table_size = 16 + +# 对应参数pt-online-schema-change中的参数alter-foreign-keys-method,具体意义可以参考OSC官方手册。默认值:none +# [auto | none | rebuild_constraints | drop_swap] +osc_alter_foreign_keys_method = "none" + +# 对应参数pt-online-schema-change中的参数recursion_method,具体意义可以参考OSC官方手册。默认值:processlist +# [processlist | hosts | none] +osc_recursion_method = "processlist" + +# 对应参数pt-online-schema-change中的参数--max-lag。默认值:3 +osc_max_lag = 3 + +# 对应参数pt-online-schema-change中的参数--[no]check-alter。默认值:ON +osc_check_alter = true + +# 对应参数pt-online-schema-change中的参数--[no]check-replication-filters。默认值:ON +osc_check_replication_filters = true + +# 对应参数pt-online-schema-change中的参数--[no]drop-old-table。默认值:ON +osc_drop_old_table = true + +# 对应参数pt-online-schema-change中的参数--[no]drop-new-table。默认值:ON +osc_drop_new_table = true + +# 对应参数pt-online-schema-change中的参数--max-load中的thread_running部分。默认值:80 +osc_max_thread_running = 80 + +# 对应参数pt-online-schema-change中的参数--max-load中的thread_connected部分。默认值:1000 +osc_max_thread_connected = 1000 + +# 对应参数pt-online-schema-change中的参数--critical-load中的thread_running部分。默认值:80 +osc_critical_thread_running = 80 + +# 对应参数pt-online-schema-change中的参数--critical-load中的thread_connected部分。默认值:1000 +osc_critical_thread_connected = 1000 + +# 对应参数pt-online-schema-change中的参数--chunk-time。默认值:1 +osc_chunk_time = 1.0 + +# 对应参数pt-online-schema-change中的参数--chunk-size-limit。默认值:4 +osc_chunk_size_limit = 4 + +# 对应参数pt-online-schema-change中的参数--chunk-size。默认值:1000 +osc_chunk_size = 1000 + +# 对应参数pt-online-schema-change中的参数--check-interval,意义是Sleep time between checks for --max-lag。默认值:5 +osc_check_interval = 5 + +osc_bin_dir = "/usr/local/bin" + + +[ghost] + +ghost_allow_on_master = true +ghost_assume_rbr = true +ghost_chunk_size = 1000 +ghost_concurrent_rowcount = true +ghost_cut_over = "atomic" +ghost_cut_over_lock_timeout_seconds = 3 +ghost_default_retries = 60 +ghost_heartbeat_interval_millis = 500 +ghost_max_lag_millis = 1500 +ghost_approve_renamed_columns = true +ghost_exponential_backoff_max_interval = 64 +ghost_dml_batch_size = 10 +ghost_ok_to_drop_table = true +ghost_skip_foreign_key_checks = true diff --git a/sqlaudit/config/install.md b/sqlaudit/config/install.md new file mode 100644 index 0000000..d3f2dbb --- /dev/null +++ b/sqlaudit/config/install.md @@ -0,0 +1,81 @@ +# mysql 审核引擎 goInception 的基本使用 +## 官网地址 +> https://github.com/hanchuanchuan/goInception + +## 安装 +``` +git clone https://github.com/hanchuanchuan/goInception.git +cd goInception + +``` + +## 修改配置 +* 开启备份 + + +``` +vim config/config.toml + +[inc] + +backup_host="127.0.0.1" +backup_port=3306 +backup_user="root" +backup_password="123456" + +``` + + +## 启动 + +``` +make parser +go build -o goInception tidb-server/main.go + +./goInception -config=config/config.toml +``` + +> pip install pymysql prettytable + + +## 代码 +``` +import pymysql +import prettytable as pt +tb = pt.PrettyTable() + +sql = '''/*--user=root;--password=123456;--host=192.168.100.90;--check=0;--port=3306;--execute=1;--backup=1;*/ +inception_magic_start; +use go; +create table t1(id int primary key,c1 int,c2 int ); +insert into t1(id,c1,c2) values(1,1,1); +inception_magic_commit;''' + +conn = pymysql.connect(host='127.0.0.1', user='', passwd='', + db='', port=4000, charset="utf8mb4") +cur = conn.cursor() +ret = cur.execute(sql) +result = cur.fetchall() +cur.close() +conn.close() + +tb.field_names = [i[0] for i in cur.description] +for row in result: + tb.add_row(row) +print(tb) +``` + +## 结果 + + +``` ++----------+----------+-------------+----------------------+---------------+----------------------------------------------------+---------------+------------------------+------------------------+--------------+---------+-------------+ +| order_id | stage | error_level | stage_status | error_message | sql | affected_rows | sequence | backup_dbname | execute_time | sqlsha1 | backup_time | ++----------+----------+-------------+----------------------+---------------+----------------------------------------------------+---------------+------------------------+------------------------+--------------+---------+-------------+ +| 1 | EXECUTED | 0 | Execute Successfully | None | use go | 0 | 1560411582_21_00000000 | None | 0.000 | None | 0 | +| 2 | EXECUTED | 0 | Execute Successfully | None | create table t1(id int primary key,c1 int,c2 int ) | 0 | 1560411582_21_00000001 | 192_168_100_90_3306_go | 0.006 | None | 0 | +| | | | Backup Successfully | | | | | | | | | +| 3 | EXECUTED | 0 | Execute Successfully | None | insert into t1(id,c1,c2) values(1,1,1) | 1 | 1560411582_21_00000002 | 192_168_100_90_3306_go | 0.002 | None | 0.004 | +| | | | Backup Successfully | | | | | | | | | ++----------+----------+-------------+----------------------+---------------+----------------------------------------------------+---------------+------------------------+------------------------+--------------+---------+-------------+ +``` \ No newline at end of file diff --git a/sqlaudit/migrations/__init__.py b/sqlaudit/migrations/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/sqlaudit/tests.py b/sqlaudit/tests.py new file mode 100644 index 0000000..fab4294 --- /dev/null +++ b/sqlaudit/tests.py @@ -0,0 +1,26 @@ +#!/usr/bin/env python +# -*- coding:utf-8 -*- + +import pymysql +import prettytable as pt +tb = pt.PrettyTable() + +sql = '''/*--user=root;--password=123456;--host=192.168.100.90;--check=0;--port=3306;--execute=1;--backup=1;*/ +inception_magic_start; +use go; +create table t1(id int primary key,c1 int,c2 int ); +insert into t1(id,c1,c2) values(1,1,1); +inception_magic_commit;''' + +conn = pymysql.connect(host='127.0.0.1', user='', passwd='', + db='', port=4000, charset="utf8mb4") +cur = conn.cursor() +ret = cur.execute(sql) +result = cur.fetchall() +cur.close() +conn.close() + +tb.field_names = [i[0] for i in cur.description] +for row in result: + tb.add_row(row) +print(tb) \ No newline at end of file