v0.5 SQL执行 板块

sql/admin.py

@@ -0,0 +1,31 @@
+from django.contrib import admin
+from sql.models import database
+
+
+class DatabaseAdmin(admin.ModelAdmin):
+    list_display = ('name', 'region', 'address', 'port', 'ctime', 'password', "get_password")
+
+
+class SqlPermAdmin(admin.ModelAdmin):
+
+    @classmethod
+    def show_approver(self, obj):
+        return [i.username for i in obj.approver.all()]
+
+    search_fields = ['group']
+    list_display = ('group', "show_approver", "ddl", "dml", "select")
+    filter_horizontal = ('approver', "ddl_data", "dml_data", "select_data")
+
+
+class SqlUserAdmin(admin.ModelAdmin):
+
+    @classmethod
+    def show_perm(self, obj):
+        return [i.group for i in obj.perm.all()]
+
+    list_display = ('user', 'show_perm')
+    search_fields = ['user']
+    filter_horizontal = ('perm',)
+
+
+admin.site.register(database, DatabaseAdmin)

sql/apps.py

@@ -0,0 +1,5 @@
+from django.apps import AppConfig
+
+
+class SqlConfig(AppConfig):
+    name = 'sql'

sql/bin/config/config.toml

@@ -0,0 +1,154 @@
+
+# IP地址
+host = "127.0.0.1"
+
+# 端口
+port = 4000
+
+# TiDB数据库目录
+path = "/tmp/tidb"
+
+[log]
+# 日志级别: debug, info, warn, error, fatal.
+level = "info"
+
+# 日志格式, one of json, text, console.
+format = "text"
+
+# 禁用时间戳输出
+disable-timestamp = false
+
+# 日志文件
+[log.file]
+# 日志文件名
+filename = ""
+
+# 日志文件的最大上限(MB)
+max-size = 300
+
+# Max日志文件的保存天数，默认值 `0`，即不清理
+max-days = 0
+
+# 要保留的最大旧日志文件数，默认值 `0`，即不清理
+max-backups = 0
+
+# 日志轮询，默认值 `true`，即开启
+log-rotate = true
+
+
+[inc]
+
+backup_host="192.168.100.200"
+backup_port=3308
+backup_user="root"
+backup_password="123456"
+
+enable_nullable = true
+enable_drop_table = false
+
+check_timestamp_count = true
+check_table_comment = false
+check_column_comment = false
+
+# 审核列类型变更
+check_column_type_change = true
+
+# 安全更新是否开启(mysql自身的功能).
+# -1 表示不做操作,基于远端数据库 [默认值]
+# 0  表示关闭安全更新
+# 1  表示开启安全更新
+sql_safe_updates = -1
+
+support_charset = "utf8,utf8mb4"
+
+lang = "en-US"
+
+# 全量日志
+general_log = false
+
+[osc]
+
+# 用来设置在arkit返回结果集中，对于原来OSC在执行过程的标准输出信息是不是要打印到结果集对应的错误信息列中，
+# 如果设置为1，就不打印，如果设置为0，就打印。而如果出现了错误，则都会打印。默认值：OFF
+osc_print_none = false
+
+# 对应参数pt-online-schema-change中的参数--print。默认值：OFF
+osc_print_sql = false
+
+# 全局的OSC开关，默认是打开的，如果想要关闭则设置为OFF，这样就会直接修改。默认值：OFF
+osc_on = false
+
+# 这个参数实际上是一个OSC开关，如果设置为0，则全部ALTER语句都使用OSC方式，
+# 如果设置为非0，则当这个表占用空间大小大于这个值时才使用OSC方式。
+# 单位为M，这个表大小的计算方式是通过语句
+# select (DATA_LENGTH + INDEX_LENGTH)/1024/1024 from information_schema.tables
+# where table_schema = 'dbname' and table_name = 'tablename' 来实现的。默认值：16
+# [0-1048576]
+osc_min_table_size = 16
+
+# 对应参数pt-online-schema-change中的参数alter-foreign-keys-method，具体意义可以参考OSC官方手册。默认值：none
+# [auto | none | rebuild_constraints | drop_swap]
+osc_alter_foreign_keys_method = "none"
+
+# 对应参数pt-online-schema-change中的参数recursion_method，具体意义可以参考OSC官方手册。默认值：processlist
+# [processlist | hosts | none]
+osc_recursion_method = "processlist"
+
+# 对应参数pt-online-schema-change中的参数--max-lag。默认值：3
+osc_max_lag = 3
+
+# 对应参数pt-online-schema-change中的参数--[no]check-alter。默认值：ON
+osc_check_alter = true
+
+# 对应参数pt-online-schema-change中的参数--[no]check-replication-filters。默认值：ON
+osc_check_replication_filters = true
+
+# 对应参数pt-online-schema-change中的参数--[no]drop-old-table。默认值：ON
+osc_drop_old_table = true
+
+# 对应参数pt-online-schema-change中的参数--[no]drop-new-table。默认值：ON
+osc_drop_new_table = true
+
+# 对应参数pt-online-schema-change中的参数--max-load中的thread_running部分。默认值：80
+osc_max_thread_running = 80
+
+# 对应参数pt-online-schema-change中的参数--max-load中的thread_connected部分。默认值：1000
+osc_max_thread_connected = 1000
+
+# 对应参数pt-online-schema-change中的参数--critical-load中的thread_running部分。默认值：80
+osc_critical_thread_running = 80
+
+# 对应参数pt-online-schema-change中的参数--critical-load中的thread_connected部分。默认值：1000
+osc_critical_thread_connected = 1000
+
+# 对应参数pt-online-schema-change中的参数--chunk-time。默认值：1
+osc_chunk_time = 1.0
+
+# 对应参数pt-online-schema-change中的参数--chunk-size-limit。默认值：4
+osc_chunk_size_limit = 4
+
+# 对应参数pt-online-schema-change中的参数--chunk-size。默认值：1000
+osc_chunk_size = 1000
+
+# 对应参数pt-online-schema-change中的参数--check-interval，意义是Sleep time between checks for --max-lag。默认值：5
+osc_check_interval = 5
+
+osc_bin_dir = "/usr/local/bin"
+
+
+[ghost]
+
+ghost_allow_on_master = true
+ghost_assume_rbr = true
+ghost_chunk_size = 1000
+ghost_concurrent_rowcount = true
+ghost_cut_over = "atomic"
+ghost_cut_over_lock_timeout_seconds = 3
+ghost_default_retries = 60
+ghost_heartbeat_interval_millis = 500
+ghost_max_lag_millis = 1500
+ghost_approve_renamed_columns = true
+ghost_exponential_backoff_max_interval = 64
+ghost_dml_batch_size = 10
+ghost_ok_to_drop_table = true
+ghost_skip_foreign_key_checks = true

sql/form.py

@@ -0,0 +1,71 @@
+from django import forms
+from sql.models import database
+import re
+import logging
+from django.forms import fields as Ffields
+from django.forms import widgets as Fwidgets
+
+logger = logging.getLogger('create-form')
+
+
+class DatabaseForm(forms.ModelForm):
+    data_base = Ffields.CharField(
+        label='数据库库名',
+        widget=Fwidgets.Select(attrs={'class': 'select2',
+                                      'data-placeholder': '----请选择库名----'}),
+    )
+    table = Ffields.CharField(
+        label='数据库表名',
+        widget=Fwidgets.Select(attrs={'class': 'select2',
+                                      'data-placeholder': '----请选择库名----'}),
+    )
+    # ps = Ffields.CharField(
+    #     label='提交说明',
+    #     widget=Fwidgets.TextInput(
+    #
+    #     ),
+    # )
+
+    backup = Ffields.BooleanField(
+        label='是否备份',
+    )
+
+    class Meta:
+        model = database
+        fields = ['region', 'name', 'data_base', 'table','backup']
+
+        widgets = {
+            'region': forms.Select(
+                attrs={'class': 'select2',
+                       'data-placeholder': '----请选择区域----'}
+            ),
+            'name': forms.Select(
+                attrs={'class': 'select2',
+                       'data-placeholder': '----请选择RDS----'}),
+            'data_base': forms.Select(
+                attrs={'class': 'select2',
+                       'data-placeholder': '----请选择库名----'}),
+            # 'table_name': forms.Select(
+            #     attrs={'class': 'select2',
+            #            'data-placeholder': '----请选择表名----'}),
+            # 'InstanceType': forms.Select(attrs={'class': 'select2',
+            #                                     'data-placeholder': '----请选择实例模板----'}),
+            # 'ImageId': forms.Select(attrs={'class': 'select2',
+            #                                     'data-placeholder': '----请选择镜像----'}),
+            # 'Vpc': forms.Select(
+            #     attrs={'class': 'select2',
+            #            'data-placeholder': '----请选择Vpc----'}),
+            # 'VSwitchId': forms.Select(
+            #     attrs={'class': 'select2',
+            #            'data-placeholder': '----请选择交换机----'}),
+            # 'SecurityGroupId': forms.Select(
+            #     attrs={'class': 'select2',
+            #            'data-placeholder': '----请选择安全组----'},
+            # ),
+            # 'Size': forms.Select(attrs={'class': 'select2'}),
+        }
+        help_texts = {
+            'region': '* 必填 ',
+            'name': '* 必填 ',
+            'data_base': '* 必填 ',
+        }

sql/handle/con_database.py

@@ -0,0 +1,167 @@
+'''
+
+About connection Database
+
+2017-11-23
+
+cookie
+'''
+
+import pymysql
+
+
+class SQLgo(object):
+    def __init__(self, ip=None, user=None, password=None, db=None, port=None):
+        self.ip = ip
+        self.user = user
+        self.password = password
+        self.db = db
+        self.port = int(port)
+        self.con = object
+
+    @staticmethod
+    def addDic(theIndex, word, value):
+        theIndex.setdefault(word, []).append(value)
+
+    def __enter__(self):
+        self.con = pymysql.connect(
+            host=self.ip,
+            user=self.user,
+            passwd=self.password,
+            db=self.db,
+            charset='utf8mb4',
+            port=self.port
+        )
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        self.con.close()
+
+    def search(self, sql=None):
+        data_dict = []
+        id = 0
+        with self.con.cursor(cursor=pymysql.cursors.DictCursor) as cursor:
+            sqllist = sql
+            cursor.execute(sqllist)
+            result = cursor.fetchall()
+            for field in cursor.description:
+                if id == 0:
+                    data_dict.append({'title': field[0], "key": field[0], "fixed": "left", "width": 150})
+                    id += 1
+                else:
+                    data_dict.append({'title': field[0], "key": field[0], "width": 200})
+            len = cursor.rowcount
+        return {'data': result, 'title': data_dict, 'len': len}
+
+    def showtable(self, table_name):
+        with self.con.cursor() as cursor:
+            sqllist = '''
+                    select aa.COLUMN_NAME,
+                    aa.DATA_TYPE,aa.COLUMN_COMMENT, cc.TABLE_COMMENT 
+                    from information_schema.`COLUMNS` aa LEFT JOIN 
+                    (select DISTINCT bb.TABLE_SCHEMA,bb.TABLE_NAME,bb.TABLE_COMMENT 
+                    from information_schema.`TABLES` bb ) cc  
+                    ON (aa.TABLE_SCHEMA=cc.TABLE_SCHEMA and aa.TABLE_NAME = cc.TABLE_NAME )
+                    where aa.TABLE_SCHEMA = '%s' and aa.TABLE_NAME = '%s';
+                    ''' % (self.db, table_name)
+            cursor.execute(sqllist)
+            result = cursor.fetchall()
+            td = [
+                {
+                    'Field': i[0],
+                    'Type': i[1],
+                    'Extra': i[2],
+                    'TableComment': i[3]
+                } for i in result
+            ]
+        return td
+
+    def gen_alter(self, table_name):
+        with self.con.cursor() as cursor:
+            sqllist = 'desc %s.%s;' % (self.db, table_name)
+            cursor.execute(sqllist)
+            result = cursor.fetchall()
+            td = [
+                {
+                    'Field': i[0],
+                    'Type': i[1],
+                    'Null': i[2],
+                    'Key': i[3],
+                    'Default': i[4]
+                } for i in result
+            ]
+            sqllist = 'show table status where NAME="%s";' % (table_name)
+            cursor.execute(sqllist)
+            result = cursor.fetchall()
+            tablecomment = result[0][-1]
+            [item.update(TableComment=tablecomment) for item in td]
+            sqllist = 'show full columns from %s;' % (table_name)
+            cursor.execute(sqllist)
+            result = cursor.fetchall()
+            for item in td:
+                for item1 in result:
+                    if item['Field'] == item1[0]:
+                        item['Extra'] = item1[-1]
+                        break
+        return td
+
+    def index(self, table_name):
+        with self.con.cursor() as cursor:
+            cursor.execute('show keys from %s' % table_name)
+            result = cursor.fetchall()
+            di = [
+                {
+                    'Non_unique': '是',
+                    'key_name': i[2],
+                    'column_name': i[4],
+                    'index_type': i[10]
+                }
+                if i[1] == 0
+                else
+                {
+                    'Non_unique': '否',
+                    'key_name': i[2],
+                    'column_name': i[4],
+                    'index_type': i[10]
+                }
+                for i in result
+            ]
+
+            dic = {}
+            c = []
+            for i in di:
+                self.addDic(dic, i['key_name'], i['column_name'])
+            for t in dic:
+                """
+                初始化第一个value
+                将value 数据变为字符串
+                转为字典对象数组
+                """
+                str1 = dic[t][0]
+
+                for i in range(1, len(dic[t])):
+                    str1 = str1 + ',' + dic[t][i]
+
+                temp = {}
+                for g in di:
+                    if t == g['key_name']:
+                        temp.setdefault('Non_unique', g['Non_unique'])
+                        temp.setdefault('index_type', g['index_type'])
+                temp.setdefault('column_name', str1)
+                temp.setdefault('key_name', t)
+                c.append(temp)
+            return c
+
+    def baseItems(self, sql=None):
+
+        with self.con.cursor() as cursor:
+            cursor.execute(sql)
+            result = cursor.fetchall()
+            data = [c for i in result for c in i]
+            return data
+
+    def query_info(self, sql=None):
+        with self.con.cursor(cursor=pymysql.cursors.DictCursor) as cursor:
+            cursor.execute(sql)
+            result = cursor.fetchall()
+        return result

sql/handle/perm.py

@@ -0,0 +1,56 @@
+# !/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+import django
+
+import sys
+
+import logging
+
+import os
+
+sys.path.append('/opt/argus')
+os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'argus.settings')
+django.setup()
+logger = logging.getLogger('api')
+
+
+def sql_data_perm():
+    from sql.models import sqluser
+
+    data = []
+    for i in sqluser.objects.all():
+
+        for j in i.perm.all():
+            data.append({
+                'ddl': j.ddl,
+                'dml': j.dml,
+                'select': j.select,
+                "approver": [x.username for x in j.approver.all()],
+                'ddl_data': [x.name for x in j.ddl_data.all()],
+                'dml_data': [x.name for x in j.dml_data.all()],
+                'select_data': [x.name for x in j.select_data.all()]
+            })
+    data2 = {
+        "ddl": False,
+        'dml': False,
+        "select": False,
+        "approver": [],
+        "ddl_data": [],
+        "dml_data": [],
+        "select_data": [],
+    }
+    for i in data:
+        for z in ["ddl", 'dml', 'select']:
+            if i[z]:
+                data2[z] = True
+        for j in ['approver', 'ddl_data', 'dml_data', 'select_data']:
+            for x in i[j]:
+                data2[j].append(x)
+
+    for i in ['approver', 'ddl_data', 'dml_data', 'select_data']:
+        data2[i] = list(set(data2[i]))
+    return data2
+
+
+

sql/models.py

@@ -0,0 +1,56 @@
+from django.db import models
+import re
+from cryptography.fernet import Fernet
+from system.models import Users
+from assets.models import Ecs
+
+
+# import base64
+# import os
+# print(base64.urlsafe_b64encode(os.urandom(32)))
+
+cipher_key = 'isSxtA8i5slddH9PrYEu8V5jzTeKCO5vwlu5pUT3eEc='
+
+
+class database(models.Model):
+
+    region = models.CharField(max_length=16, verbose_name='机房',choices=Ecs.TYPE_CHOICES)
+    name = models.CharField(max_length=64, verbose_name='RDS名称',unique=True)
+    address = models.CharField(max_length=64, verbose_name='地址', )
+    port = models.IntegerField(verbose_name='端口')
+    username = models.CharField(max_length=128, verbose_name='用户名', blank=True, null=True, )
+    password = models.CharField(max_length=128, verbose_name='密码', blank=True, null=True, )
+
+    ctime = models.DateTimeField(auto_now_add=True, null=True, verbose_name='创建时间', blank=True)
+    utime = models.DateTimeField(auto_now=True, null=True, verbose_name='更新时间', blank=True)
+
+    def get_password(self):
+        if self.password is not None and re.search("gAAAAA", self.password, ) != None:
+            f = Fernet(cipher_key)
+            p1 = self.password.encode()
+            token = f.decrypt(p1)
+            p2 = token.decode()
+            return p2
+        return None
+
+    def save(self, *args, **kwargs):
+        if self.password is not None and re.search("gAAAAA", self.password, ) == None:
+            f = Fernet(cipher_key)
+            p1 = self.password.encode()
+            token = f.encrypt(p1)
+            p2 = token.decode()
+            self.password = p2
+        super().save(*args, **kwargs)
+
+    class Meta:
+        db_table = "database"
+        verbose_name = "数据库"
+        verbose_name_plural = verbose_name
+
+    def __str__(self):
+        return self.name
+
+
+
+
+

sql/urls.py

@@ -0,0 +1,9 @@
+from django.urls import path
+from sql import views
+
+app_name = "sql"
+
+urlpatterns = [
+    path('sql.html', views.SqlDdl.as_view(), name='sql_ddl'),
+    path('sql-<str:pk>.html', views.SqlDdlQuery.as_view(), name='sql_query'),
+]

sql/views.py

@@ -0,0 +1,206 @@
+import logging
+from django.contrib.auth.mixins import LoginRequiredMixin, PermissionRequiredMixin
+from django.views.generic import ListView, View, CreateView, UpdateView, DetailView
+from django.shortcuts import render, HttpResponse
+from sql.form import DatabaseForm
+from sql.models import database
+import json
+from django.db.models import Q
+import subprocess
+from sql.handle import con_database
+import pymysql
+import prettytable as pt
+tb = pt.PrettyTable()
+logger = logging.getLogger('sql')
+
+
+class SqlDdl(LoginRequiredMixin, CreateView):
+    model = database
+    form_class = DatabaseForm
+    template_name = 'sql/sql-ddl.html'
+
+
+class SqlDdlQuery(LoginRequiredMixin, View):
+    model = database
+
+    def get(self, request, pk):
+        region = request.GET.get("region")
+        name = request.GET.get("name")
+        data_base = request.GET.get("data_base")
+        table = request.GET.get("table")
+        ret = {"data": []}
+        if pk == "name":
+            obj = database.objects.filter(region=region)
+            for i in obj:
+                ret['data'].append(i.name)
+
+        elif pk == "databases":
+            obj = database.objects.get(name=name)
+            try:
+                with con_database.SQLgo(
+                        ip=obj.address,
+                        user=obj.username,
+                        password=obj.get_password(),
+                        port=obj.port
+                ) as f:
+                    ret['data'] = f.baseItems(sql='show databases')
+            except Exception as e:
+                print(e)
+        elif pk == "tables":
+            obj = database.objects.get(name=name)
+            try:
+                with con_database.SQLgo(
+                        ip=obj.address,
+                        user=obj.username,
+                        password=obj.get_password(),
+                        port=obj.port,
+                        db=data_base
+                ) as f:
+                    ret['data'] = f.baseItems(sql='show tables')
+            except Exception as e:
+                print(e)
+        elif pk == "structure":
+            obj = database.objects.get(name=name)
+            try:
+                with con_database.SQLgo(
+                        ip=obj.address,
+                        user=obj.username,
+                        password=obj.get_password(),
+                        port=obj.port,
+                        db=data_base
+                ) as f:
+                    field = f.gen_alter(table_name=table)  # 表结构详情
+                    idx = f.index(table_name=table)  # 索引
+
+                    ret['data'] = {'idx': idx, 'field': field}
+            except Exception as e:
+                print(e)
+        else:
+            pass
+        print(ret)
+        return HttpResponse(json.dumps(ret))
+
+    def post(self, request, pk):
+        ret = {"data": []}
+        name = request.GET.get("name")
+        backup = request.GET.get("backup")
+        # 语法检查
+        if pk == "advice":
+            sql = request.POST.get("sql")
+            sql = sql.replace("\"", "'")
+
+            sql_cmd = f'echo "{sql}" | ./sql/bin/soar '
+            print(sql_cmd)
+            cmd = subprocess.Popen(sql_cmd, shell=True, stdout=subprocess.PIPE)
+            cmd = cmd.communicate()
+            cmd = cmd[0].decode().rstrip()
+            ret['data'] = cmd
+        # SQL检测
+        elif pk == "sql_test":
+            sql_post = request.POST.get("sql")
+            sql_post = sql_post.replace("\"", "'")
+            obj = database.objects.get(name=name)
+            sql = f'''/*--user={obj.username};--password={obj.get_password()};--host={obj.address};--check=1;--port={obj.port};--execute=0;--backup={backup};*/
+            inception_magic_start;
+            {sql_post}
+            inception_magic_commit;'''
+
+            conn = pymysql.connect(host='127.0.0.1', user='', passwd='',
+                                   db='', port=4000, charset="utf8mb4")
+            cur = conn.cursor()
+            cur.execute(sql)
+            result = cur.fetchall()
+            cur.close()
+            conn.close()
+
+            tb.field_names = [i[0] for i in cur.description]
+            for row in result:
+                tb.add_row(row)
+            print(tb)
+
+            for i in result:
+                ret['data'].append({
+                    "order_id":i[0],
+                    "stage": i[1],
+                    "error_level": i[2],
+                    "stage_status": i[3],
+                    "error_message": i[4],
+                    "sql": i[5],
+                    "affected_rows": i[6],
+                })
+            ret['error'] = 0
+            for j in ret['data']:
+                if j['error_level'] != 0:
+                    ret['error'] = 1
+
+        # sql执行
+        elif pk == "sql_exe":
+            sql_post = request.POST.get("sql")
+            sql_post = sql_post.replace("\"", "'")
+            obj = database.objects.get(name=name)
+            # 先检查
+            sql1 = f'''/*--user={obj.username};--password={obj.get_password()};--host={obj.address};--check=1;--port={obj.port};--execute=0;--backup={backup};*/
+                  inception_magic_start;
+                  {sql_post}
+                  inception_magic_commit;'''
+            conn = pymysql.connect(host='127.0.0.1', user='', passwd='',
+                                   db='', port=4000, charset="utf8mb4")
+            cur = conn.cursor()
+            cur.execute(sql1)
+            result = cur.fetchall()
+            cur.close()
+            conn.close()
+
+            tb.field_names = [i[0] for i in cur.description]
+            for row in result:
+                tb.add_row(row)
+            print(tb)
+
+            for i in result:
+                ret['data'].append({
+                    "order_id": i[0],
+                    "stage": i[1],
+                    "error_level": i[2],
+                    "stage_status": i[3],
+                    "error_message": i[4],
+                    "sql": i[5],
+                    "affected_rows": i[6],
+                })
+
+            for j in ret['data']:
+                if j['error_level'] != 0:
+                    ret['error'] = "检查未通过，禁止执行"
+                    return HttpResponse(json.dumps(ret))
+            # 检查通过 执行
+            sql2 = f'''/*--user={obj.username};--password={obj.get_password()};--host={obj.address};--check=0;--port={obj.port};--execute=1;--backup={backup};*/
+                              inception_magic_start;
+                              {sql_post}
+                              inception_magic_commit;'''
+
+            conn = pymysql.connect(host='127.0.0.1', user='', passwd='',
+                                   db='', port=4000, charset="utf8mb4")
+            cur = conn.cursor()
+            cur.execute(sql2)
+            result = cur.fetchall()
+            cur.close()
+            conn.close()
+
+            tb.field_names = [i[0] for i in cur.description]
+            for row in result:
+                tb.add_row(row)
+            print(tb)
+            ret = {"data": []}
+            for i in result:
+                ret['data'].append({
+                    "order_id":i[0],
+                    "stage": i[1],
+                    "error_level": i[2],
+                    "stage_status": i[3],
+                    "error_message": i[4],
+                    "sql": i[5],
+                    "affected_rows": i[6],
+                })
+        else:
+            pass
+        print(ret)
+        return HttpResponse(json.dumps(ret))