feature:first commit

.idea/.gitignore

@@ -0,0 +1,8 @@
+# 默认忽略的文件
+/shelf/
+/workspace.xml
+# 数据源本地存储已忽略文件
+/../../../../../:\charles\coding\bamboo_engine_playground\.idea/dataSources/
+/dataSources.local.xml
+# 基于编辑器的 HTTP 客户端请求
+/httpRequests/

.idea/bamboo_engine_playground.iml

@@ -0,0 +1,30 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<module type="PYTHON_MODULE" version="4">
+  <component name="FacetManager">
+    <facet type="django" name="Django">
+      <configuration>
+        <option name="rootFolder" value="$MODULE_DIR$" />
+        <option name="settingsModule" value="settings.py" />
+        <option name="manageScript" value="manage.py" />
+        <option name="environment" value="&lt;map/&gt;" />
+        <option name="doNotUseTestRunner" value="false" />
+        <option name="trackFilePattern" value="migrations" />
+      </configuration>
+    </facet>
+  </component>
+  <component name="NewModuleRootManager">
+    <content url="file://$MODULE_DIR$">
+      <excludeFolder url="file://$MODULE_DIR$/web/node_modules" />
+    </content>
+    <content url="file://$MODULE_DIR$/web/node_modules" />
+    <orderEntry type="jdk" jdkName="Python 3.6 (dj-flow)" jdkType="Python SDK" />
+    <orderEntry type="sourceFolder" forTests="false" />
+  </component>
+  <component name="PyDocumentationSettings">
+    <option name="format" value="PLAIN" />
+    <option name="myDocStringFormat" value="Plain" />
+  </component>
+  <component name="TemplatesService">
+    <option name="TEMPLATE_CONFIGURATION" value="Django" />
+  </component>
+</module>

.idea/dataSources.xml

@@ -0,0 +1,15 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="DataSourceManagerImpl" format="xml" multifile-model="true">
+    <data-source source="LOCAL" name="bomboo@localhost" uuid="19ee76cb-3424-4654-855f-f68454aff13a">
+      <driver-ref>mysql.8</driver-ref>
+      <synchronize>true</synchronize>
+      <jdbc-driver>com.mysql.cj.jdbc.Driver</jdbc-driver>
+      <jdbc-url>jdbc:mysql://localhost:3306/bomboo</jdbc-url>
+      <working-dir>$ProjectFileDir$</working-dir>
+      <driver-properties>
+        <property name="serverTimezone" value="UTC" />
+      </driver-properties>
+    </data-source>
+  </component>
+</project>

.idea/dataSources/19ee76cb-3424-4654-855f-f68454aff13a/storage_v2/_src_/schema/information_schema.FNRwLQ.meta

@@ -0,0 +1,2 @@
+#n:information_schema
+!<md> [null, 0, null, null, -2147483648, -2147483648]

.idea/inspectionProfiles/Project_Default.xml

@@ -0,0 +1,57 @@
+<component name="InspectionProjectProfileManager">
+  <profile version="1.0">
+    <option name="myName" value="Project Default" />
+    <inspection_tool class="Eslint" enabled="true" level="WARNING" enabled_by_default="true" />
+    <inspection_tool class="JSHint" enabled="true" level="ERROR" enabled_by_default="true" />
+    <inspection_tool class="PyCompatibilityInspection" enabled="true" level="WARNING" enabled_by_default="true">
+      <option name="ourVersions">
+        <value>
+          <list size="1">
+            <item index="0" class="java.lang.String" itemvalue="3.10" />
+          </list>
+        </value>
+      </option>
+    </inspection_tool>
+    <inspection_tool class="PyPackageRequirementsInspection" enabled="true" level="WARNING" enabled_by_default="true">
+      <option name="ignoredPackages">
+        <value>
+          <list size="26">
+            <item index="0" class="java.lang.String" itemvalue="cryptography" />
+            <item index="1" class="java.lang.String" itemvalue="gevent" />
+            <item index="2" class="java.lang.String" itemvalue="python-ldap" />
+            <item index="3" class="java.lang.String" itemvalue="jieba" />
+            <item index="4" class="java.lang.String" itemvalue="six" />
+            <item index="5" class="java.lang.String" itemvalue="joblib" />
+            <item index="6" class="java.lang.String" itemvalue="threadpoolctl" />
+            <item index="7" class="java.lang.String" itemvalue="scikit-learn" />
+            <item index="8" class="java.lang.String" itemvalue="python-dateutil" />
+            <item index="9" class="java.lang.String" itemvalue="kiwisolver" />
+            <item index="10" class="java.lang.String" itemvalue="wcwidth" />
+            <item index="11" class="java.lang.String" itemvalue="cycler" />
+            <item index="12" class="java.lang.String" itemvalue="MarkupSafe" />
+            <item index="13" class="java.lang.String" itemvalue="click" />
+            <item index="14" class="java.lang.String" itemvalue="Jinja2" />
+            <item index="15" class="java.lang.String" itemvalue="simplejson" />
+            <item index="16" class="java.lang.String" itemvalue="prettytable" />
+            <item index="17" class="java.lang.String" itemvalue="pyecharts" />
+            <item index="18" class="java.lang.String" itemvalue="PyMySQL" />
+            <item index="19" class="java.lang.String" itemvalue="pytz" />
+            <item index="20" class="java.lang.String" itemvalue="itsdangerous" />
+            <item index="21" class="java.lang.String" itemvalue="pyparsing" />
+            <item index="22" class="java.lang.String" itemvalue="Flask" />
+            <item index="23" class="java.lang.String" itemvalue="Pillow" />
+            <item index="24" class="java.lang.String" itemvalue="future" />
+            <item index="25" class="java.lang.String" itemvalue="uWSGI" />
+          </list>
+        </value>
+      </option>
+    </inspection_tool>
+    <inspection_tool class="PyUnresolvedReferencesInspection" enabled="true" level="WARNING" enabled_by_default="true">
+      <option name="ignoredIdentifiers">
+        <list>
+          <option value="type.with_ui" />
+        </list>
+      </option>
+    </inspection_tool>
+  </profile>
+</component>

.idea/inspectionProfiles/profiles_settings.xml

@@ -0,0 +1,6 @@
+<component name="InspectionProjectProfileManager">
+  <settings>
+    <option name="USE_PROJECT_PROFILE" value="false" />
+    <version value="1.0" />
+  </settings>
+</component>

.idea/misc.xml

@@ -0,0 +1,4 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="ProjectRootManager" version="2" project-jdk-name="Python 3.6 (dj-flow)" project-jdk-type="Python SDK" />
+</project>

.idea/modules.xml

@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="ProjectModuleManager">
+    <modules>
+      <module fileurl="file://$PROJECT_DIR$/.idea/bamboo_engine_playground.iml" filepath="$PROJECT_DIR$/.idea/bamboo_engine_playground.iml" />
+    </modules>
+  </component>
+</project>

applications/flow/admin.py

@@ -0,0 +1,3 @@
+from django.contrib import admin
+
+# Register your models here.

applications/flow/apps.py

@@ -0,0 +1,6 @@
+from django.apps import AppConfig
+
+
+class FlowConfig(AppConfig):
+    default_auto_field = 'django.db.models.BigAutoField'
+    name = 'applications.flow'

applications/flow/constants.py

@@ -0,0 +1,14 @@
+FAIL_OFFSET_UNIT_CHOICE = (
+    ("seconds", "秒"),
+    ("hours", "时"),
+    ("minutes", "分"),
+
+)
+node_type = (
+    (0, "开始节点"),
+    (1, "结束节点"),
+    (2, "作业节点"),
+    (3, "子流程"),
+    (4, "条件分支"),
+    (5, "汇聚网关"),
+)

applications/flow/migrations/0001_initial.py

@@ -0,0 +1,65 @@
+# Generated by Django 2.2.6 on 2022-01-28 07:05
+
+import datetime
+from django.db import migrations, models
+import django.db.models.deletion
+import django_mysql.models
+
+
+class Migration(migrations.Migration):
+
+    initial = True
+
+    dependencies = [
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='Category',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('name', models.CharField(max_length=255, verbose_name='分类名称')),
+            ],
+        ),
+        migrations.CreateModel(
+            name='Process',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('name', models.CharField(max_length=255, verbose_name='作业名称')),
+                ('description', models.CharField(blank=True, max_length=255, null=True, verbose_name='作业描述')),
+                ('run_type', models.CharField(max_length=32, verbose_name='调度类型')),
+                ('total_run_count', models.PositiveIntegerField(default=0, verbose_name='执行次数')),
+                ('gateways', django_mysql.models.JSONField(default=dict, verbose_name='网关信息')),
+                ('constants', django_mysql.models.JSONField(default=dict, verbose_name='内部变量信息')),
+                ('dag', django_mysql.models.JSONField(default=dict, verbose_name='DAG')),
+                ('create_by', models.CharField(max_length=64, null=True, verbose_name='创建者')),
+                ('create_time', models.DateTimeField(default=datetime.datetime.now, verbose_name='创建时间')),
+                ('update_time', models.DateTimeField(auto_now=True, verbose_name='修改时间')),
+                ('update_by', models.CharField(max_length=64, null=True, verbose_name='修改人')),
+                ('category', models.ManyToManyField(to='flow.Category')),
+            ],
+        ),
+        migrations.CreateModel(
+            name='Node',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('name', models.CharField(max_length=255, verbose_name='节点名称')),
+                ('uuid', models.CharField(max_length=255, verbose_name='UUID')),
+                ('description', models.CharField(blank=True, max_length=255, null=True, verbose_name='节点描述')),
+                ('show', models.BooleanField(default=True, verbose_name='是否显示')),
+                ('top', models.IntegerField(default=300)),
+                ('left', models.IntegerField(default=300)),
+                ('ico', models.CharField(blank=True, max_length=64, null=True, verbose_name='icon')),
+                ('fail_retry_count', models.IntegerField(default=0, verbose_name='失败重试次数')),
+                ('fail_offset', models.IntegerField(default=0, verbose_name='失败重试间隔')),
+                ('fail_offset_unit', models.CharField(choices=[('seconds', '秒'), ('hours', '时'), ('minutes', '分')], max_length=32, verbose_name='重试间隔单位')),
+                ('node_type', models.IntegerField(default=2)),
+                ('component_code', models.CharField(max_length=255, verbose_name='插件名称')),
+                ('is_skip_fail', models.BooleanField(default=False, verbose_name='忽略失败')),
+                ('is_timeout_alarm', models.BooleanField(default=False, verbose_name='超时告警')),
+                ('inputs', django_mysql.models.JSONField(default=dict, verbose_name='输入参数')),
+                ('outputs', django_mysql.models.JSONField(default=dict, verbose_name='输出参数')),
+                ('process', models.ForeignKey(db_constraint=False, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='nodes', to='flow.Process')),
+            ],
+        ),
+    ]

applications/flow/migrations/0002_auto_20220128_1556.py

@@ -0,0 +1,18 @@
+# Generated by Django 2.2.6 on 2022-01-28 07:56
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('flow', '0001_initial'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='node',
+            name='uuid',
+            field=models.CharField(max_length=255, unique=True, verbose_name='UUID'),
+        ),
+    ]

applications/flow/models.py

@@ -0,0 +1,52 @@
+from datetime import datetime
+
+from django.db import models
+from django.forms import BooleanField
+from django_mysql.models import JSONField
+
+from applications.flow.constants import FAIL_OFFSET_UNIT_CHOICE
+
+
+class Category(models.Model):
+    name = models.CharField("分类名称", max_length=255, blank=False, null=False)
+
+
+class Process(models.Model):
+    name = models.CharField("作业名称", max_length=255, blank=False, null=False)
+    description = models.CharField("作业描述", max_length=255, blank=True, null=True)
+    category = models.ManyToManyField(Category)
+    run_type = models.CharField("调度类型", max_length=32)
+    total_run_count = models.PositiveIntegerField("执行次数", default=0)
+    gateways = JSONField("网关信息", default=dict)
+    constants = JSONField("内部变量信息", default=dict)
+    dag = JSONField("DAG", default=dict)
+
+    create_by = models.CharField("创建者", max_length=64, null=True)
+    create_time = models.DateTimeField("创建时间", default=datetime.now)
+    update_time = models.DateTimeField("修改时间", auto_now=True)
+    update_by = models.CharField("修改人", max_length=64, null=True)
+
+
+class Node(models.Model):
+    process = models.ForeignKey(Process, on_delete=models.SET_NULL, null=True, db_constraint=False,
+                                related_name="nodes")
+    name = models.CharField("节点名称", max_length=255, blank=False, null=False)
+    uuid = models.CharField("UUID", max_length=255, unique=True)
+    description = models.CharField("节点描述", max_length=255, blank=True, null=True)
+
+    show = models.BooleanField("是否显示", default=True)
+    top = models.IntegerField(default=300)
+    left = models.IntegerField(default=300)
+    ico = models.CharField("icon", max_length=64, blank=True, null=True)
+
+    fail_retry_count = models.IntegerField("失败重试次数", default=0)
+    fail_offset = models.IntegerField("失败重试间隔", default=0)
+    fail_offset_unit = models.CharField("重试间隔单位", choices=FAIL_OFFSET_UNIT_CHOICE, max_length=32)
+    # 0：开始节点，1：结束节点，2：作业节点，3：其他作业流4：分支，5：汇聚
+    node_type = models.IntegerField(default=2)
+    component_code = models.CharField("插件名称", max_length=255, blank=False, null=False)
+    is_skip_fail = models.BooleanField("忽略失败", default=False)
+    is_timeout_alarm = models.BooleanField("超时告警", default=False)
+
+    inputs = JSONField("输入参数", default=dict)
+    outputs = JSONField("输出参数", default=dict)

applications/flow/serializers.py

@@ -0,0 +1,100 @@
+from django.db import transaction
+from rest_framework import serializers
+
+from applications.flow.models import Process, Node
+
+
+class ProcessViewSetsSerializer(serializers.Serializer):
+    name = serializers.CharField(required=True)
+    description = serializers.CharField(required=False, allow_blank=True)
+    category = serializers.ListField(default="null")
+    run_type = serializers.CharField(default="null")
+    pipeline_tree = serializers.JSONField(required=True)
+
+    def save(self, **kwargs):
+        node_map = {}
+        for node in self.validated_data["pipeline_tree"]["nodes"]:
+            node_map[node["uuid"]] = node
+        dag = {k: [] for k in node_map.keys()}
+        for line in self.validated_data["pipeline_tree"]["lines"]:
+            dag[line["from"]].append(line["to"])
+        with transaction.atomic():
+            process = Process.objects.create(name=self.validated_data["name"],
+                                             description=self.validated_data["description"],
+                                             run_type=self.validated_data["run_type"],
+                                             dag=dag)
+            bulk_nodes = []
+            for node in node_map.values():
+                node_data = node["node_data"]
+                bulk_nodes.append(Node(process=process,
+                                       name=node_data["node_name"],
+                                       uuid=node["uuid"],
+                                       description=node_data["description"],
+                                       fail_retry_count=node_data.get("fail_retry_count", 0) or 0,
+                                       fail_offset=node_data.get("fail_offset", 0) or 0,
+                                       fail_offset_unit=node_data.get("fail_offset_unit", "seconds"),
+                                       node_type=node.get("type", 3),
+                                       is_skip_fail=node_data["is_skip_fail"],
+                                       is_timeout_alarm=node_data["is_skip_fail"],
+                                       inputs=node_data["inputs"],
+                                       show=node["show"],
+                                       top=node["top"],
+                                       left=node["left"],
+                                       ico=node["ico"],
+                                       outputs={},
+                                       component_code="http_request"
+                                       ))
+            Node.objects.bulk_create(bulk_nodes, batch_size=500)
+
+
+class ListProcessViewSetsSerializer(serializers.ModelSerializer):
+    class Meta:
+        model = Process
+        fields = "__all__"
+
+
+class RetrieveProcessViewSetsSerializer(serializers.ModelSerializer):
+    pipeline_tree = serializers.SerializerMethodField()
+
+    # category = serializers.SerializerMethodField()
+    #
+    # def get_category(self, obj):
+    #     return obj.category.all()
+
+    def get_pipeline_tree(self, obj):
+        lines = []
+        nodes = []
+        for _from, to_list in obj.dag.items():
+            for _to in to_list:
+                lines.append({
+                    "from": _from,
+                    "to": _to
+                })
+        node_list = Node.objects.filter(process_id=obj.id).values()
+        for node in node_list:
+            nodes.append({"show": node["show"],
+                          "top": node["top"],
+                          "left": node["left"],
+                          "ico": node["ico"],
+                          "type": node["node_type"],
+                          "name": node["name"],
+                          "node_data": {
+                              "inputs": node["inputs"],
+                              "run_mark": 0,
+                              "node_name": node["name"],
+                              "description": node["description"],
+                              "fail_retry_count": node["fail_retry_count"],
+                              "fail_offset": node["fail_offset"],
+                              "fail_offset_unit": node["fail_offset_unit"],
+                              "is_skip_fail": node["is_skip_fail"],
+                              "is_timeout_alarm": node["is_timeout_alarm"]},
+                          "uuid": node["uuid"]})
+        return {"lines": lines, "nodes": nodes}
+
+    class Meta:
+        model = Process
+        fields = ("id", "name", "description", "category", "run_type", "pipeline_tree")
+
+
+class ExecuteProcessSerializer(serializers.Serializer):
+    process_id = serializers.IntegerField(required=True)

applications/flow/tests.py

@@ -0,0 +1,3 @@
+from django.test import TestCase
+
+# Create your tests here.

applications/flow/urls.py

@@ -0,0 +1,6 @@
+from rest_framework.routers import DefaultRouter
+
+from . import views
+
+flow_router = DefaultRouter()
+flow_router.register(r"", viewset=views.ProcessViewSets, base_name="flow")

applications/flow/views.py

@@ -0,0 +1,103 @@
+from bamboo_engine import api
+from bamboo_engine.builder import *
+from django.http import JsonResponse
+from pipeline.eri.runtime import BambooDjangoRuntime
+from rest_framework import mixins
+from rest_framework.decorators import action
+from rest_framework.response import Response
+
+from applications.flow.models import Process, Node
+from applications.flow.serializers import ProcessViewSetsSerializer, ListProcessViewSetsSerializer, \
+    RetrieveProcessViewSetsSerializer, ExecuteProcessSerializer
+from applications.utils.dag_helper import DAG
+from component.drf.viewsets import GenericViewSet
+
+
+class ProcessViewSets(mixins.ListModelMixin,
+                      mixins.CreateModelMixin,
+                      mixins.RetrieveModelMixin,
+                      GenericViewSet):
+    serializer_class = ProcessViewSetsSerializer
+    queryset = Process.objects.order_by("-update_time")
+
+    def get_serializer_class(self):
+        if self.action == "list":
+            return ListProcessViewSetsSerializer
+        elif self.action == "retrieve":
+            return RetrieveProcessViewSetsSerializer
+        elif self.action == "execute":
+            return ExecuteProcessSerializer
+        return ProcessViewSetsSerializer
+
+    @action(methods=["POST"], detail=False)
+    def execute(self, request, *args, **kwargs):
+        validated_data = self.is_validated_data(request.data)
+        process_id = validated_data["process_id"]
+        process = Process.objects.filter(id=process_id).first()
+        node_map = Node.objects.filter(process_id=process_id).in_bulk(field_name="uuid")
+        dag_obj = DAG()
+        dag_obj.from_dict(process.dag)
+        topological_sort = dag_obj.topological_sort()
+        start = pipeline_tree = EmptyStartEvent()
+
+        for pipeline_id in topological_sort[1:]:
+            if node_map[pipeline_id].node_type == 0:
+                act = EmptyStartEvent()
+            elif node_map[pipeline_id].node_type == 1:
+                act = EmptyEndEvent()
+            else:
+                act = ServiceActivity(component_code="http_request")
+            pipeline_tree = getattr(pipeline_tree, "extend")(act)
+
+        pipeline_data = Data()
+        pipeline = builder.build_tree(start, data=pipeline_data)
+        print(pipeline)
+        runtime = BambooDjangoRuntime()
+        api.run_pipeline(runtime=runtime, pipeline=pipeline)
+        return Response({})
+
+
+# Create your views here.
+def flow(request):
+    # 使用 builder 构造出流程描述结构
+    start = EmptyStartEvent()
+    act = ServiceActivity(component_code="http_request")
+
+    act2 = ServiceActivity(component_code="fac_cal_comp")
+    act2.component.inputs.n = Var(type=Var.PLAIN, value=50)
+
+    act3 = ServiceActivity(component_code="fac_cal_comp")
+    act3.component.inputs.n = Var(type=Var.PLAIN, value=5)
+
+    act4 = ServiceActivity(component_code="fast_execute_job")
+    act5 = ServiceActivity(component_code="fast_execute_job")
+    eg = ExclusiveGateway(
+        conditions={
+            0: '${exe_res} >= 0',
+            1: '${exe_res} < 0'
+        },
+        name='act_2 or act_3'
+    )
+    pg = ParallelGateway()
+    cg = ConvergeGateway()
+
+    end = EmptyEndEvent()
+
+    start.extend(act).extend(eg).connect(act2, act3).to(eg).converge(pg).connect(act4, act5).to(pg).converge(cg).extend(
+        end)
+    # 全局变量
+    pipeline_data = Data()
+    pipeline_data.inputs['${exe_res}'] = NodeOutput(type=Var.PLAIN, source_act=act.id, source_key='exe_res')
+
+    pipeline = builder.build_tree(start, data=pipeline_data)
+    print(pipeline)
+    # 执行流程对象
+    runtime = BambooDjangoRuntime()
+
+    api.run_pipeline(runtime=runtime, pipeline=pipeline)
+
+    result = api.get_pipeline_states(runtime=runtime, root_id=pipeline["id"])
+
+    result_output = api.get_execution_data_outputs(runtime, act.id).data
+    # api.pause_pipeline(runtime=runtime, pipeline_id=pipeline["id"])
+    return JsonResponse({})

applications/utils/dag_helper.py

@@ -0,0 +1,205 @@
+from collections import OrderedDict, defaultdict
+from copy import copy, deepcopy
+
+
+class DAG(object):
+    """ Directed acyclic graph implementation. """
+
+    def __init__(self):
+        """ Construct a new DAG with no nodes or edges. """
+        self.reset_graph()
+
+    def add_node(self, node_name, graph=None):
+        """ Add a node if it does not exist yet, or error out. """
+        if not graph:
+            graph = self.graph
+        if node_name in graph:
+            raise KeyError('node %s already exists' % node_name)
+        graph[node_name] = set()
+
+    def add_node_if_not_exists(self, node_name, graph=None):
+        try:
+            self.add_node(node_name, graph=graph)
+        except KeyError:
+            pass
+
+    def delete_node(self, node_name, graph=None):
+        """ Deletes this node and all edges referencing it. """
+        if not graph:
+            graph = self.graph
+        if node_name not in graph:
+            raise KeyError('node %s does not exist' % node_name)
+        graph.pop(node_name)
+
+        for node, edges in graph.items():
+            if node_name in edges:
+                edges.remove(node_name)
+
+    def delete_node_if_exists(self, node_name, graph=None):
+        try:
+            self.delete_node(node_name, graph=graph)
+        except KeyError:
+            pass
+
+    def add_edge(self, ind_node, dep_node, graph=None):
+        """ Add an edge (dependency) between the specified nodes. """
+        if not graph:
+            graph = self.graph
+        if ind_node not in graph or dep_node not in graph:
+            raise KeyError('one or more nodes do not exist in graph')
+        test_graph = deepcopy(graph)
+        test_graph[ind_node].add(dep_node)
+        is_valid, message = self.validate(test_graph)
+        if is_valid:
+            graph[ind_node].add(dep_node)
+        else:
+            raise Exception()
+
+    def delete_edge(self, ind_node, dep_node, graph=None):
+        """ Delete an edge from the graph. """
+        if not graph:
+            graph = self.graph
+        if dep_node not in graph.get(ind_node, []):
+            raise KeyError('this edge does not exist in graph')
+        graph[ind_node].remove(dep_node)
+
+    def rename_edges(self, old_task_name, new_task_name, graph=None):
+        """ Change references to a task in existing edges. """
+        if not graph:
+            graph = self.graph
+        for node, edges in graph.items():
+
+            if node == old_task_name:
+                graph[new_task_name] = copy(edges)
+                del graph[old_task_name]
+
+            else:
+                if old_task_name in edges:
+                    edges.remove(old_task_name)
+                    edges.add(new_task_name)
+
+    def predecessors(self, node, graph=None):
+        """ Returns a list of all predecessors of the given node """
+        if graph is None:
+            graph = self.graph
+        return [key for key in graph if node in graph[key]]
+
+    def downstream(self, node, graph=None):
+        """ Returns a list of all nodes this node has edges towards. """
+        if graph is None:
+            graph = self.graph
+        if node not in graph:
+            raise KeyError('node %s is not in graph' % node)
+        return list(graph[node])
+
+    def all_downstreams(self, node, graph=None):
+        """Returns a list of all nodes ultimately downstream
+        of the given node in the dependency graph, in
+        topological order."""
+        if graph is None:
+            graph = self.graph
+        nodes = [node]
+        nodes_seen = set()
+        i = 0
+        while i < len(nodes):
+            downstreams = self.downstream(nodes[i], graph)
+            for downstream_node in downstreams:
+                if downstream_node not in nodes_seen:
+                    nodes_seen.add(downstream_node)
+                    nodes.append(downstream_node)
+            i += 1
+        return list(
+            filter(
+                lambda node: node in nodes_seen,
+                self.topological_sort(graph=graph)
+            )
+        )
+
+    def all_leaves(self, graph=None):
+        """ Return a list of all leaves (nodes with no downstreams) """
+        if graph is None:
+            graph = self.graph
+        return [key for key in graph if not graph[key]]
+
+    def from_dict(self, graph_dict):
+        """ Reset the graph and build it from the passed dictionary.
+        The dictionary takes the form of {node_name: [directed edges]}
+        """
+
+        self.reset_graph()
+        for new_node in graph_dict.keys():
+            self.add_node(new_node)
+        for ind_node, dep_nodes in graph_dict.items():
+            if not isinstance(dep_nodes, list):
+                raise TypeError('dict values must be lists')
+            for dep_node in dep_nodes:
+                self.add_edge(ind_node, dep_node)
+
+    def reset_graph(self):
+        """ Restore the graph to an empty state. """
+        self.graph = OrderedDict()
+
+    def ind_nodes(self, graph=None):
+        """ Returns a list of all nodes in the graph with no dependencies. """
+        if graph is None:
+            graph = self.graph
+
+        dependent_nodes = set(
+            node for dependents in graph.values() for node in dependents
+        )
+        return [node for node in graph.keys() if node not in dependent_nodes]
+
+    def validate(self, graph=None):
+        """ Returns (Boolean, message) of whether DAG is valid. """
+        graph = graph if graph is not None else self.graph
+        if len(self.ind_nodes(graph)) == 0:
+            return False, 'no independent nodes detected'
+        try:
+            self.topological_sort(graph)
+        except ValueError:
+            return False, 'failed topological sort'
+        return True, 'valid'
+
+    def topological_sort(self, graph=None):
+        """ Returns a topological ordering of the DAG.
+        Raises an error if this is not possible (graph is not valid).
+        """
+        if graph is None:
+            graph = self.graph
+        result = []
+        in_degree = defaultdict(lambda: 0)
+
+        for u in graph:
+            for v in graph[u]:
+                in_degree[v] += 1
+        ready = [node for node in graph if not in_degree[node]]
+
+        while ready:
+            u = ready.pop()
+            result.append(u)
+            for v in graph[u]:
+                in_degree[v] -= 1
+                if in_degree[v] == 0:
+                    ready.append(v)
+
+        if len(result) == len(graph):
+            return result
+        else:
+            raise ValueError('graph is not acyclic')
+
+    def size(self):
+        return len(self.graph)
+
+
+if __name__ == '__main__':
+    dag = DAG()
+    dag.add_node("a")
+    dag.add_node("b")
+    dag.add_node("c")
+    dag.add_node("d")
+    dag.add_edge("a", "b")
+    dag.add_edge("a", "d")
+    dag.add_edge("b", "c")
+    print(dag.topological_sort())
+    print(dag.graph)
+    print(dag.all_downstreams("b"))

component/drf/__init__.py

@@ -0,0 +1 @@
+# -*- coding: utf-8 -*-

component/drf/authentication.py

@@ -0,0 +1,6 @@
+from rest_framework.authentication import SessionAuthentication
+
+
+class CsrfExemptSessionAuthentication(SessionAuthentication):
+    def enforce_csrf(self, request):
+        return

component/drf/constants.py

@@ -0,0 +1,31 @@
+# -*- coding: utf-8 -*-
+
+from component.utils import choices_to_namedtuple, tuple_choices
+
+# 返回状态码
+CODE_STATUS_TUPLE = (
+    "OK",
+    "UNAUTHORIZED",
+    "VALIDATE_ERROR",
+    "METHOD_NOT_ALLOWED",
+    "PERMISSION_DENIED",
+    "SERVER_500_ERROR",
+    "OBJECT_NOT_EXIST",
+)
+CODE_STATUS_CHOICES = tuple_choices(CODE_STATUS_TUPLE)
+ResponseCodeStatus = choices_to_namedtuple(CODE_STATUS_CHOICES)
+
+# 常规字段长度定义
+LEN_SHORT = 32
+LEN_NORMAL = 64
+LEN_MIDDLE = 128
+LEN_LONG = 255
+LEN_X_LONG = 1000
+LEN_XX_LONG = 10000
+LEN_XXX_LONG = 20000
+
+# 字段默认值
+EMPTY_INT = 0
+EMPTY_STRING = ""
+EMPTY_LIST = []
+EMPTY_DICT = {}

component/drf/filters.py

@@ -0,0 +1,37 @@
+# -*- coding: utf-8 -*-
+
+from rest_framework import filters
+
+
+class OrderingFilter(filters.OrderingFilter):
+    def filter_queryset(self, request, queryset, view):
+        orderings = self.get_ordering(request, queryset, view)
+
+        if orderings:
+            custom_ordering = self.get_custom_ordering(request, view, orderings)
+            return queryset.extra(select=custom_ordering, order_by=orderings)
+
+        return queryset
+
+    @staticmethod
+    def get_ordering_class(view):
+        return getattr(view, "ordering_class", None)
+
+    def get_custom_ordering(self, request, view, orderings):
+        custom_ordering = {}
+        ordering_class = self.get_ordering_class(view)
+
+        # viewset whether to define ordering class
+        if ordering_class:
+            for index, order_name in enumerate(orderings):
+                reverse = order_name.startswith("-")
+                order_func = getattr(ordering_class, order_name.lstrip("-"), None)
+
+                # ordering class whether to define order method, Note: method name cannot be the same as field name
+                if order_func:
+                    custom_order = order_func(reverse, request)
+                    custom_order_name = order_name.lstrip("-")
+                    custom_ordering.update({custom_order_name: custom_order})
+                    orderings[index] = custom_order_name
+
+        return custom_ordering

component/drf/generics.py

@@ -0,0 +1,66 @@
+# -*- coding: utf-8 -*-
+"""
+框架补充相关代码
+"""
+
+from django.http import Http404
+from rest_framework import exceptions
+from rest_framework.exceptions import PermissionDenied
+from rest_framework.response import Response
+from rest_framework.settings import api_settings
+from rest_framework.views import set_rollback
+
+from component.drf.mapping import exception_mapping
+
+
+def exception_handler(exc, context):
+    """
+    Returns the response that should be used for any given exception.
+
+    By default we handle the REST framework `APIException`, and also
+    Django's built-in `Http404` and `PermissionDenied` exceptions.
+
+    Any unhandled exceptions may return `None`, which will cause a 500 error
+    to be raised.
+    (Rewrite default method exception_handler)
+    """
+    if isinstance(exc, Http404):
+        exc = exceptions.NotFound()
+    elif isinstance(exc, PermissionDenied):
+        exc = exceptions.PermissionDenied()
+
+    if isinstance(exc, exceptions.APIException):
+        headers = {}
+        if getattr(exc, "auth_header", None):
+            headers["WWW-Authenticate"] = exc.auth_header
+        if getattr(exc, "wait", None):
+            headers["Retry-After"] = "%d" % exc.wait
+
+        if isinstance(exc.detail, (list, dict)):
+            data = exc.detail
+        else:
+            data = {"detail": exc.detail}
+
+        set_rollback()
+        # code is added blow
+        exc_class_name = exc.__class__.__name__
+        if exc_class_name in exception_mapping:
+            message_list = []
+            # data type is in (list, dict)
+            if isinstance(data, dict):
+                for (k, v) in data.items():
+                    if isinstance(v, list):
+                        # remove 'non_field_errors' key name
+                        if k in (api_settings.NON_FIELD_ERRORS_KEY, "detail"):
+                            message_list.extend([str(i) for i in v])
+                        else:
+                            message_list.extend(["{0}: {1}".format(str(k), str(i)) for i in v])
+                    else:
+                        message_list.append(str(v))
+            elif isinstance(data, list):
+                message_list.extend([str(item) for item in data])
+            raise exception_mapping[exc_class_name](";".join(message_list))
+        else:
+            return Response(data, status=exc.status_code, headers=headers)
+
+    return None

component/drf/mapping.py

@@ -0,0 +1,18 @@
+# -*- coding: utf-8 -*-
+from component.utils.exceptions import (AuthenticationError, NotAuthenticatedError,
+                                        PermissionDeniedError, MethodNotAllowedError,
+                                        NotAcceptableError, UnsupportedMediaTypeError,
+                                        ThrottledError, ParamValidationError, ResourceNotFound)
+
+# drf exception to blueapps exception
+exception_mapping = {
+    "ValidationError": ParamValidationError,
+    "AuthenticationFailed": AuthenticationError,
+    "NotAuthenticated": NotAuthenticatedError,
+    "PermissionDenied": PermissionDeniedError,
+    "NotFound": ResourceNotFound,
+    "MethodNotAllowed": MethodNotAllowedError,
+    "NotAcceptable": NotAcceptableError,
+    "UnsupportedMediaType": UnsupportedMediaTypeError,
+    "Throttled": ThrottledError
+}

component/drf/middleware.py

@@ -0,0 +1,104 @@
+# -*- coding: utf-8 -*-
+"""
+Tencent is pleased to support the open source community by making 蓝鲸智云PaaS平台社区版 (BlueKing PaaS Community
+Edition) available.
+Copyright (C) 2017-2020 THL A29 Limited, a Tencent company. All rights reserved.
+Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+http://opensource.org/licenses/MIT
+Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+specific language governing permissions and limitations under the License.
+"""
+
+import json
+import logging
+import traceback
+
+from django.conf import settings
+from django.http import Http404, JsonResponse
+from django.utils.deprecation import MiddlewareMixin
+from django.utils.translation import gettext_lazy as _
+
+from component.utils.exceptions import BlueException
+
+try:
+    from raven.contrib.django.raven_compat.models import \
+        sentry_exception_handler
+# 兼容未有安装sentry的情况
+except ImportError:
+    sentry_exception_handler = None
+
+logger = logging.getLogger('blueapps')
+
+
+class AppExceptionMiddleware(MiddlewareMixin):
+
+    def process_exception(self, request, exception):
+        """
+        app后台错误统一处理
+        """
+
+        self.exception = exception
+        self.request = request
+
+        # 用户自我感知的异常抛出
+        if isinstance(exception, BlueException):
+            logger.log(
+                exception.LOG_LEVEL,
+                (u"""捕获主动抛出异常, 具体异常堆栈->[%s] status_code->[%s] & """
+                 u"""client_message->[%s] & args->[%s] """) % (
+                    traceback.format_exc(),
+                    exception.ERROR_CODE,
+                    exception.message,
+                    exception.args
+                )
+            )
+
+            response = JsonResponse(exception.response_data())
+
+            response.status_code = exception.STATUS_CODE
+            return response
+
+        # 用户未主动捕获的异常
+        logger.error(
+            (u"""捕获未处理异常,异常具体堆栈->[%s], 请求URL->[%s], """
+             u"""请求方法->[%s] 请求参数->[%s]""") % (
+                traceback.format_exc(),
+                request.path,
+                request.method,
+                json.dumps(getattr(request, request.method, None))
+            )
+        )
+
+        # 对于check开头函数进行遍历调用，如有满足条件的函数，则不屏蔽异常
+        check_funtions = self.get_check_functions()
+        for check_function in check_funtions:
+            if check_function():
+                return None
+
+        response = JsonResponse({
+            "result": False,
+            'code': "50000",
+            'message': _(u"系统异常,请联系管理员处理"),
+            'data': None
+        })
+        response.status_code = 500
+
+        # notify sentry
+        if sentry_exception_handler is not None:
+            sentry_exception_handler(request=request)
+
+        return response
+
+    def get_check_functions(self):
+        """获取需要判断的函数列表"""
+        return [getattr(self, func) for func in dir(self) if func.startswith('check') and callable(getattr(self, func))]
+
+    def check_is_debug(self):
+        """判断是否是开发模式"""
+        return settings.DEBUG
+
+    def check_is_http404(self):
+        """判断是否基于Http404异常"""
+        return isinstance(self.exception, Http404)

component/drf/mixins.py

@@ -0,0 +1,82 @@
+# -*- coding: utf-8 -*-
+
+from rest_framework import status
+from rest_framework.response import Response
+
+from component.drf.constants import ResponseCodeStatus
+
+
+class ApiGenericMixin(object):
+    """API视图类通用函数"""
+
+    # TODO 权限部分加载基类中
+    permission_classes = ()
+
+    def finalize_response(self, request, response, *args, **kwargs):
+        """统一数据返回格式"""
+        # 文件导出时response {HttpResponse}
+        if not isinstance(response, Response):
+            return response
+        if response.data is None:
+            response.data = {"result": True, "code": ResponseCodeStatus.OK, "message": "success", "data": []}
+        elif isinstance(response.data, (list, tuple)):
+            response.data = {
+                "result": True,
+                "code": ResponseCodeStatus.OK,
+                "message": "success",
+                "data": response.data,
+            }
+        elif isinstance(response.data, dict):
+            if not ("result" in response.data):
+                response.data = {
+                    "result": True,
+                    "code": ResponseCodeStatus.OK,
+                    "message": "success",
+                    "data": response.data,
+                }
+            else:
+                response.data = {
+                    "result": response.data["result"],
+                    "code": ResponseCodeStatus.OK,
+                    "message": response.data.get("message"),
+                    "data": response.data,
+                }
+        if response.status_code == status.HTTP_204_NO_CONTENT and request.method == "DELETE":
+            response.status_code = status.HTTP_200_OK
+
+        return super(ApiGenericMixin, self).finalize_response(request, response, *args, **kwargs)
+
+
+class ApiGatewayMixin(object):
+    """对外开放API返回格式统一
+    错误码返回规范为数字：
+        正确：0
+        错误：39XXXXX
+    """
+
+    permission_classes = ()
+
+    def finalize_response(self, request, response, *args, **kwargs):
+        """统一数据返回格式"""
+
+        if not isinstance(response, Response):
+            return response
+
+        if response.data is None:
+            response.data = {"result": True, "code": 0, "message": "success", "data": []}
+        elif isinstance(response.data, (list, tuple)):
+            response.data = {
+                "result": True,
+                "code": 0,
+                "message": "success",
+                "data": response.data,
+            }
+        elif isinstance(response.data, dict) and not ("code" in response.data and "result" in response.data):
+            response.data = {
+                "result": True,
+                "code": 0,
+                "message": "success",
+                "data": response.data,
+            }
+
+        return super(ApiGatewayMixin, self).finalize_response(request, response, *args, **kwargs)

component/drf/pagination.py

@@ -0,0 +1,37 @@
+# -*- coding: utf-8 -*-
+from collections import OrderedDict
+
+from rest_framework.pagination import PageNumberPagination
+from rest_framework.response import Response
+
+
+class CustomPageNumberPagination(PageNumberPagination):
+    """
+    自定义分页格式，综合页码和url
+    """
+
+    page_size = 5
+    page_size_query_param = "page_size"
+    max_page_size = 10000
+
+    def get_paginated_response(self, data):
+        return Response(
+            OrderedDict(
+                [
+                    ("page", self.page.number),
+                    ("total_page", self.page.paginator.num_pages),
+                    ("count", self.page.paginator.count),
+                    ("items", data),
+                ]
+            )
+        )
+
+    def get_paginated_data(self, data):
+        return OrderedDict(
+            [
+                ("page", self.page.number),
+                ("total_page", self.page.paginator.num_pages),
+                ("count", self.page.paginator.count),
+                ("items", data),
+            ]
+        )

component/drf/renderers.py

@@ -0,0 +1,64 @@
+# -*- coding: utf-8 -*-
+"""
+自定义drf renderers 使返回格式和ESB接口返回格式相同
+使用方法:
+django settings 中添加 rest_framework配置
+REST_FRAMEWORK = {
+    "DEFAULT_RENDERER_CLASSES": ("component.drf.renderers.CustomRenderer",),
+}
+
+"""
+from rest_framework import status
+from rest_framework.renderers import JSONRenderer
+
+
+class CustomRenderer(JSONRenderer):
+    @staticmethod
+    def _format_validation_message(detail):
+        """格式化drf校验错误信息"""
+
+        if isinstance(detail, list):
+            message = "; ".join(["{}:{}".format(k, v) for k, v in enumerate(detail)])
+        elif isinstance(detail, dict):
+            messages = []
+            for k, v in detail.items():
+                if isinstance(v, list):
+                    try:
+                        messages.append("{}:{}".format(k, ",".join(v)))
+                    except TypeError:
+                        messages.append("{}:{}".format(k, "部分列表元素的参数不合法，请检查"))
+                else:
+                    messages.append("{}:{}".format(k, v))
+            message = ";".join(messages)
+        else:
+            message = detail
+
+        return message
+
+    def render(self, data, accepted_media_type=None, renderer_context=None):
+        """重构render方法"""
+
+        request = renderer_context.get("request")
+        response = renderer_context.get("response")
+
+        # 更改删除成功的状态码, 204 --> 200
+        if response.status_code == status.HTTP_204_NO_CONTENT and request.method == "DELETE":
+            response.status_code = status.HTTP_200_OK
+
+        # 重新构建返回的JSON字典
+        if response and status.is_success(response.status_code):
+            ret = {
+                "result": True,
+                "code": str(response.status_code * 100),
+                "message": "success",
+                "data": data,
+            }
+        else:
+            ret = {
+                "result": False,
+                "code": str((response.status_code if response else 500) * 100),
+                "message": self._format_validation_message(detail=data.get("detail", "") or data),
+                "data": data,
+            }
+        # 返回JSON数据
+        return super(CustomRenderer, self).render(ret, accepted_media_type, renderer_context)

component/drf/viewsets.py

@@ -0,0 +1,136 @@
+# -*- coding: utf-8 -*-
+"""
+views相关模块代码
+"""
+import math
+from collections import OrderedDict
+
+from django.db import transaction
+from rest_framework import mixins, viewsets
+from rest_framework.response import Response
+from rest_framework.views import APIView as _APIView
+
+from component.drf.mixins import ApiGenericMixin
+
+
+class APIView(ApiGenericMixin, _APIView):
+    """APIView"""
+
+    pass
+
+
+class ModelViewSet(ApiGenericMixin, viewsets.ModelViewSet):
+    """按需改造DRF默认的ModelViewSet类"""
+
+    def perform_create(self, serializer):
+        """创建时补充基础Model中的字段"""
+        user = serializer.context.get("request").user
+        username = getattr(user, "username", "guest")
+        serializer.save(creator=username, updated_by=username)
+
+    def perform_update(self, serializer):
+        """更新时补充基础Model中的字段"""
+        user = serializer.context.get("request").user
+        username = getattr(user, "username", "guest")
+        serializer.save(updated_by=username)
+
+
+class ReadOnlyModelViewSet(ApiGenericMixin, viewsets.ReadOnlyModelViewSet):
+    """按需改造DRF默认的ModelViewSet类"""
+
+    pass
+
+
+class ViewSet(ApiGenericMixin, viewsets.ViewSet):
+    """按需改造DRF默认的ViewSet类"""
+
+    pass
+
+
+class GenericViewSet(ApiGenericMixin, viewsets.GenericViewSet):
+    """按需改造DRF默认的GenericViewSet类"""
+
+    def is_validated_data(self, data):
+        serializer = self.get_serializer(data=data)
+        serializer.is_valid(raise_exception=True)
+        return serializer.validated_data
+
+    def get_page_info(self, validated_data):
+        page = validated_data.get("page", 1)
+        page_size = validated_data.get("page_size", 5)
+        start = (int(page) - 1) * int(page_size)
+        end = start + int(page_size)
+        return start, end
+
+    def my_paginated_response(self, validated_data, total_count, return_data):
+        page = int(validated_data.get("page", 1))
+        page_size = int(validated_data.get("page_size", 5))
+        total_page = math.ceil(total_count / page_size)
+        return Response(
+            OrderedDict(
+                [
+                    ("page", page),
+                    ("total_page", total_page),
+                    ("count", total_count),
+                    ("items", return_data),
+                ]
+            )
+        )
+
+    def convert_post_to_get(self, request):
+        data = request.query_params
+        _mutable = data._mutable
+        data._mutable = True
+        data.update(request.data)
+        data._mutable = _mutable
+
+
+class CreateModelAndLogMixin(mixins.CreateModelMixin):
+    """
+    Create a model instance and log.
+    """
+
+    @transaction.atomic()
+    def perform_create(self, serializer):
+        # 补充基础Model--MaintainerFieldsMixin中的字段
+        user = serializer.context.get("request").user
+        username = getattr(user, "username", "guest")
+        instance = serializer.save()
+        log_type, obj, detail = instance.get_summary_title().split("/")
+
+
+class UpdateModelAndLogMixin(mixins.UpdateModelMixin):
+    """
+    Update a model instance and log.
+    """
+
+    @transaction.atomic()
+    def perform_update(self, serializer):
+        # 补充基础Model--MaintainerFieldsMixin中的字段
+        user = serializer.context.get("request").user
+        username = getattr(user, "username", "guest")
+        instance = serializer.save(updated_by=username)
+        log_type, obj, detail = instance.get_summary_title().split("/")
+
+
+class DestroyModelAndLogMixin(mixins.DestroyModelMixin):
+    """
+    Destroy a model instance and log.
+    """
+
+    def perform_destroy(self, instance):
+        with transaction.atomic():
+            log_type, obj, detail = instance.get_summary_title().split("/")
+            username = getattr(self, "request").user.username
+            instance.delete()
+
+
+class ModelAndLogViewSet(
+    mixins.ListModelMixin,
+    CreateModelAndLogMixin,
+    mixins.RetrieveModelMixin,
+    UpdateModelAndLogMixin,
+    DestroyModelAndLogMixin,
+    GenericViewSet,
+):
+    pass

component/utils/__init__.py

@@ -0,0 +1,4 @@
+# -*- coding: utf-8 -*-
+
+from .basic import *  # noqa
+from .drf import *  # noqa

component/utils/basic.py

@@ -0,0 +1,18 @@
+# -*- coding: utf-8 -*-
+
+from collections import namedtuple
+
+
+def tuple_choices(tupl):
+    """从django-model的choices转换到namedtuple"""
+    return [(t, t) for t in tupl]
+
+
+def dict_to_namedtuple(dic):
+    """从dict转换到namedtuple"""
+    return namedtuple("AttrStore", list(dic.keys()))(**dic)
+
+
+def choices_to_namedtuple(choices):
+    """从django-model的choices转换到namedtuple"""
+    return dict_to_namedtuple(dict(choices))

component/utils/drf.py

@@ -0,0 +1,24 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+
+def format_validation_message(e):
+    """格式化drf校验错误信息"""
+
+    if isinstance(e.detail, list):
+        message = "; ".join(["{}:{}".format(k, v) for k, v in enumerate(e.detail)])
+    elif isinstance(e.detail, dict):
+        messages = []
+        for k, v in e.detail.items():
+            if isinstance(v, list):
+                try:
+                    messages.append("{}:{}".format(k, ",".join(v)))
+                except TypeError:
+                    messages.append("{}:{}".format(k, "部分列表元素的参数不合法，请检查"))
+            else:
+                messages.append("{}:{}".format(k, v))
+        message = ";".join(messages)
+    else:
+        message = e.detail
+
+    return message

component/utils/exceptions.py

@@ -0,0 +1,117 @@
+# -*- coding: utf-8 -*-
+import logging
+
+from django.utils.translation import ugettext as _
+
+
+class BlueException(Exception):
+    ERROR_CODE = "0000000"
+    MESSAGE = _("APP异常")
+    STATUS_CODE = 500
+    LOG_LEVEL = logging.ERROR
+
+    def __init__(self, message=None, data=None, *args):
+        """
+        :param message: 错误消息
+        :param data: 其他数据
+        :param context: 错误消息 format dict
+        :param args: 其他参数
+        """
+        super(BlueException, self).__init__(*args)
+        self.message = self.MESSAGE if message is None else message
+        self.data = data
+
+    def render_data(self):
+        return self.data
+
+    def response_data(self):
+        return {
+            "result": False,
+            "code": self.ERROR_CODE,
+            "message": self.message,
+            "data": self.render_data()
+        }
+
+
+class ClientBlueException(BlueException):
+    MESSAGE = _("客户端请求异常")
+    ERROR_CODE = "40000"
+    STATUS_CODE = 400
+
+
+class ServerBlueException(BlueException):
+    MESSAGE = _("服务端服务异常")
+    ERROR_CODE = "50000"
+    STATUS_CODE = 500
+
+
+class AuthenticationError(ClientBlueException):
+    MESSAGE = _("认证失败")
+    ERROR_CODE = "40100"
+    STATUS_CODE = 401
+
+
+class NotAuthenticatedError(ClientBlueException):
+    MESSAGE = _("未提供身份验证凭据")
+    ERROR_CODE = "40101"
+    STATUS_CODE = 401
+
+
+class PermissionDeniedError(ClientBlueException):
+    MESSAGE = _("您无权执行此操作")
+    ERROR_CODE = "40302"
+    STATUS_CODE = 403
+
+
+class MethodNotAllowedError(ClientBlueException):
+    MESSAGE = _("请求方法不被允许")
+    ERROR_CODE = "40504"
+    STATUS_CODE = 405
+
+
+class NotAcceptableError(ClientBlueException):
+    MESSAGE = _("无法满足请求Accept头")
+    ERROR_CODE = "40600"
+    STATUS_CODE = 406
+
+
+class UnsupportedMediaTypeError(ClientBlueException):
+    MESSAGE = _("不支持的媒体类型")
+    ERROR_CODE = "41500"
+    STATUS_CODE = 415
+
+
+class ThrottledError(ClientBlueException):
+    MESSAGE = _("请求被限制")
+    ERROR_CODE = "42900"
+    STATUS_CODE = 429
+
+
+class DeleteError(ServerBlueException):
+    MESSAGE = _("数据删除失败")
+    ERROR_CODE = "50001"
+    STATUS_CODE = 500
+
+
+class UpdateError(ServerBlueException):
+    MESSAGE = _("数据更新失败")
+    ERROR_CODE = "50002"
+    STATUS_CODE = 500
+
+
+class BkEsbReturnError(ServerBlueException):
+    MESSAGE = _("ESB调用返回错误")
+    ERROR_CODE = "50302"
+    STATUS_CODE = 503
+
+
+class ParamValidationError(ClientBlueException):
+    MESSAGE = _("参数验证失败")
+    ERROR_CODE = "40000"
+    STATUS_CODE = 400
+
+
+class ResourceNotFound(ClientBlueException):
+    MESSAGE = _("找不到请求的资源")
+    ERROR_CODE = "40400"
+    STATUS_CODE = 404

custom_plugins/__init__.py

@@ -0,0 +1,2 @@
+# -*- coding: utf-8 -*-
+

custom_plugins/apps.py

@@ -0,0 +1,7 @@
+# -*- coding: utf-8 -*-
+from django.apps import AppConfig
+
+
+class CustomPluginsConfig(AppConfig):
+    default_auto_field = 'django.db.models.BigAutoField'
+    name = 'custom_plugins'

custom_plugins/components/__init__.py

@@ -0,0 +1,10 @@
+# -*- coding: utf-8 -*-
+
+
+import logging
+
+from pipeline.conf import settings
+from pipeline.core.flow.activity import Service
+from pipeline.component_framework.component import Component
+
+logger = logging.getLogger('celery')

custom_plugins/components/collections/__init__.py

@@ -0,0 +1,10 @@
+# -*- coding: utf-8 -*-
+
+
+import logging
+
+from pipeline.conf import settings
+from pipeline.core.flow.activity import Service
+from pipeline.component_framework.component import Component
+
+logger = logging.getLogger('celery')

custom_plugins/components/collections/plugins.py

@@ -0,0 +1,22 @@
+# -*- coding: utf-8 -*-
+import math
+from pipeline.core.flow.activity import Service, StaticIntervalGenerator
+from pipeline.component_framework.component import Component
+import json
+import eventlet
+
+requests = eventlet.import_patched('requests')
+
+
+class HttpRequestService(Service):
+    __need_schedule__ = False
+
+    def execute(self, data, parent_data):
+        print("执行了")
+        return True
+
+
+class HttpRequestComponent(Component):
+    name = "HttpRequestComponent"
+    code = "http_request"
+    bound_service = HttpRequestService

custom_plugins/migrations/__init__.py

@@ -0,0 +1 @@
+# -*- coding: utf-8 -*-

custom_plugins/static/custom_plugins/plugins.js

@@ -0,0 +1,12 @@
+
+/**
+* Tencent is pleased to support the open source community by making À¶¾¨ÖÇÔÆPaaSƽ̨ÉçÇø°æ (BlueKing PaaS Community
+* Edition) available.
+* Copyright (C) 2017-2021 THL A29 Limited, a Tencent company. All rights reserved.
+* Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License.
+* You may obtain a copy of the License at
+* http://opensource.org/licenses/MIT
+* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+* an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+* specific language governing permissions and limitations under the License.
+*/

custom_plugins/tests/__init__.py

@@ -0,0 +1,21 @@
+
+# -*- coding: utf-8 -*-
+"""
+Tencent is pleased to support the open source community by making À¶¾¨ÖÇÔÆPaaSƽ̨ÉçÇø°æ (BlueKing PaaS Community
+Edition) available.
+Copyright (C) 2017-2021 THL A29 Limited, a Tencent company. All rights reserved.
+Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+http://opensource.org/licenses/MIT
+Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+specific language governing permissions and limitations under the License.
+"""
+
+import logging
+
+from pipeline.conf import settings
+from pipeline.core.flow.activity import Service
+from pipeline.component_framework.component import Component
+
+logger = logging.getLogger('celery')

custom_plugins/tests/components/__init__.py

@@ -0,0 +1,21 @@
+
+# -*- coding: utf-8 -*-
+"""
+Tencent is pleased to support the open source community by making À¶¾¨ÖÇÔÆPaaSƽ̨ÉçÇø°æ (BlueKing PaaS Community
+Edition) available.
+Copyright (C) 2017-2021 THL A29 Limited, a Tencent company. All rights reserved.
+Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+http://opensource.org/licenses/MIT
+Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+specific language governing permissions and limitations under the License.
+"""
+
+import logging
+
+from pipeline.conf import settings
+from pipeline.core.flow.activity import Service
+from pipeline.component_framework.component import Component
+
+logger = logging.getLogger('celery')

custom_plugins/tests/components/collections/__init__.py

@@ -0,0 +1,21 @@
+
+# -*- coding: utf-8 -*-
+"""
+Tencent is pleased to support the open source community by making À¶¾¨ÖÇÔÆPaaSƽ̨ÉçÇø°æ (BlueKing PaaS Community
+Edition) available.
+Copyright (C) 2017-2021 THL A29 Limited, a Tencent company. All rights reserved.
+Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+http://opensource.org/licenses/MIT
+Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+specific language governing permissions and limitations under the License.
+"""
+
+import logging
+
+from pipeline.conf import settings
+from pipeline.core.flow.activity import Service
+from pipeline.component_framework.component import Component
+
+logger = logging.getLogger('celery')

custom_plugins/tests/components/collections/plugins_test/__init__.py

@@ -0,0 +1,21 @@
+
+# -*- coding: utf-8 -*-
+"""
+Tencent is pleased to support the open source community by making À¶¾¨ÖÇÔÆPaaSƽ̨ÉçÇø°æ (BlueKing PaaS Community
+Edition) available.
+Copyright (C) 2017-2021 THL A29 Limited, a Tencent company. All rights reserved.
+Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+http://opensource.org/licenses/MIT
+Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+specific language governing permissions and limitations under the License.
+"""
+
+import logging
+
+from pipeline.conf import settings
+from pipeline.core.flow.activity import Service
+from pipeline.component_framework.component import Component
+
+logger = logging.getLogger('celery')

dj_flow/__init__.py

@@ -0,0 +1,3 @@
+from .celery_app import app as celery_app
+
+__all__ = ('celery_app',)

dj_flow/celery_app.py

@@ -0,0 +1,41 @@
+# -*- coding: utf-8 -*-
+"""
+Tencent is pleased to support the open source community by making 蓝鲸智云PaaS平台社区版 (BlueKing PaaS Community
+Edition) available.
+Copyright (C) 2017-2020 THL A29 Limited, a Tencent company. All rights reserved.
+Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+http://opensource.org/licenses/MIT
+Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+specific language governing permissions and limitations under the License.
+"""
+
+from __future__ import absolute_import, unicode_literals
+
+import os
+import time
+from celery import Celery, platforms
+from django.conf import settings
+
+platforms.C_FORCE_ROOT = True
+
+# set the default Django settings module for the 'celery' program.
+os.environ.setdefault("DJANGO_SETTINGS_MODULE", "dj_flow.settings")
+
+app = Celery("dj_flow")
+
+# Using a string here means the worker don't have to serialize
+# the configuration object to child processes.
+# - namespace='CELERY' means all celery-related configuration keys
+#   should have a `CELERY_` prefix.
+app.config_from_object("django.conf:settings")
+
+# Load task modules from all registered Django app configs.
+app.autodiscover_tasks(lambda: settings.INSTALLED_APPS)
+
+
+@app.task(bind=True)
+def debug_task(self):
+    print("Request: {!r}".format(self.request))
+    time.sleep(2)

dj_flow/settings.py

@@ -0,0 +1,146 @@
+import os
+from pathlib import Path
+
+# Build paths inside the project like this: BASE_DIR / 'subdir'.
+BASE_DIR = Path(__file__).resolve().parent.parent
+
+BROKER_URL = "redis://localhost:6379/3"
+# Quick-start development settings - unsuitable for production
+# See https://docs.djangoproject.com/en/3.2/howto/deployment/checklist/
+
+# SECURITY WARNING: keep the secret key used in production secret!
+SECRET_KEY = 'django-insecure-u5_r=pekio0@zt!y(kgbufuosb9mddu8*qeejkzj@=7uyvb392'
+
+# SECURITY WARNING: don't run with debug turned on in production!
+DEBUG = True
+
+ALLOWED_HOSTS = ["*"]
+CORS_ALLOW_CREDENTIALS = True
+CSRF_COOKIE_NAME = "dj-flow_csrftoken"
+CORS_ORIGIN_WHITELIST = [
+    "http://127.0.0.1:8080"
+]
+
+# Application definition
+
+INSTALLED_APPS = [
+    "corsheaders",
+    "pipeline",
+    "pipeline.engine",
+    "pipeline.component_framework",
+    "pipeline.eri",
+    'django.contrib.admin',
+    'django.contrib.auth',
+    'django.contrib.contenttypes',
+    'django.contrib.sessions',
+    'django.contrib.messages',
+    'django.contrib.staticfiles',
+    "custom_plugins",
+    "rest_framework",
+    "applications.flow"
+
+]
+
+MIDDLEWARE = [
+    "corsheaders.middleware.CorsMiddleware",
+    'django.middleware.security.SecurityMiddleware',
+    'django.contrib.sessions.middleware.SessionMiddleware',
+    'django.middleware.common.CommonMiddleware',
+    'django.middleware.csrf.CsrfViewMiddleware',
+    'django.contrib.auth.middleware.AuthenticationMiddleware',
+    'django.contrib.messages.middleware.MessageMiddleware',
+    'django.middleware.clickjacking.XFrameOptionsMiddleware',
+    "component.drf.middleware.AppExceptionMiddleware"
+]
+
+ROOT_URLCONF = 'dj_flow.urls'
+
+TEMPLATES = [
+    {
+        'BACKEND': 'django.template.backends.django.DjangoTemplates',
+        'DIRS': [],
+        'APP_DIRS': True,
+        'OPTIONS': {
+            'context_processors': [
+                'django.template.context_processors.debug',
+                'django.template.context_processors.request',
+                'django.contrib.auth.context_processors.auth',
+                'django.contrib.messages.context_processors.messages',
+            ],
+        },
+    },
+]
+
+WSGI_APPLICATION = 'dj_flow.wsgi.application'
+TIME_ZONE = "Asia/Shanghai"
+LANGUAGE_CODE = "zh-hans"
+# Database
+# https://docs.djangoproject.com/en/3.2/ref/settings/#databases
+
+DATABASES = {
+    "default": {
+        "ENGINE": "django.db.backends.mysql",
+        "NAME": "bomboo",  # noqa
+        "USER": "root",
+        "PASSWORD": "xhongc",
+        "HOST": "localhost",
+        "PORT": "3306",
+        # 单元测试 DB 配置，建议不改动
+        "TEST": {"NAME": "test_db", "CHARSET": "utf8", "COLLATION": "utf8_general_ci"},
+    },
+}
+
+# Password validation
+# https://docs.djangoproject.com/en/3.2/ref/settings/#auth-password-validators
+
+AUTH_PASSWORD_VALIDATORS = [
+    {
+        'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator',
+    },
+    {
+        'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator',
+    },
+    {
+        'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator',
+    },
+    {
+        'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator',
+    },
+]
+
+LANGUAGE_CODE = 'en-us'
+
+TIME_ZONE = 'UTC'
+
+USE_I18N = True
+
+USE_L10N = True
+
+USE_TZ = True
+
+STATIC_URL = '/static/'
+
+DEFAULT_AUTO_FIELD = 'django.db.models.BigAutoField'
+IS_USE_CELERY = True
+
+if IS_USE_CELERY:
+    INSTALLED_APPS += ("django_celery_beat", "django_celery_results")
+    CELERY_ENABLE_UTC = False
+    CELERY_TASK_SERIALIZER = "pickle"
+    CELERY_ACCEPT_CONTENT = ['pickle', ]
+    CELERYBEAT_SCHEDULER = "django_celery_beat.schedulers.DatabaseScheduler"
+
+REST_FRAMEWORK = {
+    "EXCEPTION_HANDLER": "component.drf.generics.exception_handler",
+    "DEFAULT_PERMISSION_CLASSES": ("rest_framework.permissions.IsAuthenticated",),
+    "DEFAULT_PAGINATION_CLASS": "component.drf.pagination.CustomPageNumberPagination",
+    "PAGE_SIZE": 10,
+    "TEST_REQUEST_DEFAULT_FORMAT": "json",
+    "DEFAULT_AUTHENTICATION_CLASSES": ("rest_framework.authentication.SessionAuthentication",),
+    "DEFAULT_FILTER_BACKENDS": (
+        "django_filters.rest_framework.DjangoFilterBackend",
+        "rest_framework.filters.OrderingFilter",
+    ),
+    "DATETIME_FORMAT": "%Y-%m-%d %H:%M:%S",
+    "NON_FIELD_ERRORS_KEY": "params_error",
+}

dj_flow/urls.py

@@ -0,0 +1,24 @@
+"""URL Configuration
+
+The `urlpatterns` list routes URLs to views. For more information please see:
+    https://docs.djangoproject.com/en/3.2/topics/http/urls/
+Examples:
+Function views
+    1. Add an import:  from my_app import views
+    2. Add a URL to urlpatterns:  path('', views.home, name='home')
+Class-based views
+    1. Add an import:  from other_app.views import Home
+    2. Add a URL to urlpatterns:  path('', Home.as_view(), name='home')
+Including another URLconf
+    1. Import the include() function: from django.urls import include, path
+    2. Add a URL to urlpatterns:  path('blog/', include('blog.urls'))
+"""
+from django.contrib import admin
+from django.urls import path, include
+from applications.flow.urls import flow_router
+
+urlpatterns = [
+    path('admin/', admin.site.urls),
+    path("process/", include(flow_router.urls)),
+
+]

dj_flow/wsgi.py

@@ -0,0 +1,16 @@
+"""
+WSGI config for  project.
+
+It exposes the WSGI callable as a module-level variable named ``application``.
+
+For more information on this file, see
+https://docs.djangoproject.com/en/3.2/howto/deployment/wsgi/
+"""
+
+import os
+
+from django.core.wsgi import get_wsgi_application
+
+os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'dj_flow.settings')
+
+application = get_wsgi_application()

manage.py

@@ -0,0 +1,22 @@
+#!/usr/bin/env python
+"""Django's command-line utility for administrative tasks."""
+import os
+import sys
+
+
+def main():
+    """Run administrative tasks."""
+    os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'dj_flow.settings')
+    try:
+        from django.core.management import execute_from_command_line
+    except ImportError as exc:
+        raise ImportError(
+            "Couldn't import Django. Are you sure it's installed and "
+            "available on your PYTHONPATH environment variable? Did you "
+            "forget to activate a virtual environment?"
+        ) from exc
+    execute_from_command_line(sys.argv)
+
+
+if __name__ == '__main__':
+    main()

requirements.txt

@@ -0,0 +1,9 @@
+Django==2.2.6
+bamboo-pipeline==3.14.0
+mysqlclient==1.4.4
+eventlet==0.33.0
+djangorestframework==3.8.1
+pyyaml==6.0
+django-cors-headers==3.2.1
+django-filter==2.0.0
+django-mysql==3.8.1

templates/index.html

@@ -0,0 +1,25 @@
+<!DOCTYPE html>
+<html>
+	<head>
+		<meta charset="utf-8">
+		<meta name="viewport" content="width=device-width,initial-scale=1">
+		<title>调度平台</title>
+		<link href="{{ STATIC_URL }}dist/css/app.css" rel="stylesheet">
+		<link href="{{ STATIC_URL }}img/{{ APP_CODE }}.png" rel="shortcut icon" type="image/x-icon">
+	</head>
+	<body>
+		<script>
+			if ('{{ RUN_MODE }}' === 'DEVELOP') {
+				window.siteUrl = window.location.origin
+			} else {
+				window.siteUrl = '{{ BK_PLAT_HOST }}{{ SITE_URL }}'
+			}
+			window.APP_CODE = '{{ APP_CODE }}';
+			window.CSRF_COOKIE_NAME = '{{ CSRF_COOKIE_NAME }}';
+		</script>
+		<div id="app"></div>
+		<script type="text/javascript" src="{{ STATIC_URL }}dist/js/manifest.js"></script>
+		<script type="text/javascript" src="{{ STATIC_URL }}dist/js/vendor.js"></script>
+		<script type="text/javascript" src="{{ STATIC_URL }}dist/js/app.js"></script>
+	</body>
+</html>

web/.babelrc

@@ -0,0 +1,15 @@
+{
+  "presets": [
+    ["env", {
+      "modules": false,
+      "targets": {
+        "browsers": ["> 1%", "last 2 versions", "not ie <= 8"]
+      }
+    }],
+    "stage-2"
+  ],
+  "plugins": ["transform-vue-jsx", "transform-runtime", ["import", {
+      "libraryName": "view-design",
+      "libraryDirectory": "src/components"
+  }]]
+}

web/.editorconfig

@@ -0,0 +1,9 @@
+root = true
+
+[*]
+charset = utf-8
+indent_style = space
+indent_size = 4
+end_of_line = lf
+insert_final_newline = true
+trim_trailing_whitespace = true

web/.eslintignore

@@ -0,0 +1,8 @@
+/build/
+/config/
+/dist/
+/*.js
+/node_modules/
+/src/assets/
+/src/components/
+/src/common/validate.js

web/.gitignore

@@ -0,0 +1,14 @@
+.DS_Store
+node_modules/
+/dist/
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+
+# Editor directories and files
+.idea
+.vscode
+*.suo
+*.ntvs*
+*.njsproj
+*.sln

web/.postcssrc.js

@@ -0,0 +1,10 @@
+// https://github.com/michael-ciniawsky/postcss-load-config
+
+module.exports = {
+  "plugins": {
+    "postcss-import": {},
+    "postcss-url": {},
+    // to edit target browsers: use "browserslist" field in package.json
+    "autoprefixer": {}
+  }
+}