refactor(性能测试): 切割csv文件

backend/src/main/java/io/metersphere/performance/engine/EngineContext.java

@@ -12,6 +12,7 @@ public class EngineContext {
     private String resourcePoolId;
     private String reportId;
     private Integer resourceIndex;
+    private double[] ratios;
     private Map<String, Object> properties = new HashMap<>();
     private Map<String, byte[]> testResourceFiles = new HashMap<>();
 
@@ -92,6 +93,13 @@ public class EngineContext {
         this.resourceIndex = resourceIndex;
     }
 
+    public double[] getRatios() {
+        return ratios;
+    }
+
+    public void setRatios(double[] ratios) {
+        this.ratios = ratios;
+    }
 
     public Map<String, byte[]> getTestResourceFiles() {
         return testResourceFiles;

backend/src/main/java/io/metersphere/performance/engine/EngineFactory.java

@@ -117,6 +117,7 @@ public class EngineFactory {
         engineContext.setResourcePoolId(loadTest.getTestResourcePoolId());
         engineContext.setReportId(reportId);
         engineContext.setResourceIndex(resourceIndex);
+        engineContext.setRatios(ratios);
 
         if (StringUtils.isNotEmpty(loadTest.getLoadConfiguration())) {
             final JSONArray jsonArray = JSONObject.parseArray(loadTest.getLoadConfiguration());
@@ -166,6 +167,15 @@ public class EngineFactory {
             MSException.throwException("File type unknown");
         }
 
+        if (CollectionUtils.isNotEmpty(resourceFiles)) {
+            Map<String, byte[]> data = new HashMap<>();
+            resourceFiles.forEach(cf -> {
+                FileContent csvContent = fileService.getFileContent(cf.getId());
+                data.put(cf.getName(), csvContent.getFile());
+            });
+            engineContext.setTestResourceFiles(data);
+        }
+
         try (ByteArrayInputStream source = new ByteArrayInputStream(jmxBytes)) {
             String content = engineSourceParser.parse(engineContext, source);
             engineContext.setContent(content);
@@ -177,15 +187,6 @@ public class EngineFactory {
             MSException.throwException(e);
         }
 
-        if (CollectionUtils.isNotEmpty(resourceFiles)) {
-            Map<String, byte[]> data = new HashMap<>();
-            resourceFiles.forEach(cf -> {
-                FileContent csvContent = fileService.getFileContent(cf.getId());
-                data.put(cf.getName(), csvContent.getFile());
-            });
-            engineContext.setTestResourceFiles(data);
-        }
-
         return engineContext;
     }
 

backend/src/main/java/io/metersphere/performance/parse/xml/reader/jmx/JmeterDocumentParser.java

@@ -4,9 +4,9 @@ import com.alibaba.fastjson.JSON;
 import com.alibaba.fastjson.JSONObject;
 import io.metersphere.commons.exception.MSException;
 import io.metersphere.commons.utils.CommonBeanFactory;
-import io.metersphere.jmeter.utils.ScriptEngineUtils;
 import io.metersphere.config.KafkaProperties;
 import io.metersphere.i18n.Translator;
+import io.metersphere.jmeter.utils.ScriptEngineUtils;
 import io.metersphere.performance.engine.EngineContext;
 import io.metersphere.performance.parse.xml.reader.DocumentParser;
 import org.apache.commons.lang3.BooleanUtils;
@@ -22,8 +22,10 @@ import javax.xml.transform.TransformerFactory;
 import javax.xml.transform.dom.DOMSource;
 import javax.xml.transform.stream.StreamResult;
 import java.io.StringWriter;
+import java.nio.charset.StandardCharsets;
 import java.util.HashSet;
 import java.util.List;
+import java.util.StringTokenizer;
 
 public class JmeterDocumentParser implements DocumentParser {
     private final static String HASH_TREE_ELEMENT = "hashTree";
@@ -267,10 +269,73 @@ public class JmeterDocumentParser implements DocumentParser {
                 if (StringUtils.equals(filenameTag, "filename")) {
                     // 截取文件名
                     handleFilename(item);
+                    // 切割CSV文件
+                    splitCsvFile(item);
+                    break;
+                }
+            }
+        }
+    }
+
+    private void splitCsvFile(Node item) {
+        Object csvConfig = context.getProperty("csvConfig");
+        if (csvConfig == null) {
+            return;
+        }
+        double[] ratios = context.getRatios();
+        int resourceIndex = context.getResourceIndex();
+        String filename = item.getTextContent();
+        byte[] content = context.getTestResourceFiles().get(filename);
+        StringTokenizer tokenizer = new StringTokenizer(new String(content), "\n");
+        if (!tokenizer.hasMoreTokens()) {
+            return;
+        }
+        StringBuilder csv = new StringBuilder();
+        Object config = ((JSONObject) csvConfig).get(filename);
+        boolean csvSplit = ((JSONObject) (config)).getBooleanValue("csvSplit");
+        if (!csvSplit) {
+            return;
+        }
+        boolean csvHasHeader = ((JSONObject) (config)).getBooleanValue("csvHasHeader");
+        if (csvHasHeader) {
+            String header = tokenizer.nextToken();
+            csv.append(header).append("\n");
+        }
+        int count = tokenizer.countTokens();
+
+        long current, offset = 0;
+
+        // 计算偏移量
+        for (int k = 0; k < resourceIndex; k++) {
+            offset += Math.round(count * ratios[k]);
+        }
+
+        if (resourceIndex + 1 == ratios.length) {
+            current = count - offset; // 最后一个点可以分到的数量
+        } else {
+            current = Math.round(count * ratios[resourceIndex]); // 当前节点可以分到的数量
+        }
+
+        long index = 0;
+        while (tokenizer.hasMoreTokens()) {
+            if (current == 0) { // 节点一个都没有分到，把所有的数据都给这个节点（极端情况）
+                String line = tokenizer.nextToken();
+                csv.append(line).append("\n");
+            } else {
+                if (index < offset) {
+                    index++;
+                    continue;
+                }
+                if (index > current + offset) {
                     break;
                 }
+                String line = tokenizer.nextToken();
+                csv.append(line).append("\n");
             }
+            index++;
         }
+        // 替换文件
+        context.getTestResourceFiles().put(filename, csv.toString().getBytes(StandardCharsets.UTF_8));
     }
 
     private void processResponseAssertion(Element element) {

frontend/src/business/components/performance/test/EditPerformanceTest.vue

@@ -316,6 +316,7 @@ export default {
     fileChange(threadGroups) {
       let handler = this.$refs.pressureConfig;
 
+      let csvSet = new Set;
       threadGroups.forEach(tg => {
         tg.threadNumber = tg.threadNumber || 10;
         tg.duration = tg.duration || 10;
@@ -325,12 +326,21 @@ export default {
         tg.threadType = tg.threadType || 'DURATION';
         tg.iterateNum = tg.iterateNum || 1;
         tg.iterateRampUp = tg.iterateRampUp || 10;
+
+        if (tg.csvFiles) {
+          tg.csvFiles.map(item => csvSet.add(item));
+        }
       });
+      let csvFiles = [];
+      for (const f of csvSet) {
+        csvFiles.push({name: f, csvSplit: false, csvHasHeader: true});
+      }
 
       this.$set(handler, "threadGroups", threadGroups);
 
       this.$refs.basicConfig.threadGroups = threadGroups;
       this.$refs.pressureConfig.threadGroups = threadGroups;
+      this.$refs.advancedConfig.csvFiles = csvFiles;
 
       handler.calculateTotalChart();
     },

frontend/src/business/components/performance/test/components/PerformanceAdvancedConfig.vue

 <template>
   <div>
+    <!--  基本配置  -->
+    <el-row>
+      <el-col :span="6">
+        <el-form :inline="true">
+          <el-form-item>
+            <div>{{ $t('load_test.connect_timeout') }}</div>
+          </el-form-item>
+          <el-form-item>
+            <el-input-number
+              :disabled="readOnly" size="mini" v-model="timeout"
+              :min="0"/>
+          </el-form-item>
+          <el-form-item>
+            ms
+          </el-form-item>
+        </el-form>
+      </el-col>
+      <el-col :span="6">
+        <el-form :inline="true">
+          <el-form-item>
+            <div>{{ $t('load_test.response_timeout') }}</div>
+          </el-form-item>
+          <el-form-item>
+            <el-input-number
+              :disabled="readOnly" size="mini" :min="0"
+              v-model="responseTimeout"/>
+          </el-form-item>
+          <el-form-item>
+            ms
+          </el-form-item>
+        </el-form>
+      </el-col>
+      <el-col :span="6">
+        <el-form :inline="true">
+          <el-form-item>
+            <div>
+              {{ $t('load_test.granularity') }}
+              <el-popover
+                placement="left"
+                width="300"
+                trigger="hover">
+                <el-table :data="granularityData">
+                  <el-table-column property="start" :label="$t('load_test.duration')">
+                    <template v-slot:default="scope">
+                      <span>{{ scope.row.start }}S - {{ scope.row.end }}S</span>
+                    </template>
+                  </el-table-column>
+                  <el-table-column property="granularity" :label="$t('load_test.granularity')"/>
+                </el-table>
+                <i slot="reference" class="el-icon-info pointer"/>
+              </el-popover>
+            </div>
+          </el-form-item>
+          <el-form-item>
+            <el-select v-model="granularity" :placeholder="$t('commons.please_select')" size="mini"
+                       clearable>
+              <el-option v-for="op in granularityData" :key="op.granularity" :label="op.granularity"
+                         :value="op.granularity"></el-option>
+            </el-select>
+          </el-form-item>
+        </el-form>
+      </el-col>
+      <el-col :span="6">
+        <el-form :inline="true">
+          <el-form-item>
+            <div>{{ $t('load_test.custom_http_code') }}</div>
+          </el-form-item>
+          <el-form-item>
+            <el-input
+              :disabled="readOnly" size="mini" v-model="statusCodeStr"
+              :placeholder="$t('load_test.separated_by_commas')"
+              @input="checkStatusCode"></el-input>
+          </el-form-item>
+        </el-form>
+      </el-col>
+    </el-row>
+    <!--  csv 配置  -->
+    <el-row>
+      <el-col :span="8">
+        <h3>CSVDataSet</h3>
+      </el-col>
+    </el-row>
+    <el-row>
+      <el-col :span="24">
+        <el-table :data="csvFiles" size="mini" class="tb-edit" align="center" border highlight-current-row>
+          <el-table-column
+            align="center"
+            prop="name"
+            :label="$t('commons.name')">
+          </el-table-column>
+          <el-table-column align="center" prop="csvSplit" :label="$t('load_test.csv_split')">
+            <template v-slot:default="{row}">
+              <el-switch :disabled="readOnly" v-model="row.csvSplit"/>
+            </template>
+          </el-table-column>
+          <el-table-column align="center" prop="csvHasHeader" :label="$t('load_test.csv_has_header')">
+            <template v-slot:default="{row}">
+              <el-switch :disabled="readOnly || !row.csvSplit" v-model="row.csvHasHeader"/>
+            </template>
+          </el-table-column>
+        </el-table>
+      </el-col>
+    </el-row>
+
+    <!-- 参数列表 -->
     <el-row>
       <el-col :span="8">
         <h3>{{ $t('load_test.params') }}</h3>
@@ -74,82 +179,6 @@
       </el-col>
     </el-row>
 
-    <el-row>
-      <el-col :span="8">
-        <el-form :inline="true">
-          <el-form-item>
-            <div>{{ $t('load_test.connect_timeout') }}</div>
-          </el-form-item>
-          <el-form-item>
-            <el-input-number :disabled="readOnly" size="mini" v-model="timeout"
-                             :min="0"></el-input-number>
-          </el-form-item>
-          <el-form-item>
-            ms
-          </el-form-item>
-        </el-form>
-      </el-col>
-      <el-col :span="8">
-        <el-form :inline="true">
-          <el-form-item>
-            <div>{{ $t('load_test.response_timeout') }}</div>
-          </el-form-item>
-          <el-form-item>
-            <el-input-number :disabled="readOnly" size="mini" :min="0"
-                             v-model="responseTimeout"></el-input-number>
-          </el-form-item>
-          <el-form-item>
-            ms
-          </el-form-item>
-        </el-form>
-      </el-col>
-    </el-row>
-    <el-row>
-      <el-col :span="8">
-        <el-form :inline="true">
-          <el-form-item>
-            <div>{{ $t('load_test.custom_http_code') }}</div>
-          </el-form-item>
-          <el-form-item>
-            <el-input :disabled="readOnly" size="mini" v-model="statusCodeStr"
-                      :placeholder="$t('load_test.separated_by_commas')"
-                      @input="checkStatusCode"></el-input>
-          </el-form-item>
-        </el-form>
-      </el-col>
-    </el-row>
-    <el-row>
-      <el-col :span="8">
-        <el-form :inline="true">
-          <el-form-item>
-            <div>
-              {{ $t('load_test.granularity') }}
-              <el-popover
-                placement="bottom"
-                width="400"
-                trigger="hover">
-                <el-table :data="granularityData">
-                  <el-table-column property="start" :label="$t('load_test.duration')">
-                    <template v-slot:default="scope">
-                      <span>{{ scope.row.start }}S - {{ scope.row.end }}S</span>
-                    </template>
-                  </el-table-column>
-                  <el-table-column property="granularity" :label="$t('load_test.granularity')"/>
-                </el-table>
-                <i slot="reference" class="el-icon-info pointer"/>
-              </el-popover>
-            </div>
-          </el-form-item>
-          <el-form-item>
-            <el-select v-model="granularity" :placeholder="$t('commons.please_select')" size="mini" clearable>
-              <el-option v-for="op in granularityData" :key="op.granularity" :label="op.granularity"
-                         :value="op.granularity"></el-option>
-            </el-select>
-          </el-form-item>
-        </el-form>
-      </el-col>
-    </el-row>
-
     <el-row>
       <el-col :span="8">
         <h3>监控集成</h3>
@@ -158,54 +187,57 @@
         </el-button>
       </el-col>
     </el-row>
-    <el-col :span="24">
-      <el-table :data="monitorParams" size="mini" class="tb-edit" align="center" border highlight-current-row>
-        <el-table-column
-          align="center"
-          prop="name"
-          label="名称">
-        </el-table-column>
-        <!--        <el-table-column-->
-        <!--          align="center"-->
-        <!--          prop="environmentName"-->
-        <!--          label="所属环境">-->
-        <!--        </el-table-column>-->
-        <!--        <el-table-column-->
-        <!--          align="center"-->
-        <!--          prop="authStatus"-->
-        <!--          label="认证状态">-->
-        <!--        </el-table-column>-->
-        <!--        <el-table-column-->
-        <!--          align="center"-->
-        <!--          prop="monitorStatus"-->
-        <!--          label="监控状态">-->
-        <el-table-column
-          align="center"
-          prop="ip"
-          label="IP">
-        </el-table-column>
-        <el-table-column
-          align="center"
-          prop="port"
-          label="Port">
-        </el-table-column>
-        <el-table-column
-          align="center"
-          prop="description"
-          label="描述">
-        </el-table-column>
-        <el-table-column align="center" :label="$t('load_test.operating')">
-          <template v-slot:default="{row, $index}">
-            <ms-table-operator-button :disabled="readOnly" tip="编辑" icon="el-icon-edit"
-                                      type="primary"
-                                      @exec="modifyMonitor(row, $index)"/>
-            <ms-table-operator-button :disabled="readOnly" :tip="$t('commons.delete')" icon="el-icon-delete"
-                                      type="danger"
-                                      @exec="delMonitor(row, $index)"/>
-          </template>
-        </el-table-column>
-      </el-table>
-    </el-col>
+    <el-row>
+
+      <el-col :span="24">
+        <el-table :data="monitorParams" size="mini" class="tb-edit" align="center" border highlight-current-row>
+          <el-table-column
+            align="center"
+            prop="name"
+            label="名称">
+          </el-table-column>
+          <!--        <el-table-column-->
+          <!--          align="center"-->
+          <!--          prop="environmentName"-->
+          <!--          label="所属环境">-->
+          <!--        </el-table-column>-->
+          <!--        <el-table-column-->
+          <!--          align="center"-->
+          <!--          prop="authStatus"-->
+          <!--          label="认证状态">-->
+          <!--        </el-table-column>-->
+          <!--        <el-table-column-->
+          <!--          align="center"-->
+          <!--          prop="monitorStatus"-->
+          <!--          label="监控状态">-->
+          <el-table-column
+            align="center"
+            prop="ip"
+            label="IP">
+          </el-table-column>
+          <el-table-column
+            align="center"
+            prop="port"
+            label="Port">
+          </el-table-column>
+          <el-table-column
+            align="center"
+            prop="description"
+            label="描述">
+          </el-table-column>
+          <el-table-column align="center" :label="$t('load_test.operating')">
+            <template v-slot:default="{row, $index}">
+              <ms-table-operator-button :disabled="readOnly" tip="编辑" icon="el-icon-edit"
+                                        type="primary"
+                                        @exec="modifyMonitor(row, $index)"/>
+              <ms-table-operator-button :disabled="readOnly" :tip="$t('commons.delete')" icon="el-icon-delete"
+                                        type="danger"
+                                        @exec="delMonitor(row, $index)"/>
+            </template>
+          </el-table-column>
+        </el-table>
+      </el-col>
+    </el-row>
 
     <edit-monitor ref="monitorDialog" :testId="testId" :list.sync="monitorParams"/>
   </div>
@@ -227,6 +259,8 @@ export default {
       domains: [],
       params: [],
       monitorParams: [],
+      csvFiles: [],
+      csvConfig: [],
       statusCodeStr: '',
       granularity: undefined,
       granularityData: [
@@ -257,6 +291,14 @@ export default {
       if (this.testId) {
         this.getAdvancedConfig();
       }
+    },
+    csvFiles() {
+      if (this.csvConfig && this.csvFiles) {
+        this.csvFiles.forEach(f => {
+          f.csvSplit = this.csvConfig[f.name].csvSplit;
+          f.csvHasHeader = this.csvConfig[f.name].csvHasHeader;
+        });
+      }
     }
   },
   methods: {
@@ -272,6 +314,7 @@ export default {
           this.params = data.params || [];
           this.granularity = data.granularity;
           this.monitorParams = data.monitorParams || [];
+          this.csvConfig = data.csvConfig;
         }
       });
     },
@@ -357,6 +400,10 @@ export default {
         responseTimeout: this.responseTimeout,
         statusCode: statusCode,
         params: this.params,
+        csvConfig: this.csvFiles.reduce((result, curr) => {
+          result[curr.name] = {csvHasHeader: curr.csvHasHeader, csvSplit: curr.csvSplit};
+          return result;
+        }, {}),
         domains: this.domains,
         granularity: this.granularity,
         monitorParams: this.monitorParams

frontend/src/i18n/en-US.js

@@ -649,6 +649,8 @@ export default {
     load_api_automation_jmx: 'Import API automation scenario',
     project_file_exist: "The file already exists in the project, please import it directly",
     project_file_update_type_error: 'Updated file types must be consistent',
+    csv_has_header: 'Contains Title',
+    csv_split: 'CSV Split',
     report: {
       diff: "Compare",
       set_default: 'Set to Default',

frontend/src/i18n/zh-CN.js

@@ -617,9 +617,9 @@ export default {
     param_is_duplicate: '参数名不能重复',
     domain_ip_is_empty: '域名和IP不能为空',
     param_name_value_is_empty: '参数名和参数值不能为空',
-    connect_timeout: '建立连接超时时间',
-    response_timeout: '响应超时时间',
-    custom_http_code: '自定义 HTTP 响应成功状态码',
+    connect_timeout: '连接超时',
+    response_timeout: '响应超时',
+    custom_http_code: '自定义响应码',
     separated_by_commas: '按逗号分隔',
     create: '创建测试',
     run: '一键运行',
@@ -647,6 +647,8 @@ export default {
     threadgroup_at_least_one: '至少启用一个线程组',
     load_api_automation_jmx: '引用接口自动化场景',
     project_file_exist: "项目中已存在该文件，请直接引用",
+    csv_has_header: '包含表头',
+    csv_split: 'CSV分割',
     report: {
       diff: "对比",
       set_default: '恢复默认',

frontend/src/i18n/zh-TW.js

@@ -617,9 +617,9 @@ export default {
     param_is_duplicate: '參數名不能重復',
     domain_ip_is_empty: '域名和IP不能為空',
     param_name_value_is_empty: '參數名和參數值不能為空',
-    connect_timeout: '建立連接超時時間',
-    response_timeout: '響應超時時間',
-    custom_http_code: '自定義 HTTP 響應成功狀態碼',
+    connect_timeout: '連接超時',
+    response_timeout: '響應超時',
+    custom_http_code: '自定義響應碼',
     separated_by_commas: '按逗號分隔',
     create: '創建測試',
     run: '一鍵運行',
@@ -647,6 +647,8 @@ export default {
     threadgroup_at_least_one: '至少啟用一個線程組',
     load_api_automation_jmx: '引用接口自動化場景',
     project_file_exist: "項目中已存在該文件，請直接引用",
+    csv_has_header: '包含表头',
+    csv_split: 'CSV分割',
     report: {
       diff: "對比",
       set_default: '恢復默認',