|
|
@@ -1,32 +1,29 @@
|
|
|
package com.acquisition.service.DataAcquisition;
|
|
|
|
|
|
import com.acquisition.config.KeyTableFile;
|
|
|
-import com.acquisition.config.ReadConfigExcel;
|
|
|
-import org.apache.poi.ss.usermodel.Cell;
|
|
|
-import org.apache.poi.ss.usermodel.Row;
|
|
|
-import org.apache.poi.ss.usermodel.Sheet;
|
|
|
-import org.apache.poi.ss.usermodel.Workbook;
|
|
|
-import org.apache.poi.xssf.usermodel.XSSFWorkbook;
|
|
|
+import com.github.pjfanning.xlsx.StreamingReader;
|
|
|
+import org.apache.poi.ss.usermodel.*;
|
|
|
+import org.apache.poi.xssf.streaming.SXSSFWorkbook;
|
|
|
+import org.slf4j.Logger;
|
|
|
+import org.slf4j.LoggerFactory;
|
|
|
import org.springframework.stereotype.Service;
|
|
|
|
|
|
import javax.annotation.Resource;
|
|
|
-import java.io.File;
|
|
|
-import java.io.FileInputStream;
|
|
|
-import java.io.FileOutputStream;
|
|
|
+import java.io.*;
|
|
|
import java.util.ArrayList;
|
|
|
+import java.util.Arrays;
|
|
|
import java.util.List;
|
|
|
|
|
|
@Service
|
|
|
public class FilesdataCopy {
|
|
|
+
|
|
|
+ private static final Logger log = LoggerFactory.getLogger(FilesdataCopy.class);
|
|
|
+
|
|
|
@Resource
|
|
|
private KeyTableFile keyTableFile;
|
|
|
- @Resource
|
|
|
- private ReadConfigExcel readConfigExcel;
|
|
|
-
|
|
|
|
|
|
private final String collectionServiceDir;
|
|
|
|
|
|
-
|
|
|
public FilesdataCopy() {
|
|
|
this.collectionServiceDir = System.getProperty("user.home") + "/Desktop/CollectionService";
|
|
|
}
|
|
|
@@ -35,111 +32,215 @@ public class FilesdataCopy {
|
|
|
String inputPath = collectionServiceDir + "/demo/配置总表.xlsx";
|
|
|
String outputPath = collectionServiceDir + "/104转发协议测点.xlsx";
|
|
|
|
|
|
- FileInputStream fis = new FileInputStream(new File(inputPath));
|
|
|
- Workbook inputWorkbook = new XSSFWorkbook(fis);
|
|
|
- Sheet inputSheet = inputWorkbook.getSheetAt(0);
|
|
|
+ // 列索引（0-based）：A=0, T=19
|
|
|
+ final int nameCol = 0; // 名称列
|
|
|
+ final int mongoKeyCol = 19; // mongoKey列（T列）
|
|
|
|
|
|
- Workbook outputWorkbook = new XSSFWorkbook();
|
|
|
- Sheet outputSheet = outputWorkbook.createSheet("Sheet1");
|
|
|
+ // 计数与地址分配
|
|
|
+ int id = 1;
|
|
|
+ int ai104Addr = 16385, di104Addr = 1;
|
|
|
+ int aiPublicAddr = 1, diPublicAddr = 1;
|
|
|
|
|
|
- // 写入表头
|
|
|
-// String[] headers = {"id", "测点", "测点类型", "公共地址", "104地址", "系数", "场站简称", "测点描述"};
|
|
|
- String[] headers = {"id", "point", "pointtype", "publicaddr", "pointaddr", "coef", "staionid", "org"};
|
|
|
- Row headerRow = outputSheet.createRow(0);
|
|
|
- for (int i = 0; i < headers.length; i++) {
|
|
|
- headerRow.createCell(i).setCellValue(headers[i]);
|
|
|
- }
|
|
|
+ // 进度统计
|
|
|
+ long rowsScanned = 0;
|
|
|
+ long rowsWritten = 0;
|
|
|
+
|
|
|
+ // 使用 DataFormatter 更稳健地读取各种类型的单元格
|
|
|
+ DataFormatter formatter = new DataFormatter(false);
|
|
|
|
|
|
- List<String> mongoKeys = new ArrayList<>();
|
|
|
- List<String> names = new ArrayList<>();
|
|
|
+ // 读 + 写 都放到 try-with-resources，确保真实落盘
|
|
|
+ try (InputStream fis = new FileInputStream(new File(inputPath));
|
|
|
+ Workbook inputWorkbook = StreamingReader.builder()
|
|
|
+ .rowCacheSize(100) // 缓存100行
|
|
|
+ .bufferSize(4096) // 4K缓冲
|
|
|
+ .open(fis);
|
|
|
+ SXSSFWorkbook outputWorkbook = new SXSSFWorkbook(100); // 写时内存保留100行
|
|
|
+ FileOutputStream fos = new FileOutputStream(outputPath)) {
|
|
|
|
|
|
- int mongoKeyCol = 19; // T列 (索引从0开始)
|
|
|
- int nameCol = 0; // 名称列 assumed to be 第1列
|
|
|
+ outputWorkbook.setCompressTempFiles(true);
|
|
|
|
|
|
- for (int i = 1; i <= inputSheet.getLastRowNum(); i++) {
|
|
|
- Row row = inputSheet.getRow(i);
|
|
|
- if (row == null) continue;
|
|
|
+ Sheet inputSheet = inputWorkbook.getSheetAt(0);
|
|
|
+ Sheet outputSheet = outputWorkbook.createSheet("Sheet1");
|
|
|
|
|
|
- Cell mongoKeyCell = row.getCell(mongoKeyCol);
|
|
|
- Cell nameCell = row.getCell(nameCol);
|
|
|
+ // 表头
|
|
|
+ String[] headers = {"id", "point", "pointtype", "publicaddr", "pointaddr", "coef", "staionid", "org"};
|
|
|
+ Row headerRow = outputSheet.createRow(0);
|
|
|
+ for (int i = 0; i < headers.length; i++) {
|
|
|
+ headerRow.createCell(i).setCellValue(headers[i]);
|
|
|
+ }
|
|
|
|
|
|
- if (mongoKeyCell == null || nameCell == null) continue;
|
|
|
+ // ⚠️ 不要依赖 row.getRowNum()（在流式模式下可能异常/不准确）
|
|
|
+ boolean isFirstRow = true;
|
|
|
+
|
|
|
+ for (Row row : inputSheet) {
|
|
|
+ rowsScanned++;
|
|
|
+
|
|
|
+ // 跳过表头（首行）
|
|
|
+ if (isFirstRow) {
|
|
|
+ isFirstRow = false;
|
|
|
+ // 可打印一下首行包含哪些列，便于核对列号是否正确
|
|
|
+ if (log.isInfoEnabled()) {
|
|
|
+ StringBuilder cols = new StringBuilder();
|
|
|
+ for (Cell c : row) {
|
|
|
+ cols.append(c.getColumnIndex()).append(",");
|
|
|
+ }
|
|
|
+ log.info("[Header] 首行包含的列索引: {}", cols.toString());
|
|
|
+ }
|
|
|
+ continue;
|
|
|
+ }
|
|
|
|
|
|
- mongoKeys.add(mongoKeyCell.toString().trim());
|
|
|
- names.add(nameCell.toString().trim());
|
|
|
- }
|
|
|
+ // —— 稳健读取：遍历该行出现过的单元格，用列号挑出需要的两个值 ——
|
|
|
+ String mongoKey = "";
|
|
|
+ String name = "";
|
|
|
|
|
|
- int id = 1;
|
|
|
- int ai104Addr = 16385, di104Addr = 1;
|
|
|
- int aiPublicAddr = 1, diPublicAddr = 1;
|
|
|
- int aiCount = 0, diCount = 0;
|
|
|
-
|
|
|
- for (int i = 0; i < mongoKeys.size(); i++) {
|
|
|
- String mongoKey = mongoKeys.get(i);
|
|
|
- String name = names.get(i);
|
|
|
-
|
|
|
- String pointType = "";
|
|
|
- int lastUnderscoreIndex = mongoKey.lastIndexOf('_');
|
|
|
- if (lastUnderscoreIndex != -1 && lastUnderscoreIndex < mongoKey.length() - 1) {
|
|
|
- String suffix = mongoKey.substring(lastUnderscoreIndex + 1).toUpperCase();
|
|
|
- if (suffix.startsWith("AI") || suffix.startsWith("PI")) {
|
|
|
- pointType = "AI";
|
|
|
- } else if (suffix.startsWith("DI")) {
|
|
|
- pointType = "DI";
|
|
|
+ // 记录该行出现的列（排障用，前若干行打印）
|
|
|
+ StringBuilder seenCols = null;
|
|
|
+ if (rowsScanned <= 20) { // 仅前20行打印，避免刷屏
|
|
|
+ seenCols = new StringBuilder();
|
|
|
}
|
|
|
- }
|
|
|
|
|
|
+ for (Cell cell : row) {
|
|
|
+ int colIdx = cell.getColumnIndex();
|
|
|
+ String val = formatter.formatCellValue(cell);
|
|
|
+ if (colIdx == mongoKeyCol) {
|
|
|
+ mongoKey = val == null ? "" : val.trim();
|
|
|
+ } else if (colIdx == nameCol) {
|
|
|
+ name = val == null ? "" : val.trim();
|
|
|
+ }
|
|
|
+ if (seenCols != null) {
|
|
|
+ seenCols.append(colIdx).append("|");
|
|
|
+ }
|
|
|
+ }
|
|
|
|
|
|
- String pointTypeUpper = pointType.toUpperCase();
|
|
|
+ if (rowsScanned <= 20 && seenCols != null) {
|
|
|
+ log.info("第{}行（从1开始计数）出现的列索引: {}", rowsScanned + 0, seenCols.toString());
|
|
|
+ }
|
|
|
|
|
|
- int current104Addr;
|
|
|
- int currentPublicAddr;
|
|
|
+ // 该行没有目标列或 mongoKey 空，直接跳过
|
|
|
+ if (mongoKey.isEmpty()) {
|
|
|
+ // 仅前20行给出详细提示，帮助校验列号是否正确
|
|
|
+ if (rowsScanned <= 20) {
|
|
|
+ log.warn("第{}行未读取到 mongoKey（目标列索引 {}）。请确认 Excel 的 T 列是否为 mongoKey。", rowsScanned + 0, mongoKeyCol);
|
|
|
+ }
|
|
|
+ continue;
|
|
|
+ }
|
|
|
|
|
|
- if ("AI".equals(pointTypeUpper)) {
|
|
|
- current104Addr = ai104Addr;
|
|
|
- currentPublicAddr = aiPublicAddr;
|
|
|
- ai104Addr++;
|
|
|
- aiCount++;
|
|
|
-// if (ai104Addr > 24384) {
|
|
|
- if (ai104Addr > 20480) {
|
|
|
- ai104Addr = 16385;
|
|
|
- aiPublicAddr++;
|
|
|
+ // 解析类型
|
|
|
+ String pointType = "";
|
|
|
+ int lastUnderscoreIndex = mongoKey.lastIndexOf('_');
|
|
|
+ if (lastUnderscoreIndex != -1 && lastUnderscoreIndex < mongoKey.length() - 1) {
|
|
|
+ String suffix = mongoKey.substring(lastUnderscoreIndex + 1).toUpperCase();
|
|
|
+ if (suffix.startsWith("AI") || suffix.startsWith("PI")) {
|
|
|
+ pointType = "AI";
|
|
|
+ } else if (suffix.startsWith("DI")) {
|
|
|
+ pointType = "DI";
|
|
|
+ }
|
|
|
}
|
|
|
- } else if ("DI".equals(pointTypeUpper)) {
|
|
|
- current104Addr = di104Addr;
|
|
|
- currentPublicAddr = diPublicAddr;
|
|
|
- di104Addr++;
|
|
|
- diCount++;
|
|
|
-// if (di104Addr > 16000) {
|
|
|
- if (di104Addr > 16384) {
|
|
|
- di104Addr = 1;
|
|
|
- diPublicAddr++;
|
|
|
+ String pointTypeUpper = pointType.toUpperCase();
|
|
|
+ if (!"AI".equals(pointTypeUpper) && !"DI".equals(pointTypeUpper)) {
|
|
|
+ // 非AI/DI，跳过
|
|
|
+ continue;
|
|
|
+ }
|
|
|
+
|
|
|
+ int current104Addr;
|
|
|
+ int currentPublicAddr;
|
|
|
+
|
|
|
+ if ("AI".equals(pointTypeUpper)) {
|
|
|
+ current104Addr = ai104Addr;
|
|
|
+ currentPublicAddr = aiPublicAddr;
|
|
|
+ ai104Addr++;
|
|
|
+// if (ai104Addr > 20384) {
|
|
|
+ if (ai104Addr > 20480) {
|
|
|
+ ai104Addr = 16385;
|
|
|
+ aiPublicAddr++;
|
|
|
+ }
|
|
|
+ } else { // DI
|
|
|
+ current104Addr = di104Addr;
|
|
|
+ currentPublicAddr = diPublicAddr;
|
|
|
+ di104Addr++;
|
|
|
+// if (di104Addr > 16000) {
|
|
|
+ if (di104Addr > 16384) {
|
|
|
+ di104Addr = 1;
|
|
|
+ diPublicAddr++;
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ // 写出一行
|
|
|
+ Row outputRow = outputSheet.createRow(id);
|
|
|
+ outputRow.createCell(0).setCellValue(id);
|
|
|
+ outputRow.createCell(1).setCellValue(mongoKey);
|
|
|
+ outputRow.createCell(2).setCellValue(pointTypeUpper);
|
|
|
+ outputRow.createCell(3).setCellValue(currentPublicAddr);
|
|
|
+ outputRow.createCell(4).setCellValue(current104Addr);
|
|
|
+ outputRow.createCell(5).setCellValue(1);
|
|
|
+ outputRow.createCell(6).setCellValue("GYEE");
|
|
|
+ outputRow.createCell(7).setCellValue(name);
|
|
|
+
|
|
|
+ id++;
|
|
|
+ rowsWritten++;
|
|
|
+
|
|
|
+ // 进度日志：每1000行输入/每1000行输出打印一次
|
|
|
+ if (rowsScanned % 1000 == 0) {
|
|
|
+ log.info("已扫描输入行数：{}", rowsScanned);
|
|
|
+ }
|
|
|
+ if (rowsWritten % 1000 == 0) {
|
|
|
+ log.info("已写出有效数据行数：{}", rowsWritten);
|
|
|
}
|
|
|
- } else {
|
|
|
- continue; // 跳过无法识别类型的行
|
|
|
}
|
|
|
|
|
|
- Row outputRow = outputSheet.createRow(id);
|
|
|
- outputRow.createCell(0).setCellValue(id); // id
|
|
|
- outputRow.createCell(1).setCellValue(mongoKey); // 测点
|
|
|
- outputRow.createCell(2).setCellValue(pointTypeUpper); // 测点类型
|
|
|
- outputRow.createCell(3).setCellValue(currentPublicAddr); // 公共地址
|
|
|
- outputRow.createCell(4).setCellValue(current104Addr); // 104地址
|
|
|
- outputRow.createCell(5).setCellValue(1); // 系数
|
|
|
- outputRow.createCell(6).setCellValue("GYEE"); // 分公司简称
|
|
|
- outputRow.createCell(7).setCellValue(name); // 测点描述
|
|
|
-
|
|
|
- id++;
|
|
|
+ // 真正落盘
|
|
|
+ outputWorkbook.write(fos);
|
|
|
+ fos.flush();
|
|
|
+
|
|
|
+ log.info("处理完成：扫描输入行数={}, 写出有效数据行数={}", rowsScanned, rowsWritten);
|
|
|
+ if (rowsWritten == 0) {
|
|
|
+ log.warn("警告：没有任何数据被写出。请检查：1) mongoKey 是否确实在 T 列(索引19)；2) mongoKey 格式是否以 _AI/_PI/_DI 结尾；3) 第一张表是否有数据。");
|
|
|
+ }
|
|
|
+ }
|
|
|
+ // ====== 第二步：读取“104转发协议测点.xlsx”，准备数据并生成 sqlite ======
|
|
|
+ log.info("开始读取刚生成的 Excel：{}", outputPath);
|
|
|
+
|
|
|
+ // 关键：第一行必须是表头，供 generateKeyTableFile 使用
|
|
|
+ final List<String> HEADERS = Arrays.asList("id", "point", "pointtype", "publicaddr", "pointaddr", "coef", "staionid", "org");
|
|
|
+ List<List<Object>> configExcelData = new ArrayList<>();
|
|
|
+ configExcelData.add(new ArrayList<>(HEADERS)); // 表头放在第 0 行
|
|
|
+
|
|
|
+ long rowsReadForSqlite = 0;
|
|
|
+ DataFormatter fmt = new DataFormatter(false);
|
|
|
+
|
|
|
+ try (InputStream fis2 = new FileInputStream(new File(outputPath));
|
|
|
+ Workbook wb2 = StreamingReader.builder().rowCacheSize(200).bufferSize(8192).open(fis2)) {
|
|
|
+
|
|
|
+ Sheet sh = wb2.getSheetAt(0);
|
|
|
+ boolean isHeader = true;
|
|
|
+
|
|
|
+ for (Row row : sh) {
|
|
|
+ if (isHeader) {
|
|
|
+ // 这里不依赖文件中的表头内容，直接使用我们定义好的 HEADERS
|
|
|
+ isHeader = false;
|
|
|
+ continue;
|
|
|
+ }
|
|
|
+
|
|
|
+ // 固定按 0..7 列读取，避免“稀疏单元格”导致列错位
|
|
|
+ List<Object> rowData = new ArrayList<>(HEADERS.size());
|
|
|
+ for (int c = 0; c < HEADERS.size(); c++) {
|
|
|
+ Cell cell = row.getCell(c, Row.MissingCellPolicy.RETURN_BLANK_AS_NULL);
|
|
|
+ String val = (cell == null) ? "" : fmt.formatCellValue(cell);
|
|
|
+ rowData.add(val);
|
|
|
+ }
|
|
|
+ configExcelData.add(rowData);
|
|
|
+ rowsReadForSqlite++;
|
|
|
+
|
|
|
+ if (rowsReadForSqlite % 2000 == 0) {
|
|
|
+ log.info("读取用于 sqlite 的数据行数：{}", rowsReadForSqlite);
|
|
|
+ }
|
|
|
+ }
|
|
|
}
|
|
|
|
|
|
- FileOutputStream fos = new FileOutputStream(outputPath);
|
|
|
- outputWorkbook.write(fos);
|
|
|
- fos.close();
|
|
|
+ log.info("读取完成：用于 sqlite 的数据行数（不含表头）={}", rowsReadForSqlite);
|
|
|
|
|
|
- inputWorkbook.close();
|
|
|
- outputWorkbook.close();
|
|
|
-// 1. 读取配置总表.xlsx中的数据并生成sqlite3文件
|
|
|
- List<List<Object>> configExcelData = readConfigExcel.readConfigExcel(outputPath);
|
|
|
- keyTableFile.generateKeyTableFile(collectionServiceDir, configExcelData, 0, 7,2);
|
|
|
+ // 生成 sqlite（dbName=2 → 表名为 point）
|
|
|
+ keyTableFile.generateKeyTableFile(collectionServiceDir, configExcelData, 0, 7, 2);
|
|
|
+ log.info("成功生成 sqlite 文件（表名=point——正在写入数据。");
|
|
|
}
|
|
|
-}
|
|
|
+}
|
