暂时注同步数据测试

siwei-modules/siwei-apply/src/main/java/com/siwei/apply/service/cadastre/impl/CadastreManageServiceImpl.java

@@ -25,6 +25,8 @@ import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Service;
 import org.springframework.transaction.annotation.Propagation;
 import org.springframework.transaction.annotation.Transactional;
+import org.springframework.transaction.support.TransactionTemplate;
+import org.springframework.transaction.PlatformTransactionManager;
 
 import java.io.File;
 import java.nio.file.Files;
@@ -47,6 +49,9 @@ public class CadastreManageServiceImpl implements CadastreManageService {
     @Autowired
     private CadastreFileMapper cadastreFileMapper;
 
+    @Autowired
+    private PlatformTransactionManager transactionManager;
+
     @Autowired
     private FzssFxrwrzHandleService fzssFxrwrzHandleService;
 
@@ -148,13 +153,11 @@ public class CadastreManageServiceImpl implements CadastreManageService {
     }
 
 
-    @Transactional(rollbackFor = Exception.class)
     @Override
     public String addBatchMdbData(String uploadId) {
         String result = "success";
         try {
             CadastreFile cadastreFile = cadastreFileMapper.get(uploadId);
-            //这里判断是否校验通过
             if (!cadastreFile.getStatus().equalsIgnoreCase("3")) {
                 log.info("校验未通过，不能入库");
                 result = "校验未通过，不能入库";
@@ -175,15 +178,12 @@ public class CadastreManageServiceImpl implements CadastreManageService {
             fzssFxrwrzHandleService.insertFxrwrz(uploadId, "地籍库管理", "正在执行入库操作。。。", "info");
             for (Map<String, Object> tableMap : allMdbTableList) {
                 String talbleName = String.valueOf(tableMap.get("name"));
-                //List<Map<String, Object>> theValueList = getValueList(layer, dbFieldNameAndTypeMap, haveGeom);
                 Map<String, Object> allMdbTableFieldMap = MdbUtil.MdbTableContent(gdbPath + "/" + cadastreFile.getName(), talbleName, "", "", 1, 1000);
                 List<Map<String, Object>> theValueList = (List<Map<String, Object>>) allMdbTableFieldMap.get("rows");
 
                 fzssFxrwrzHandleService.insertFxrwrz(uploadId, "地籍库管理", "正在执行-" + talbleName + "-入库", "info");
-                //这里直接数据入库，区分增量或者全量
                 Boolean addRes;
                 if ("2".equalsIgnoreCase(cadastreFile.getReadStatus())) {
-                    //增量更新
                     addRes = incrementDataStorage(talbleName, theValueList);
                     if (addRes.equals(true)) {
                         fzssFxrwrzHandleService.insertFxrwrz(uploadId, "地籍库管理", "增量更新-" + talbleName + "-成功", "info");
@@ -196,22 +196,28 @@ public class CadastreManageServiceImpl implements CadastreManageService {
                 }
                 log.error("当前属性名{},入库结果 {}", talbleName, addRes);
             }
-            //成功则更新状态
             CadastreFile updateCadastreFile = new CadastreFile();
             updateCadastreFile.setId(cadastreFile.getId());
-            updateCadastreFile.setStatus("1");//入库成功
-            updateCadastreFile.setReadMessage("入库成功");//入库成功
+            updateCadastreFile.setStatus("1");
+            updateCadastreFile.setReadMessage("入库成功");
             cadastreFileMapper.update(updateCadastreFile);
             fzssFxrwrzHandleService.insertFxrwrz(uploadId, "地籍库管理", "当前批次-" + cadastreFile.getId() + "-入库成功", "info");
         } catch (Exception e) {
             log.error("mdb表格数据入库异常，uploadId: {}", uploadId, e);
-            //更新状态
-            CadastreFile updateCadastreFile = new CadastreFile();
-            updateCadastreFile.setId(uploadId);
-            updateCadastreFile.setStatus("2");//入库失败
-            updateCadastreFile.setReadMessage("入库失败");//入库失败
-            cadastreFileMapper.update(updateCadastreFile);
-            fzssFxrwrzHandleService.insertFxrwrz(uploadId, "地籍库管理", "当前批次-" + uploadId + "-入库失败", "info");
+            try {
+                TransactionTemplate txTemplate = new TransactionTemplate(transactionManager);
+                txTemplate.setPropagationBehavior(TransactionTemplate.PROPAGATION_REQUIRES_NEW);
+                txTemplate.execute(status -> {
+                    CadastreFile updateCadastreFile = new CadastreFile();
+                    updateCadastreFile.setId(uploadId);
+                    updateCadastreFile.setStatus("2");
+                    updateCadastreFile.setReadMessage("入库失败");
+                    cadastreFileMapper.update(updateCadastreFile);
+                    return null;
+                });
+            } catch (Exception ex) {
+                log.error("更新失败状态异常，uploadId: {}", uploadId, ex);
+            }
             throw new ServiceException("表格数据入库异常，" + e.getMessage());
         }
         return result;
@@ -940,20 +946,71 @@ public class CadastreManageServiceImpl implements CadastreManageService {
         boolean addRes = false;
         Integer shpDbSRID = 4326;
         if (!CollectionUtils.isEmpty(theValueList)) {
-            //先删除数据
+            List<Map<String, String>> dbFieldNameAndTypeList = cadastreFileMapper.selectTableCollumAndType(tableName.toLowerCase(Locale.ROOT));
+            Map<String, String> columnTypeMap = new LinkedHashMap<>();
+            for (Map<String, String> col : dbFieldNameAndTypeList) {
+                columnTypeMap.put(col.get("column_name"), col.get("data_type"));
+            }
             cadastreFileMapper.deleteByValidFlag("1", tableName);
-            //先把当前版本都修改为历史版本
             cadastreFileMapper.updateValidFlag("1", "0", tableName);
             for (Map<String, Object> map : theValueList) {
-//                Collection<Object> values = map.values();
-//                Collection<String> keys = map.keySet();
-                map.put("valid_flag", 0);//有效数据标识
+                map.put("valid_flag", 0);
+                cleanDataForInsert(map, columnTypeMap);
                 cadastreFileMapper.insertTableData(tableName, map, shpDbSRID);
             }
+            addRes = true;
         }
         return addRes;
     }
 
+    private void cleanDataForInsert(Map<String, Object> map, Map<String, String> columnTypeMap) {
+        Set<String> numericTypes = new HashSet<>(Arrays.asList(
+                "integer", "bigint", "smallint", "int2", "int4", "int8",
+                "numeric", "decimal", "real", "double precision", "float4", "float8"
+        ));
+        map.keySet().removeIf(key -> {
+            if ("valid_flag".equalsIgnoreCase(key) || "geom".equalsIgnoreCase(key)) {
+                return false;
+            }
+            Object val = map.get(key);
+            if (val == null) return true;
+            if (val instanceof String && ((String) val).trim().isEmpty()) {
+                String colType = columnTypeMap.get(key.toLowerCase(Locale.ROOT));
+                if (colType != null && numericTypes.contains(colType.toLowerCase(Locale.ROOT))) {
+                    return true;
+                }
+            }
+            return false;
+        });
+        map.replaceAll((k, v) -> {
+            if (v instanceof String) {
+                String sv = ((String) v).trim();
+                if (sv.isEmpty()) {
+                    return null;
+                }
+                String colType = columnTypeMap.get(k.toLowerCase(Locale.ROOT));
+                if (colType != null && numericTypes.contains(colType.toLowerCase(Locale.ROOT))) {
+                    try {
+                        if (colType.equalsIgnoreCase("integer") || colType.equalsIgnoreCase("int4")
+                                || colType.equalsIgnoreCase("smallint") || colType.equalsIgnoreCase("int2")
+                                || colType.equalsIgnoreCase("bigint") || colType.equalsIgnoreCase("int8")) {
+                            return Integer.parseInt(sv);
+                        } else if (colType.equalsIgnoreCase("real") || colType.equalsIgnoreCase("float4")) {
+                            return Float.parseFloat(sv);
+                        } else if (colType.equalsIgnoreCase("double precision") || colType.equalsIgnoreCase("float8")) {
+                            return Double.parseDouble(sv);
+                        } else {
+                            return new java.math.BigDecimal(sv);
+                        }
+                    } catch (NumberFormatException e) {
+                        return null;
+                    }
+                }
+            }
+            return v;
+        });
+    }
+
 
     /**
      * 判断两个Map中的数据是否完全一致（排除geom字段的比较方式，或根据业务需求处理）
@@ -1037,21 +1094,16 @@ public class CadastreManageServiceImpl implements CadastreManageService {
      */
     @Transactional(propagation = Propagation.REQUIRED, rollbackFor = Exception.class)
     public Boolean incrementDataStorage(String tableName, List<Map<String, Object>> theValueList) {
-        /**
-         * 1。查询出当前表所有数据。
-         * 2.修改当前数据的valid_flag为1，表示历史版本
-         * 3.把当前数据和数据库数据进行对比，获取新增数据和修改数据。
-         * 4.新增数据直接插入，修改数据根据主键进行更新。
-         * 任务标识码相同，并且geom数据有相交则认为数据相同。
-         */
         boolean addRes = false;
         Integer shpDbSRID = 4326;
         if (!CollectionUtils.isEmpty(theValueList)) {
-            //先删除数据
+            List<Map<String, String>> dbFieldNameAndTypeList = cadastreFileMapper.selectTableCollumAndType(tableName.toLowerCase(Locale.ROOT));
+            Map<String, String> columnTypeMap = new LinkedHashMap<>();
+            for (Map<String, String> col : dbFieldNameAndTypeList) {
+                columnTypeMap.put(col.get("column_name"), col.get("data_type"));
+            }
             cadastreFileMapper.deleteByValidFlag("1", tableName);
-            //查询出当前表所有数据。
             List<Map<String, Object>> dbDataList = cadastreFileMapper.selectTableData("0", tableName);
-            //筛选出需要更新的数据
             List<Map<String, Object>> dbExistsList = new ArrayList<>();
             for (Map<String, Object> map : theValueList) {
                 Map<String, Object> dbExistsMap = cadastreFileMapper.selectExistsSameData("0", map, tableName);
@@ -1060,24 +1112,23 @@ public class CadastreManageServiceImpl implements CadastreManageService {
                 }
             }
 
-            // 在dbDataList 中找出 dbExistsMap中各个字段完全一致的数据 并从dbDataList移除
             if (CollectionUtils.isNotEmpty(dbExistsList)) {
                 for (Map<String, Object> dbExistsMap : dbExistsList) {
                     dbDataList.removeIf(dbData -> isSameData(dbData, dbExistsMap));
                 }
             }
 
-            //这里合并两个list 最终写入数据表
             if (CollectionUtils.isNotEmpty(dbDataList)) {
                 theValueList.addAll(dbDataList);
             }
 
-            //先把当前版本都修改为历史版本
             cadastreFileMapper.updateValidFlag("1", "0", tableName);
             for (Map<String, Object> map : theValueList) {
-                map.put("valid_flag", 0);//有效数据标识
+                map.put("valid_flag", 0);
+                cleanDataForInsert(map, columnTypeMap);
                 cadastreFileMapper.insertTableData(tableName, map, shpDbSRID);
             }
+            addRes = true;
         }
         return addRes;
     }

siwei-modules/siwei-apply/src/main/resources/mapper/cadastre/CadastreFileMapper.xml

@@ -156,7 +156,7 @@
         )
     </insert>
 
-    <insert id="insertTableData">
+    <insert id="insertTableData" parameterType="map">
         INSERT INTO vector.${tableName} (
             <foreach collection="data" index="key" item="value" separator=",">
                 ${key}
@@ -165,10 +165,12 @@
         VALUES (
                 <foreach collection="data" index="key" item="value" separator=",">
                     <choose>
-                        <!-- 判断 geom 是否为空，并进行转换 -->
                         <when test="key == 'geom' and value != null">
                             public.st_transform(public.st_geomfromewkt(#{value}), #{shpDbSRID})
                         </when>
+                        <when test="key == 'valid_flag' and value != null">
+                            #{value}::int2
+                        </when>
                         <otherwise>
                             #{value}
                         </otherwise>