自动抓取不动产数据

siwei-modules/siwei-apply/src/main/java/com/siwei/apply/common/Constant.java

@@ -1,5 +1,7 @@
 package com.siwei.apply.common;
 
+import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.List;
 import java.util.Map;
 
@@ -102,6 +104,225 @@ public class Constant {
             "DX2026"
     );
 
+
+    public static final List<String> BDCDYH_LIST = new ArrayList<>(Arrays.asList(
+            "360122009006GB00002W00000000",
+            "360122009006GB00159W00000000",
+            "360122009006GB00160W00000000",
+            "360122009006GB00161W00000000",
+            "360122009006GB60008W00000000",
+            "360122009006GB60107W00000000",
+            "360122009006GB60168W00000000",
+            "360122009006GB60169W00000000",
+            "360122009006GB60170W00000000",
+            "360122009006GB60171W00000000",
+            "360122009006GB60182W00000000",
+            "360122009006GB60183W00000000",
+            "360122009006GB60190W00000000",
+            "360122009006GB60242W00000000",
+            "360122009006GB60243W00000000",
+            "360122009006GB60273W00000000",
+            "360122009015GB00006W00000000",
+            "360122009015GB00013W00000000",
+            "360122009015GB00015W00000000",
+            "360122009015GB00016W00000000",
+            "360122009015GB60002W00000000",
+            "360122009015GB60053W00000000",
+            "360122009015GB60056W00000000",
+            "360122009015GB60057W00000000",
+            "360122009015GB60058W00000000",
+            "360122009015GB60059W00000000",
+            "360122009015GB60060W00000000",
+            "360122011002GB00004W00000000",
+            "360122011002GB00008W00000000",
+            "360122011002GB00013W00000000",
+            "360122011002GB00029W00000000",
+            "360122011002GB00030W00000000",
+            "360122011002GB00031W00000000",
+            "360122011002GB00308W00000000",
+            "360122011002GB60006W00000000",
+            "360122011002GB60007W00000000",
+            "360122011002GB60012W00000000",
+            "360122011002GB60013W00000000",
+            "360122011003GB00077W00000000",
+            "360122011003GB00246W00000000",
+            "360122011003GB00332W00000000",
+            "360122011003GB00333W00000000",
+            "360122011003GB00334W00000000",
+            "360122011003GB00335W00000000",
+            "360122011003GB00486W00000000",
+            "360122011004GB00381W00000000",
+            "360122011004GB00383W00000000",
+            "360122011004GB00389W00000000",
+            "360122011004GB00391W00000000",
+            "360122011004GB00392W00000000",
+            "360122011004GB00393W00000000",
+            "360122011004GB00394W00000000",
+            "360122011004GB00395W00000000",
+            "360122011004GB00418W00000000",
+            "360122011004GB00419W00000000",
+            "360122011004GB00533W00000000",
+            "360122011004GB00534W00000000",
+            "360122011005GB00002W00000000",
+            "360122011005GB00003W00000000",
+            "360122011005GB60002W00000000",
+            "360122011005GB60003W00000000",
+            "360122011005GB60005W00000000",
+            "360122011005GB60006W00000000",
+            "360122011006GB00001W00000000",
+            "360122011006GB00002W00000000",
+            "360122011006GB00007W00000000",
+            "360122011006GB00008W00000000",
+            "360122011006GB00009W00000000",
+            "360122011006GB00319W00000000",
+            "360122011006GB00320W00000000",
+            "360122011007GB00002W00000000",
+            "360122011007GB00005W00000000",
+            "360122011007GB00009W00000000",
+            "360122011007GB00016W00000000",
+            "360122011007GB00310W00000000",
+            "360122011007GB00324W00000000",
+            "360122011007GB00333W00000000",
+            "360122011007GB00336W00000000",
+            "360122011007GB00337W00000000",
+            "360122011007GB00338W00000000",
+            "360122011007GB00340W00000000",
+            "360122011007GB00341W00000000",
+            "360122011009GB00334W00000000",
+            "360122011009GB00335W00000000",
+            "360122011010GB00018W00000000",
+            "360122011010GB00021W00000000",
+            "360122011010GB00308W00000000",
+            "360122011010GB00312W00000000",
+            "360122011010GB00318W00000000",
+            "360122011010GB00319W00000000",
+            "360122011010GB60011W00000000",
+            "360122013029GB60001W00000000",
+            "360122014003GB00011W00000000",
+            "360122014003GB00045W00000000",
+            "360122014003GB00115W00000000",
+            "360122014003GB60001W00000000",
+            "360122014003GB60002W00000000",
+            "360122014003GB60003W00000000",
+            "360122014003GB60014W00000000",
+            "360122014003GB62229W00000000",
+            "360122014003GB62230W00000000",
+            "360122014003GB62231W00000000",
+            "360122014003GB62232W00000000",
+            "360122014003GB62233W00000000",
+            "360122014003GB62234W00000000",
+            "360122014003GB62235W00000000",
+            "360122014003GB62236W00000000",
+            "360122014003GB62237W00000000",
+            "360122014003GB62238W00000000",
+            "360122014003GB62243W00000000",
+            "360122014003GB62270W00000000",
+            "360122014003GB62277W00000000",
+            "360122014003GB62280W00000000",
+            "360122014003GB62302W00000000",
+            "360122014003GB62305W00000000",
+            "360122014003GB62306W00000000",
+            "360122014003GB62307W00000000",
+            "360122014003GB62309W00000000",
+            "360122014003GB62310W00000000",
+            "360122014003GB62317W00000000",
+            "360122014003GB62328W00000000",
+            "360122014003GB62340W00000000",
+            "360122014003GB62341W00000000",
+            "360122014003GB62360W00000000",
+            "360122014003GB62361W00000000",
+            "360122014003GB62362W00000000",
+            "360122014003GB62367W00000000",
+            "360122014003GB62368W00000000",
+            "360122014004GB00071W00000000",
+            "360122014004GB00072W00000000",
+            "360122014004GB00073W00000000",
+            "360122014004GB00074W00000000",
+            "360122014004GB00075W00000000",
+            "360122014004GB00076W00000000",
+            "360122014004GB00077W00000000",
+            "360122014004GB00078W00000000",
+            "360122014004GB00079W00000000",
+            "360122014004GB00081W00000000",
+            "360122014004GB00083W00000000",
+            "360122014004GB00084W00000000",
+            "360122014004GB00085W00000000",
+            "360122014004GB00086W00000000",
+            "360122014004GB00087W00000000",
+            "360122014004GB00088W00000000",
+            "360122014004GB00089W00000000",
+            "360122014004GB00090W00000000",
+            "360122014004GB00092W00000000",
+            "360122014004GB00094W00000000",
+            "360122014004GB00098W00000000",
+            "360122014004GB00099W00000000",
+            "360122014004GB00104W00000000",
+            "360122014004GB00105W00000000",
+            "360122014004GB00106W00000000",
+            "360122014004GB00107W00000000",
+            "360122014004GB00108W00000000",
+            "360122014004GB00109W00000000",
+            "360122014004GB00110W00000000",
+            "360122014004GB00111W00000000",
+            "360122014004GB00113W00000000",
+            "360122014004GB00114W00000000",
+            "360122014004GB00121W00000000",
+            "360122014004GB00128W00000000",
+            "360122014004GB00129W00000000",
+            "360122014004GB60001W00000000",
+            "360122014004GB60006W00000000",
+            "360122014004GB60017W00000000",
+            "360122014004GB60018W00000000",
+            "360122014004GB60024W00000000",
+            "360122014004GB60035W00000000",
+            "360122014005GB00044W00000000",
+            "360122014005GB00046W00000000",
+            "360122014005GB00047W00000000",
+            "360122014005GB00048W00000000",
+            "360122014005GB00050W00000000",
+            "360122014005GB60001W00000000",
+            "360122014005GB60012W00000000",
+            "360122014008GB00050W00000000",
+            "360122015001GB60001W00000000",
+            "360122015001GB60010W00000000",
+            "360122015001GB60011W00000000",
+            "360122015001GB60012W00000000",
+            "360122015001GB60013W00000000",
+            "360122015001GB60014W00000000",
+            "360122015001GB70001W00000000",
+            "360122015001GB70002W00000000",
+            "360122015002GB00001W00000000",
+            "360122015002GB00002W00000000",
+            "360122015002GB00003W00000000",
+            "360122015002GB00005W00000000",
+            "360122015002GB00019W00000000",
+            "360122015002GB00020W00000000",
+            "360122015002GB00021W00000000",
+            "360122015002GB00022W00000000",
+            "360122015002GB00023W00000000",
+            "360122015002GB00024W00000000",
+            "360122015002GB00025W00000000",
+            "360122015003GB00003W00000000",
+            "360122015003GB00004W00000000",
+            "360122015003GB00005W00000000",
+            "360122020005GB00137W00000000",
+            "360122999000GB60002W00000000",
+            "360425001012GB00027W00000000",
+            "360425005005GB60001W00000000",
+            "360425005005GB60003W00000000",
+            "360425005005GB60005W00000000",
+            "360425005005GB60006W00000000",
+            "360425005005GB60007W00000000",
+            "360425005005GB60008W00000000",
+            "360425005005GB60009W00000000",
+            "360425005005GB60014W00000000",
+            "360425005005GB60015W00000000",
+            "360425005005GB60017W00000000"
+    ));
+
+
+
+
     //项目excel导入路径
     //public static final String  IMPORT_EXCEL_DATA_FILE_PATH = "D:\\one_code_manage\\project_excel_data";
     public static final String  IMPORT_EXCEL_DATA_FILE_PATH = "D:\\tmp\\one_code_manage\\excel_deom";
@@ -110,7 +331,6 @@ public class Constant {
 
     public static final String  GG = "GONGGAI";
     public static final String  BDC = "BUDONGCHAN";
-
-
+    public static final String  BDC_FILE_DOWN_PATH = "C:\\Users\\Administrator\\Desktop\\11\\excel_out\\output\\02";
 
 }

siwei-modules/siwei-apply/src/main/java/com/siwei/apply/controller/third/DealExcelController.java

@@ -30,10 +30,31 @@ import java.util.*;
 public class DealExcelController extends BaseController {
     protected final Logger logger = LoggerFactory.getLogger(this.getClass());
 
-
     @Autowired
     DealExcelServiceImpl  dealExcelService;
 
+    /**
+     * 不动产数据下载
+     */
+    @GetMapping("/excelImmobileDown/{fileName}")
+    public R<Map<String, List<Map<String,String>>>> excelImmobileDown(@PathVariable String fileName) {
+        try {
+            String uploadFile = "C:\\Users\\Administrator\\Desktop\\11\\excel_out"+"\\"+fileName;
+            Path filePath = Paths.get(uploadFile);
+            File file = filePath.toFile();
+            Map<String, List<Map<String, String>>> dataMap = readExcelWithMergedHeaders(file);
+            String res = dealExcelService.excelImmobileDownFile(dataMap);
+            logger.info("检查数据结果{}",res);
+            return R.ok(dataMap);
+        } catch (Exception e) {
+            e.printStackTrace();
+            return R.fail(e.getMessage());
+        }
+    }
+
+
+
+
 
     /**
      * 处理excel数据，批次报批数据

siwei-modules/siwei-apply/src/main/java/com/siwei/apply/service/third/impl/DealExcelServiceImpl.java

@@ -1,26 +1,22 @@
 package com.siwei.apply.service.third.impl;
 
+import com.alibaba.fastjson.serializer.SerializerFeature;
+import com.alibaba.fastjson2.JSON;
+import com.alibaba.fastjson2.JSONObject;
+import com.alibaba.fastjson2.JSONWriter;
 import com.siwei.apply.common.Constant;
 import com.siwei.apply.domain.LandType;
 import com.siwei.apply.domain.NodeLand;
 import com.siwei.apply.domain.ProjectWorkflow;
 import com.siwei.apply.domain.YdbpData;
 import com.siwei.apply.domain.res.TdgyRes;
-import com.siwei.apply.domain.vo.GyjsydscdjVo;
-import com.siwei.apply.domain.vo.GyjsydjfwscdjVo;
-import com.siwei.apply.domain.vo.JsgcghxkVo;
-import com.siwei.apply.domain.vo.JsydghxkVo;
-import com.siwei.apply.domain.vo.ProjectVo;
-import com.siwei.apply.domain.vo.TdgyVo;
-import com.siwei.apply.domain.vo.TdhyhsVo;
-import com.siwei.apply.domain.vo.TjyydhxVo;
-import com.siwei.apply.domain.vo.YdbpDataVo;
-import com.siwei.apply.domain.vo.YdbpVo;
-import com.siwei.apply.domain.vo.YdysyxzVo;
+import com.siwei.apply.domain.vo.*;
 import com.siwei.apply.enums.AloneWorkFlowEnum;
 import com.siwei.apply.mapper.CadastreFileMapper;
 import com.siwei.apply.mapper.LandTypeMapper;
 import com.siwei.apply.service.*;
+import com.siwei.apply.utils.ServiceFileUtil;
+import com.siwei.apply.utils.ServiceUtil;
 import com.siwei.common.core.constant.SecurityConstants;
 import com.siwei.common.core.domain.R;
 import com.siwei.common.core.utils.StringUtils;
@@ -28,6 +24,7 @@ import com.siwei.spatial.api.RemoteSpatialFilesDbService;
 import com.siwei.spatial.api.domain.file.TGeomDb;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.collections4.CollectionUtils;
+import org.apache.commons.collections4.MapUtils;
 import org.apache.commons.io.FileUtils;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Service;
@@ -58,6 +55,8 @@ public class DealExcelServiceImpl {
     private final String rootPath2020 = Constant.IMPORT_EXCEL_DATA_FILE_PATH + "\\数据治理批量导入示例数据\\赣江新区项目文件夹建立\\1.出让项目\\2020";
     private final String rootPathZip = Constant.IMPORT_EXCEL_DATA_FILE_PATH + "\\zip";
     private final String targectDir = Constant.FILE_UPLOAD_PATH;
+    private static final String bdcFileDownPath =  Constant.BDC_FILE_DOWN_PATH;
+
 
     public final static List<LandType> allLandTypeList = new ArrayList<>();
 
@@ -112,6 +111,151 @@ public class DealExcelServiceImpl {
     @Autowired
     private LandOneCodeService landOneCodeService;
 
+
+    /**
+     * 下载相关的不动产材料附件
+     * @param dataMap
+     * @return
+     */
+    public String excelImmobileDownFile(Map<String, List<Map<String, String>>> dataMap) {
+        List<Map<String, String>> immobileList = dataMap.get("不动产权信息列表");
+        int i=0;
+        for (Map<String, String> batch : immobileList) {
+            String ywbh = batch.get("业务号");
+            String bdcdyh = batch.get("不动产单元号");
+            //ywbh = "BL202603195001";
+
+            Map<String, Object> retMap = new HashMap<>();
+            //todo 这里进行远程请求，获取相关数据，解析后入库
+            try {
+                Map<String,Object> paramMap = new LinkedHashMap<>();
+                ywbh+="01";
+                paramMap.put("ywbh",ywbh);
+                paramMap.put("yfbdcdyh","");
+
+                if(Constant.BDCDYH_LIST.contains(bdcdyh)){
+                    log.info("已经处理过了，pass了");
+                    continue;
+                }
+
+
+
+
+
+
+                String remoteResult = ServiceUtil.getRemoteDataByHttpClient("query.landRegist.datas",paramMap);
+                JSONObject root = JSON.parseObject(remoteResult);
+                log.info("远程调用返回结果: {}", root);
+
+                JSONObject innerDataCheck = root.getJSONObject("data");
+                if(!innerDataCheck.get("code").equals("200")){
+                    log.warn("接口请求返回: {}，ywh:{}", innerDataCheck,ywbh);
+                    continue;
+                }
+
+                //todo 这里需要确认用户返回的是数组还是对象
+                JSONObject innerData = (JSONObject) root.getJSONObject("data").getJSONArray("data").get(0);
+                LandFirstParamVo body = JSON.parseObject(innerData.toString(), LandFirstParamVo.class);
+                String prettyJson = JSON.toJSONString(innerData, JSONWriter.Feature.PrettyFormat);
+                //这里记录日志
+                writeToLog(bdcdyh+".txt",prettyJson);
+                retMap.put("body",body);
+            } catch (Exception e) {
+                log.error("调用矢量保存文件错误,业务编号：{}: ", ywbh);
+                e.printStackTrace();
+                throw new RuntimeException(e);
+            }
+
+            //第二部分获取附件数据
+            try {
+                if(MapUtils.isNotEmpty(retMap)){
+                    Map<String,Object> paramMap = new LinkedHashMap<>();
+                    paramMap.put("ywbh",ywbh);
+                    String remoteResult = ServiceUtil.getRemoteDataByHttpClient("query.ywFiles.datas",paramMap);
+                    JSONObject root = JSON.parseObject(remoteResult);
+                    JSONObject innerData = root.getJSONObject("data").getJSONObject("data");
+                    LandFirstParamVo body = JSON.parseObject(innerData.toString(), LandFirstParamVo.class);
+                    //这里保存附件信息
+                    if(Objects.nonNull(body) && Objects.nonNull(body.getAttachment()) && !body.getAttachment().isEmpty()){
+                        this.getThirdAttachmentToLocal(bdcdyh,body.getAttachment());
+                    }
+                }
+            } catch (Exception e) {
+                log.error("调用不动产获取附件信息错误,业务号：{}: ", ywbh);
+                e.printStackTrace();
+                throw new RuntimeException(e);
+            }
+            //if(++i>=3)break;
+        }//end for
+        return null;
+    }
+
+
+    public String getThirdAttachmentToLocal(String nodeName,List<Map<String, Object>> attachmentList) {
+        String fileDownLoadPath = "";
+        try {
+            //这里增加一个方法，把文件下载下来
+            String unsafeChars = "[<>:\"/\\\\|?*]";
+            if (CollectionUtils.isNotEmpty(attachmentList)) {
+                String[] subDirectoryNameArr = attachmentList.stream().map(s->s.get("dir_name").toString()).distinct().toArray(String[]::new);
+                String[] subDirectoryNameNew = Arrays.stream(subDirectoryNameArr)
+                        .map(name -> name.replaceAll(unsafeChars, "_")).toArray(String[]::new);
+                //重新创建子目录
+                fileDownLoadPath = createMultipartDirectory(nodeName,subDirectoryNameNew);
+                for (Map<String, Object> attachment : attachmentList) {
+                    String url = Optional.ofNullable(attachment.getOrDefault("path", attachment.get("PATH")))
+                            .map(String::valueOf)
+                            .orElse("");
+
+                    if(org.apache.commons.lang3.StringUtils.isBlank(url)){ //这里避免空值造成数据无法传输
+                        continue;
+                    }
+                    //todo 这里需要进行url替换转
+                    String baseUrl = ServiceUtil.THIRD_API_DOMAIN+"/estate";
+                    String otherDomian = "http://172.16.6.23";
+                    url = org.apache.commons.lang3.StringUtils.replace(url,otherDomian,baseUrl);
+                    //子目录路径
+                    String directoryName = Optional.ofNullable(attachment.getOrDefault("dir_name", attachment.get("DIR_NAME")))
+                            .map(String::valueOf)
+                            .orElse("");
+                    //去掉特殊字符
+                    directoryName = directoryName.replaceAll(unsafeChars,"_");
+                    String fileName = Optional.ofNullable(attachment.getOrDefault("file_name", attachment.get("FILE_NAME")))
+                            .map(String::valueOf)
+                            .orElse("");
+
+                    String subDownLoadFilePath = fileDownLoadPath + "/" + directoryName;
+
+                    //todo 这里不需要try catch 如果下载不了,不进行入库,可以便于查找问题
+                    try {
+                        ServiceFileUtil.downloadFile(url,fileName, subDownLoadFilePath,Constant.BDC);
+                    }catch (IOException e){
+                        System.out.println("处理文件异常1" + e);
+                    }
+                }
+            }
+            return "";
+        } catch (Exception e) {
+            System.out.println("处理文件异常2" + e);
+        }
+
+        return "";
+    }
+
+
+    public static String createMultipartDirectory(String directoryName, String[] subDirectoryName) {
+        String directoryPath = bdcFileDownPath;
+        String directoryPathAndName = ServiceFileUtil.createDirectory(directoryPath, directoryName);
+        Arrays.stream(subDirectoryName).forEach(name -> {
+            ServiceFileUtil.createDirectory(directoryPathAndName, name);
+        });
+        return directoryPath + "/" + directoryName;
+    }
+
+
+
+
+
     /**
      * 报批数据导入
      *
@@ -1098,7 +1242,7 @@ public class DealExcelServiceImpl {
      */
     public static String writeToLog(String fileName, String msg) {
         // 转换为节点
-        String filePath = Constant.IMPORT_EXCEL_DATA_FILE_PATH + "\\log\\" + fileName;
+        String filePath = Constant.BDC_FILE_DOWN_PATH + "\\log\\" + fileName;
         //todo 完善当前方法，功能：1.将msg写入到filePath指定的文件中；2.如果文件不存在则创建；3.每次写入都追加到文件末尾；4.写入完成后返回写入结果（成功/失败）
         // 文件就是一个文本文件，每次写入一行日志，格式为：时间戳 + msg
         try {
@@ -1109,8 +1253,8 @@ public class DealExcelServiceImpl {
             }
             // 时间戳 + msg
             String timestamp = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss").format(new Date());
-            String logEntry = timestamp + " " + msg + System.lineSeparator();
-
+            //String logEntry = timestamp + " " + msg + System.lineSeparator();
+            String logEntry = msg;
             // 追加写入
             FileUtils.writeStringToFile(file, logEntry, StandardCharsets.UTF_8, true);
             return "成功";