fix(人员增量同步bug修改): 人员增量同步bug修改

人员增量同步bug修改

dcuc-auth-service/src/main/java/com/dragoninfo/dcuc/auth/config/DcucAuthConfig.java

@@ -10,8 +10,8 @@ import org.springframework.stereotype.Component;
  * @author fuzq
  * @date 2019/3/13
  */
-@ConfigurationProperties(prefix = "dcuc.auth")
 @Data
+@ConfigurationProperties(prefix = "dcuc.auth")
 @Component
 public class DcucAuthConfig {
 
@@ -82,7 +82,6 @@ public class DcucAuthConfig {
      */
     private String userCenterUrl;
 
-
     /**
      * ADMIN ID
      */
@@ -94,8 +93,8 @@ public class DcucAuthConfig {
     private String rootOrgId;
 
     /**
-     * 人员机构同步定时任务请求头中携带的idcard
+     * 定时任务等系统默认操作人员身份证号
      */
-    private String syncTaskIdcard;
+    private String operatorIdcard ;
 
 }

dcuc-auth-service/src/main/java/com/dragoninfo/dcuc/auth/sub/business/impl/SubSyncBusinessImpl.java

@@ -280,22 +280,24 @@ public class SubSyncBusinessImpl implements ISubSyncBusiness {
             }
             String[] projectCodes = orgRelData.split(StrUtil.COMMA);
             List<AuthUserOrgRel> rels = Arrays.stream(projectCodes).distinct().map(appCode -> {
+                String orgCode = appCodeKeyOrgCodeMap.get(appCode);
+                if (StringUtils.isBlank(orgCode)) {
+                    return null;
+                }
+                return orgCode;
+            }).filter(Objects::nonNull).distinct().map(orgCode->{
                 AuthUserOrgRel rel = new AuthUserOrgRel();
                 rel.setUserId(userInfo.getId());
                 rel.setIdcard(userInfo.getIdcard());
                 rel.setUserType(UserTypeEnum.EXTERNAL.getValue());
                 rel.setRelStatus(JobTypeEnum.ZZ.getValue());
-                String orgCode = appCodeKeyOrgCodeMap.get(appCode);
-                if (StringUtils.isBlank(orgCode)) {
-                    return null;
-                }
                 rel.setOrgCode(orgCode);
                 AuthOrgInfo authOrgInfo = orgCodeKeyOrgMap.get(orgCode);
                 if (null != authOrgInfo) {
                     rel.setOrgId(authOrgInfo.getId());
                 }
                 return rel;
-            }).filter(Objects::nonNull).collect(Collectors.toList());
+            }).collect(Collectors.toList());
             return rels;
         }).filter(Objects::nonNull).flatMap(List::stream).collect(Collectors.toList());
     }
@@ -309,15 +311,13 @@ public class SubSyncBusinessImpl implements ISubSyncBusiness {
                     if (StringUtils.isBlank(userInfo.getOrgCode())) {
                         return null;
                     }
-                    String jsonData = e.getJsonData();
-                    JSONObject jsonObject = JSON.parseObject(jsonData, JSONObject.class);
                     AuthUserOrgRel rel = new AuthUserOrgRel();
                     rel.setUserId(userInfo.getId());
                     rel.setIdcard(userInfo.getIdcard());
                     rel.setUserType(UserTypeEnum.HELPER.getValue());
                     rel.setOrgId(userInfo.getOrgId());
                     rel.setOrgCode(userInfo.getOrgCode());
-                    rel.setRelStatus(jsonObject.getString("userStatus"));
+                    rel.setRelStatus(JobTypeEnum.ZZ.getValue());
                     return rel;
                 }).filter(Objects::nonNull).collect(Collectors.toList());
     }
@@ -370,7 +370,7 @@ public class SubSyncBusinessImpl implements ISubSyncBusiness {
             rel.setRelStatus(jsonObject.getString("jobType"));
             rel.setOrgCode(orgCode);
             return rel;
-        }).collect(Collectors.toList());
+        }).filter(Objects::nonNull).collect(Collectors.toList());
     }
 
 
@@ -444,7 +444,7 @@ public class SubSyncBusinessImpl implements ISubSyncBusiness {
 
     private List<Header> getHeads(String idcard) {
         if(StringUtils.isBlank(idcard)) {
-            idcard = authConfig.getSyncTaskIdcard();
+            idcard = authConfig.getOperatorIdcard();
         }
         return UserCenterBusiUtil.getHeads(idcard, authConfig.getAppCode());
     }
@@ -546,33 +546,32 @@ public class SubSyncBusinessImpl implements ISubSyncBusiness {
             setPoliceOrgCode(content, userType, userInfoList);
         }
         setUserOrgInfos(userType, userInfoList);
-        authUserInfoService.batchSaveByIdcard(userInfoList);
-        return userInfoList;
+        return authUserInfoService.batchSaveByIdcard(userInfoList);
     }
 
-    private void setPoliceOrgCode(List<? extends AuthUserOriginalBase> content, UserTypeEnum userType, List<AuthUserInfo> userInfoList) {
+    private <T extends AuthUserOriginalBase> void setPoliceOrgCode(List<T> content, UserTypeEnum userType, List<AuthUserInfo> userInfoList) {
         //警员根据职务排序,取职务序号小的为主机构
-        Map<String, AuthUserInfo> idcardMap = userInfoList
+        Map<String, AuthUserOriginalBase> contentMap = content
                 .stream()
-                .collect(Collectors.toMap(AuthUserInfo::getIdcard, e -> e));
-        content.forEach(e->{
-            String orgRelData = e.getOrgRelData();
-            JSONArray jsonArray = JSON.parseObject(orgRelData, JSONArray.class);
-            if(!jsonArray.isEmpty()) {
-                String orgCode = jsonArray.stream()
-                        .map(json -> (JSONObject) json).min((x, y) -> {
-                            String xJobType = x.getString("jobType");
-                            String yJobType = y.getString("jobType");
-                            return xJobType.compareTo(yJobType);
-                        }).orElse(new JSONObject()).getString("orgCode");
-                if(StringUtils.isNotBlank(orgCode)) {
-                    AuthUserInfo userInfo = idcardMap.get(e.getIdcard());
-                    if(null != userInfo) {
-                        userInfo.setOrgCode(orgCode);
+                .collect(Collectors.toMap(AuthUserOriginalBase::getIdcard, e -> e));
+        userInfoList.stream()
+                .filter(e->BooleanEnum.FALSE.value.equals(e.getDeleted()))
+                .forEach(e->{
+                    AuthUserOriginalBase base = contentMap.get(e.getIdcard());
+                    String orgRelData = base.getOrgRelData();
+                    JSONArray jsonArray = JSON.parseObject(orgRelData, JSONArray.class);
+                    if(null != jsonArray) {
+                        String orgCode = jsonArray.stream()
+                                .map(json -> (JSONObject) json).min((x, y) -> {
+                                    String xJobType = x.getString("jobType");
+                                    String yJobType = y.getString("jobType");
+                                    return xJobType.compareTo(yJobType);
+                                }).orElse(new JSONObject()).getString("orgCode");
+                        if(StringUtils.isNotBlank(orgCode)) {
+                            e.setOrgCode(orgCode);
+                        }
                     }
-                }
-            }
-        });
+                });
     }
 
     private void setUserOrgInfos(UserTypeEnum userType, List<AuthUserInfo> userInfoList) {
@@ -586,7 +585,10 @@ public class SubSyncBusinessImpl implements ISubSyncBusiness {
         } else {
             //修改orgId
             List<String> orgCodes = userInfoList.stream()
-                    .map(AuthUserInfo::getOrgCode).distinct()
+                    .filter(e->BooleanEnum.FALSE.value.equals(e.getDeleted()))
+                    .map(AuthUserInfo::getOrgCode)
+                    .filter(Objects::nonNull)
+                    .distinct()
                     .collect(Collectors.toList());
             if(!orgCodes.isEmpty()) {
                 List<AuthOrgInfo> authOrgInfos = authOrgInfoService.getOrgByCodes(orgCodes);
@@ -598,7 +600,9 @@ public class SubSyncBusinessImpl implements ISubSyncBusiness {
     private void setUpOrgInfo(List<AuthUserInfo> userInfoList, List<AuthOrgInfo> authOrgInfos) {
         Map<String, AuthOrgInfo> orgCodeMap = authOrgInfos.stream()
                 .collect(Collectors.toMap(AuthOrgInfo::getCode, e -> e, (old, last) -> last));
-        userInfoList.forEach(e->{
+        userInfoList.stream()
+                .filter(e->BooleanEnum.FALSE.value.equals(e.getDeleted()))
+                .forEach(e->{
             AuthOrgInfo authOrgInfo ;
             String orgCode = e.getOrgCode();
             if(StringUtils.isBlank(orgCode) || null == (authOrgInfo = orgCodeMap.get(orgCode))) {
@@ -716,9 +720,9 @@ public class SubSyncBusinessImpl implements ISubSyncBusiness {
                 Date updateTime = new Date();
                 watch.start("getUserInfoAddFromRemote");
                 //获取删除人员信息
-                getDelUserInfoFromRemote(subSyncConfig.getPageSize(), syncStartTime, headers);
+                getDelUserInfoFromRemote(subSyncConfig.getPageSize(), syncStartTime, updateTime, headers);
                 //增量同步用户信息
-                String addUserInfoStatus = getAddUserInfo(subSyncConfig.getPageSize(), syncStartTime, headers);
+                String addUserInfoStatus = getAddUserInfo(subSyncConfig.getPageSize(), syncStartTime, updateTime, headers);
                 watch.stop();
                 if(syncAllFail.equals(addUserInfoStatus)) {
                     log.info("getUserInfoAddFromRemote all failed");
@@ -735,10 +739,10 @@ public class SubSyncBusinessImpl implements ISubSyncBusiness {
                 //同步用户信息
                 watch.start("userInfoAddSync");
                 addUserInfoSync();
-                watch.stop();
-
                 //记录同步成功时间
                 updateAddSyncRecord(AddSyncContance.ADD_SYNC_TYPE_USER, true, updateTime, addUserInfoStatus);
+                watch.stop();
+                log.info("userAddSync total time:{}", watch.prettyPrint());
             } catch (Exception e) {
                 log.error("userAddSync error.", e);
                 //记录同步失败时间
@@ -749,7 +753,7 @@ public class SubSyncBusinessImpl implements ISubSyncBusiness {
         };
     }
 
-    private void getDelUserInfoFromRemote(Integer pageSize, Date syncStartTime, List<Header> headers) {
+    private void getDelUserInfoFromRemote(Integer pageSize, Date syncStartTime, Date updateTime, List<Header> headers) {
         int currentPage = 1;
         Integer totalPage = null;
         ApiSearchReq apiSearchReq = new ApiSearchReq();
@@ -759,10 +763,12 @@ public class SubSyncBusinessImpl implements ISubSyncBusiness {
         //设置同步时间
         Map<String, SearchParam> filters = new HashMap<>();
         SearchParam updateTimeSearch = new SearchParam();
-        updateTimeSearch.setOperator(SearchOperator.ge.name());
-        String dateFormat = DateUtils.getDateFormat(syncStartTime, DateConst.HYPHEN_DISPLAY_TIME);
-        updateTimeSearch.setValue(dateFormat);
-        filters.put("beginUpdateTime", updateTimeSearch);
+        updateTimeSearch.setOperator(SearchOperator.rangeDate.name());
+        String start = DateUtils.getDateFormat(syncStartTime, DateConst.HYPHEN_DISPLAY_TIME);
+        String end = DateUtils.getDateFormat(updateTime, DateConst.HYPHEN_DISPLAY_TIME);
+        updateTimeSearch.setValue(new String[] {start, end});
+        filters.put("updateTime", updateTimeSearch);
+
         SearchParam deleteSearch = new SearchParam();
         deleteSearch.setOperator(SearchOperator.eq.name());
         deleteSearch.setValue(BooleanEnum.TRUE.value);
@@ -822,6 +828,7 @@ public class SubSyncBusinessImpl implements ISubSyncBusiness {
     private void diffTypeAddUserSync(UserTypeEnum userType) {
         int currentPage = 0;
         Searchable searchable = Searchable.newSearchable();
+        searchable.addSearchFilter("userType", SearchOperator.eq, userType.getValue());
         searchable.addSort(Sort.Direction.ASC, "userUpdateTime");
         do {
             searchable.setPage(currentPage, 1000);
@@ -931,16 +938,16 @@ public class SubSyncBusinessImpl implements ISubSyncBusiness {
     }
 
 
-    private String getAddUserInfo(Integer pageSize, Date syncStartTime, List<Header> headers) {
-        String policeStatus = getAddDiffTypeUserInfo(pageSize, syncStartTime, UserApiConstance.POLICE_SYNC_API, UserTypeEnum.POLICE, headers);
-        String helpStatus = getAddDiffTypeUserInfo(pageSize, syncStartTime, UserApiConstance.AUXILIRY_SYNC_API, UserTypeEnum.HELPER, headers);
-        String externalStatus = getAddDiffTypeUserInfo(pageSize, syncStartTime, UserApiConstance.EXTERNAL_SYNC_API, UserTypeEnum.EXTERNAL, headers);
+    private String getAddUserInfo(Integer pageSize, Date syncStartTime, Date updateTime, List<Header> headers) {
+        String policeStatus = getAddDiffTypeUserInfo(pageSize, syncStartTime, updateTime, UserApiConstance.POLICE_SYNC_API, UserTypeEnum.POLICE, headers);
+        String helpStatus = getAddDiffTypeUserInfo(pageSize, syncStartTime, updateTime, UserApiConstance.AUXILIRY_SYNC_API, UserTypeEnum.HELPER, headers);
+        String externalStatus = getAddDiffTypeUserInfo(pageSize, syncStartTime, updateTime, UserApiConstance.EXTERNAL_SYNC_API, UserTypeEnum.EXTERNAL, headers);
         String status = policeStatus + helpStatus + externalStatus;
         log.info("getAddUserInfo status :{}",status);
         return status;
     }
 
-    private String getAddDiffTypeUserInfo(Integer pageSize, Date syncStartTime, String userInfoApi, UserTypeEnum userTypeEnum, List<Header> headers) {
+    private String getAddDiffTypeUserInfo(Integer pageSize, Date syncStartTime, Date syncEndTime, String userInfoApi, UserTypeEnum userTypeEnum, List<Header> headers) {
         int currentPage = 1;
         Integer totalPage = null;
         ApiSearchReq apiSearchReq = new ApiSearchReq();
@@ -950,10 +957,11 @@ public class SubSyncBusinessImpl implements ISubSyncBusiness {
         //设置同步时间
         SearchParam searchParam = new SearchParam();
         Map<String, SearchParam> filters = new HashMap<>();
-        searchParam.setOperator(SearchOperator.ge.name());
-        String dateFormat = DateUtils.getDateFormat(syncStartTime, DateConst.HYPHEN_DISPLAY_TIME);
-        searchParam.setValue(dateFormat);
-        filters.put("beginUpdateTime", searchParam);
+        searchParam.setOperator(SearchOperator.rangeDate.name());
+        String start = DateUtils.getDateFormat(syncStartTime, DateConst.HYPHEN_DISPLAY_TIME);
+        String end = DateUtils.getDateFormat(syncEndTime, DateConst.HYPHEN_DISPLAY_TIME);
+        searchParam.setValue(new String[] {start, end});
+        filters.put("updateTime", searchParam);
         apiSearchReq.setFilters(filters);
         apiSearchReq.setPage(pageReq);
         String url = Joiner.on("").join(authConfig.getUserCenterUrl(), userInfoApi);
@@ -1007,6 +1015,7 @@ public class SubSyncBusinessImpl implements ISubSyncBusiness {
                 originals.add(userOri);
                 continue ;
             }
+            userOri.setUserDeleted(BooleanEnum.FALSE.value);
             setJSONData(userType, json, userOri);
             originals.add(userOri);
         }
@@ -1098,7 +1107,7 @@ public class SubSyncBusinessImpl implements ISubSyncBusiness {
                 Date updateTime = new Date();
 
                 watch.start("getOrgInfoAddFromRemote");
-                Integer total = getOrgInfoAddFromRemote(subSyncConfig.getPageSize(), syncStartTime, headers);
+                Integer total = getOrgInfoAddFromRemote(subSyncConfig.getPageSize(), syncStartTime, updateTime,headers);
                 watch.stop();
 
                 if(total == null) {
@@ -1143,6 +1152,12 @@ public class SubSyncBusinessImpl implements ISubSyncBusiness {
     }
 
     private List<String> setAddPathAndUpId(List<AuthOrgInfo> updateOrgInfos) {
+        //最顶级机构被修改无需修改自己的path和upId,需要过滤
+        //避免最顶级的机构被修改导致下级机构path和id无法设置
+        updateOrgInfos = updateOrgInfos
+                .stream()
+                .filter(e->null != e.getUpGovCode())
+                .collect(Collectors.toList());
         Set<String> addOrgCodes = updateOrgInfos.stream().map(AuthOrgInfo::getCode).collect(Collectors.toSet());
         //获取新增或修改的机构
         List<String> upOrgInfoCodes = updateOrgInfos.stream()
@@ -1151,6 +1166,9 @@ public class SubSyncBusinessImpl implements ISubSyncBusiness {
                 .collect(Collectors.toList());
         //查询增量同步机构中的上级机构，且是已存在机构表中的机构
         List<String> collect = upOrgInfoCodes.stream().filter(e -> !addOrgCodes.contains(e)).collect(Collectors.toList());
+        if(CollectionUtils.isEmpty(collect)) {
+            return Collections.emptyList();
+        }
         List<AuthOrgInfo> upOrgInfos = authOrgInfoService.getOrgByCodes(collect);
         //设置path和upGovId
         Map<String, AuthOrgInfo> upOrgMap = upOrgInfos.stream()
@@ -1221,7 +1239,7 @@ public class SubSyncBusinessImpl implements ISubSyncBusiness {
         return list;
     }
 
-    private Integer getOrgInfoAddFromRemote(Integer pageSize, Date syncStartTime, List<Header> headers) {
+    private Integer getOrgInfoAddFromRemote(Integer pageSize, Date syncStartTime, Date syncEndTime, List<Header> headers) {
         int currentPage = 1;
         Integer totalPage = null;
         Integer total = null;
@@ -1232,10 +1250,11 @@ public class SubSyncBusinessImpl implements ISubSyncBusiness {
         //设置同步时间
         SearchParam searchParam = new SearchParam();
         Map<String, SearchParam> filters = new HashMap<>();
-        searchParam.setOperator(SearchOperator.ge.name());
-        String dateFormat = DateUtils.getDateFormat(syncStartTime, DateConst.HYPHEN_DISPLAY_TIME);
-        searchParam.setValue(dateFormat);
-        filters.put("beginUpdateTime", searchParam);
+        searchParam.setOperator(SearchOperator.rangeDate.name());
+        String start = DateUtils.getDateFormat(syncStartTime, DateConst.HYPHEN_DISPLAY_TIME);
+        String end = DateUtils.getDateFormat(syncEndTime, DateConst.HYPHEN_DISPLAY_TIME);
+        searchParam.setValue(new String[] {start, end});
+        filters.put("updateDate", searchParam);
         apiSearchReq.setFilters(filters);
         apiSearchReq.setPage(pageReq);
         String url = Joiner.on("").join(authConfig.getUserCenterUrl(), UserApiConstance.ORG_SYNC_API);
@@ -1256,7 +1275,8 @@ public class SubSyncBusinessImpl implements ISubSyncBusiness {
             JSONObject obj = (JSONObject) apiResult.getResult();
             ApiResultPage<JSONObject> page = JSON.parseObject(obj.toJSONString(), new TypeReference<ApiResultPage<JSONObject>>(){});
             //从结果解析总页数
-            totalPage = getTotalPage(pageSize, totalPage, page.getTotal().intValue());
+            total = page.getTotal().intValue();
+            totalPage = getTotalPage(pageSize, totalPage, total);
             //解析请求结果
             saveAddOrgOri(page);
         } while (++currentPage <= totalPage);
@@ -1268,7 +1288,7 @@ public class SubSyncBusinessImpl implements ISubSyncBusiness {
         List<JSONObject> content = page.getContent();
         List<AuthOrgAddOriginal> collect = content.stream().map(json -> {
             AuthOrgAddOriginal original = new AuthOrgAddOriginal();
-            String orgCode = json.getString("code");
+            String orgCode = json.getString("orgCode");
             String orgDeleted = json.getString("deleted");
             Date updateTime = json.getDate("updateTime");
             original.setCode(orgCode);

dcuc-auth-service/src/main/java/com/dragoninfo/dcuc/auth/sub/service/impl/AuthOrgTreeServiceImpl.java

@@ -205,6 +205,7 @@ public class AuthOrgTreeServiceImpl implements IAuthOrgTreeService {
          * @param allTopNods
          */
         private void constructTrees(Map<String, OrgTreeNodeDTO> treeNodeMap, List<OrgTreeNodeDTO> allTopNods) {
+            log.info("---------------constructTrees start-----------------");
             List<AuthOrgInfo> orgInfos = authOrgInfoService.findAll();
             //转成树节点对象
             List<OrgTreeNodeDTO> treeNods = getTreeNods(orgInfos);
@@ -221,7 +222,7 @@ public class AuthOrgTreeServiceImpl implements IAuthOrgTreeService {
                     .stream()
                     .collect(Collectors.groupingBy(OrgTreeNodeDTO::getPid));
             getTreeConstruct(treeNodeMap, rootNodes, upIdGroupMap);
-
+            log.info("---------------constructTrees end,treeNodes:{}-----------------", orgInfos.size());
         }
 
         /**

dcuc-auth-service/src/main/java/com/dragoninfo/dcuc/auth/sub/service/impl/AuthUserInfoService.java

@@ -74,31 +74,31 @@ public class AuthUserInfoService implements IAuthUserInfoService {
         if(CollectionUtils.isEmpty(userInfoList)) {
             return Collections.emptyList();
         }
+        List<AuthUserInfo> list = new ArrayList<>();
         List<String> idcards = userInfoList.stream().map(AuthUserInfo::getIdcard).distinct().collect(Collectors.toList());
         List<AuthUserInfo> idAndIdcards = getIdAndIdcards(idcards);
         Map<String, AuthUserInfo> existMap = idAndIdcards.stream()
                 .collect(Collectors.toMap(AuthUserInfo::getIdcard, e -> e, (old, last) -> last));
-        Iterator<AuthUserInfo> iterator = userInfoList.iterator();
-        while (iterator.hasNext()) {
-            AuthUserInfo authUserInfo = iterator.next();
-            String idcard = authUserInfo.getIdcard();
-            AuthUserInfo exist = existMap.get(idcard);
-            //删除人员信息
-            if (BooleanEnum.TRUE.value.equals(authUserInfo.getDeleted())) {
-                userInfoRepository.deleteByIdcard(idcard);
-                iterator.remove();
-                continue;
-            }
-            //新增或更新人员信息
-            if(null == exist) {
-                authUserInfo.setDeleted(BooleanEnum.FALSE.value);
-                userInfoRepository.save(authUserInfo);
-            } else {
-                BeanUtils.copyProperties(authUserInfo, exist, "id","createTime","createUser");
-                userInfoRepository.update(exist);
-            }
-        }
-        return userInfoList;
+       userInfoList.forEach(e->{
+           String idcard = e.getIdcard();
+           AuthUserInfo exist = existMap.get(idcard);
+           //删除人员信息
+           if (BooleanEnum.TRUE.value.equals(e.getDeleted())) {
+               userInfoRepository.deleteByIdcard(idcard);
+               return ;
+           }
+           //新增或更新人员信息
+           if(null == exist) {
+               e.setDeleted(BooleanEnum.FALSE.value);
+               AuthUserInfo save = userInfoRepository.save(e);
+               list.add(save);
+           } else {
+               BeanUtils.copyProperties(e, exist, "id","createTime","createUser");
+               AuthUserInfo update = userInfoRepository.update(exist);
+               list.add(update);
+           }
+       });
+       return list;
     }
 
     @Override