Skip to content

Commit

Permalink
Merge pull request #594 from jsonwan/github_perf/3.3.x_log
Browse files Browse the repository at this point in the history 
perf: 3.3.x日志优化，便于排查同步异常问题 #593
  • Loading branch information
@jsonwan
jsonwan authored Jan 5, 2022
2 parents e94de8f + 7c83aee commit dd7ad1a
Show file tree
Hide file tree
Showing 7 changed files with 119 additions and 87 deletions.
155 changes: 93 additions & 62 deletions src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/sdk/EsbCcClient.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -770,6 +770,98 @@ public List<ApplicationHostInfoDTO> findModuleHostRelationConcurrently(long appI
return applicationHostInfoDTOList;
}

private void fillAgentInfo(
ApplicationHostInfoDTO applicationHostInfoDTO,
FindModuleHostRelationResult.HostProp host
) {
String multiIp = host.getIp();
multiIp = multiIp.trim();
if (queryAgentStatusClient != null) {
if (multiIp.contains(",")) {
Pair<String, Boolean> pair = queryAgentStatusClient.getHostIpWithAgentStatus(multiIp,
host.getCloudAreaId());
if (pair != null) {
log.debug("query agent status:{}:{}", pair.getLeft(), pair.getRight());
String ipWithCloudId = pair.getLeft();
applicationHostInfoDTO.setGseAgentAlive(pair.getRight());
if (ipWithCloudId.contains(":")) {
String[] arr = ipWithCloudId.split(":");
applicationHostInfoDTO.setCloudAreaId(Long.parseLong(arr[0]));
applicationHostInfoDTO.setIp(arr[1]);
} else {
applicationHostInfoDTO.setIp(ipWithCloudId);
}
} else {
log.warn("Fail to get agentStatus, host={}", JsonUtils.toJson(host));
}
} else {
applicationHostInfoDTO.setGseAgentAlive(false);
applicationHostInfoDTO.setCloudAreaId(host.getCloudAreaId());
applicationHostInfoDTO.setIp(multiIp);
}
} else {
log.warn("queryAgentStatusClient==null, please check!");
List<String> ipList = Utils.getNotBlankSplitList(multiIp, ",");
if (ipList.size() > 0) {
applicationHostInfoDTO.setIp(ipList.get(0));
} else {
log.warn("no available ip after queryAgentStatusClient");
}
}
}

private ApplicationHostInfoDTO convertToHostInfoDTO(
Long appId,
FindModuleHostRelationResult.HostWithModules hostWithModules
) {
FindModuleHostRelationResult.HostProp host = hostWithModules.getHost();
String multiIp = host.getIp();
if (multiIp != null) {
multiIp = multiIp.trim();
} else {
log.warn("multiIp is null, appId={}, host={}", appId, hostWithModules);
}
//包装为ApplicationHostInfoDTO
ApplicationHostInfoDTO applicationHostInfoDTO = new ApplicationHostInfoDTO();
applicationHostInfoDTO.setAppId(appId);
applicationHostInfoDTO.setDisplayIp(multiIp);
applicationHostInfoDTO.setCloudAreaId(host.getCloudAreaId());
applicationHostInfoDTO.setHostId(host.getHostId());
fillAgentInfo(applicationHostInfoDTO, host);
List<FindModuleHostRelationResult.ModuleProp> modules = hostWithModules.getModules();
for (FindModuleHostRelationResult.ModuleProp module : modules) {
if (module == null || null == module.getModuleId()) {
log.warn("invalid host:" + JsonUtils.toJson(applicationHostInfoDTO));
}
}
List<FindModuleHostRelationResult.ModuleProp> validModules =
hostWithModules.getModules().stream().filter(Objects::nonNull).collect(Collectors.toList());
applicationHostInfoDTO.setModuleId(
validModules.stream()
.map(FindModuleHostRelationResult.ModuleProp::getModuleId)
.collect(Collectors.toList()));
applicationHostInfoDTO.setSetId(
validModules.stream()
.map(FindModuleHostRelationResult.ModuleProp::getSetId)
.collect(Collectors.toList()));
applicationHostInfoDTO.setModuleType(validModules.stream().map(it -> {
try {
return Long.parseLong(it.getModuleType());
} catch (Exception e) {
return 0L;
}
}).collect(Collectors.toList()));
applicationHostInfoDTO.setIpDesc(host.getHostName());
String os = host.getOs();
if (os != null && os.length() > 512) {
applicationHostInfoDTO.setOs(os.substring(0, 512));
} else {
applicationHostInfoDTO.setOs(os);
}
applicationHostInfoDTO.setOsType(host.getOsType());
return applicationHostInfoDTO;
}

private List<ApplicationHostInfoDTO> convertToHostInfoDTOList(
long appId,
List<FindModuleHostRelationResult.HostWithModules> hostWithModulesList) {
Expand All @@ -784,68 +876,7 @@ private List<ApplicationHostInfoDTO> convertToHostInfoDTOList(
ipSet.add(host.getCloudAreaId() + ":" + host.getIp());
String multiIp = host.getIp();
if (!StringUtils.isBlank(multiIp)) {
multiIp = multiIp.trim();
//包装为ApplicationHostInfoDTO
ApplicationHostInfoDTO applicationHostInfoDTO = new ApplicationHostInfoDTO();
applicationHostInfoDTO.setAppId(appId);
applicationHostInfoDTO.setDisplayIp(multiIp);
applicationHostInfoDTO.setCloudAreaId(host.getCloudAreaId());
applicationHostInfoDTO.setHostId(host.getHostId());
if (queryAgentStatusClient != null) {
if (multiIp.contains(",")) {
Pair<String, Boolean> pair = queryAgentStatusClient.getHostIpWithAgentStatus(host.getIp(),
host.getCloudAreaId());
if (pair != null) {
log.debug("query agent status:{}:{}", pair.getLeft(), pair.getRight());
String ipWithCloudId = pair.getLeft();
applicationHostInfoDTO.setGseAgentAlive(pair.getRight());
if (ipWithCloudId.contains(":")) {
String[] arr = ipWithCloudId.split(":");
applicationHostInfoDTO.setCloudAreaId(Long.parseLong(arr[0]));
applicationHostInfoDTO.setIp(arr[1]);
} else {
applicationHostInfoDTO.setIp(ipWithCloudId);
}
} else {
log.warn("Fail to get agentStatus, host={}", JsonUtils.toJson(host));
}
} else {
applicationHostInfoDTO.setGseAgentAlive(false);
applicationHostInfoDTO.setCloudAreaId(host.getCloudAreaId());
applicationHostInfoDTO.setIp(multiIp);
}
} else {
log.warn("queryAgentStatusClient==null, please check!");
List<String> ipList = Utils.getNotBlankSplitList(multiIp, ",");
if (ipList.size() > 0) {
applicationHostInfoDTO.setIp(ipList.get(0));
} else {
log.warn("no available ip after queryAgentStatusClient");
}
}
applicationHostInfoDTO.setModuleId(
hostWithModules.getModules().stream()
.map(FindModuleHostRelationResult.ModuleProp::getModuleId)
.collect(Collectors.toList()));
applicationHostInfoDTO.setSetId(
hostWithModules.getModules().stream()
.map(FindModuleHostRelationResult.ModuleProp::getSetId)
.collect(Collectors.toList()));
applicationHostInfoDTO.setModuleType(hostWithModules.getModules().stream().map(it -> {
try {
return Long.parseLong(it.getModuleType());
} catch (Exception e) {
return 0L;
}
}).collect(Collectors.toList()));
applicationHostInfoDTO.setIpDesc(host.getHostName());
String os = host.getOs();
if (os != null && os.length() > 512) {
applicationHostInfoDTO.setOs(os.substring(0, 512));
} else {
applicationHostInfoDTO.setOs(os);
}
applicationHostInfoDTO.setOsType(host.getOsType());
ApplicationHostInfoDTO applicationHostInfoDTO = convertToHostInfoDTO(appId, hostWithModules);
applicationHostInfoDTOList.add(applicationHostInfoDTO);
} else {
log.info("bk_host_innerip is blank, ignore, hostId={},host={}", host.getHostId(),
Expand Down
1 change: 0 additions & 1 deletion ...anage/src/main/java/com/tencent/bk/job/manage/service/impl/sync/AppHostsUpdateHelper.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -86,7 +86,6 @@ public void waitForAppHostsUpdatingLock(Long appId, int intervalMills, int print

public boolean isAppHostsUpdating(Long appId) {
String updatingMachine = getAppHostsUpdatingMachine(appId);
// log.debug("{} is updating app {}", updatingMachine, appId);
return StringUtils.isNotBlank(updatingMachine);
}

Expand Down
4 changes: 2 additions & 2 deletions ...-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/sync/AppWatchThread.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@
import com.tencent.bk.job.manage.dao.ApplicationInfoDAO;
import com.tencent.bk.job.manage.service.ApplicationService;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.lang3.StringUtils;
import org.jooq.DSLContext;
import org.jooq.exception.DataAccessException;
import org.springframework.data.redis.core.RedisTemplate;
Expand Down Expand Up @@ -103,7 +103,7 @@ private void handleOneEvent(ResourceEvent<AppEventDetail> event) {
if (errorMessage.contains("Duplicate entry") && errorMessage.contains("PRIMARY")) {
// 若已存在则忽略
} else {
log.error("insertApp fail:appInfo={}", appInfoDTO, e);
log.error("insertApp fail:appInfo=" + appInfoDTO, e);
}
}
}
Expand Down
4 changes: 2 additions & 2 deletions ...e-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/sync/EventsHandler.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,7 @@ public void commitEvent(Long appId, ResourceEvent<T> event) {
this.appId = appId;
}
} catch (Exception e) {
log.warn("Fail to commitEvent:{}", event, e);
log.warn("Fail to commitEvent:" + event, e);
}
}

Expand All @@ -74,7 +74,7 @@ public void run() {
} catch (InterruptedException e) {
log.warn("queue.take interrupted", e);
} catch (Throwable t) {
log.warn("Fail to handleOneEvent:{}", event, t);
log.warn("Fail to handleOneEvent:" + event, t);
} finally {
if (queue.size() == 0) {
this.idle = true;
Expand Down
6 changes: 3 additions & 3 deletions ...ge/src/main/java/com/tencent/bk/job/manage/service/impl/sync/HostRelationWatchThread.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@
import com.tencent.bk.job.manage.model.dto.HostTopoDTO;
import com.tencent.bk.job.manage.service.SyncService;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.lang3.StringUtils;
import org.jooq.DSLContext;
import org.jooq.exception.DataAccessException;
import org.springframework.data.redis.core.RedisTemplate;
Expand Down Expand Up @@ -157,7 +157,7 @@ private void handleOneEvent(ResourceEvent<HostRelationEventDetail> event) {
log.debug("handle hostRelationEvent:" + watch.prettyPrint());
}
} catch (Throwable t) {
log.error("Fail to handle hostRelationEvent of appId {}, event:{}", appId, event, t);
log.error(String.format("Fail to handle hostRelationEvent of appId %d, event:%s", appId, event), t);
} finally {
appHostsUpdateHelper.endToUpdateAppHosts(appId);
}
Expand All @@ -176,7 +176,7 @@ private void handleOneEventIndeed(ResourceEvent<HostRelationEventDetail> event)
if (errorMessage.contains("Duplicate entry") && errorMessage.contains("PRIMARY")) {
// 若已存在则忽略
} else {
log.error("insertHostTopo fail:hostTopoInfo={}", hostTopoDTO, e);
log.error("insertHostTopo fail:hostTopoInfo=" + hostTopoDTO, e);
}
}
// 若主机存在需将拓扑信息同步至主机信息冗余字段
Expand Down
6 changes: 3 additions & 3 deletions ...job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/sync/HostWatchThread.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@
import com.tencent.bk.job.common.util.json.JsonUtils;
import com.tencent.bk.job.manage.dao.ApplicationHostDAO;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.lang3.StringUtils;
import org.jooq.DSLContext;
import org.jooq.exception.DataAccessException;
import org.springframework.data.redis.core.RedisTemplate;
Expand Down Expand Up @@ -145,7 +145,7 @@ private void handleOneEventRelatedToApp(ResourceEvent<HostEventDetail> event) {
appHostsUpdateHelper.waitAndStartAppHostsUpdating(appId);
handleOneEventIndeed(event);
} catch (Throwable t) {
log.error("Fail to handle hostEvent of appId {}, event:{}", appId, event, t);
log.error(String.format("Fail to handle hostEvent of appId %d, event:%s", appId, event), t);
} finally {
appHostsUpdateHelper.endToUpdateAppHosts(appId);
}
Expand Down Expand Up @@ -205,7 +205,7 @@ private void handleOneEventIndeed(ResourceEvent<HostEventDetail> event) {
if (errorMessage.contains("Duplicate entry") && errorMessage.contains("PRIMARY")) {
// 若已存在则忽略
} else {
log.error("insertHost fail:hostInfo={}", hostInfoDTO, e);
log.error("insertHost fail:hostInfo=" + hostInfoDTO, e);
}
}
}
Expand Down
30 changes: 16 additions & 14 deletions ...job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/sync/SyncServiceImpl.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@
import com.tencent.bk.job.manage.service.ApplicationService;
import com.tencent.bk.job.manage.service.SyncService;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.lang3.StringUtils;
import org.jooq.DSLContext;
import org.jooq.exception.DataAccessException;
import org.springframework.beans.factory.annotation.Autowired;
Expand Down Expand Up @@ -328,21 +328,21 @@ public Long syncApp() {
try {
addAppToDb(applicationInfoDTO, Collections.emptyList());
} catch (Throwable t) {
log.error("FATAL: insertApp fail:appId={}", applicationInfoDTO.getId(), t);
log.error("FATAL: insertApp fail:appId=" + applicationInfoDTO.getId(), t);
}
});
updateList.forEach(applicationInfoDTO -> {
try {
applicationInfoDAO.updateAppInfo(dslContext, applicationInfoDTO);
} catch (Throwable t) {
log.error("FATAL: updateApp fail:appId={}", applicationInfoDTO.getId(), t);
log.error("FATAL: updateApp fail:appId=" + applicationInfoDTO.getId(), t);
}
});
deleteList.forEach(applicationInfoDTO -> {
try {
deleteAppFromDb(applicationInfoDTO);
} catch (Throwable t) {
log.error("FATAL: deleteApp fail:appId={}", applicationInfoDTO.getId(), t);
log.error("FATAL: deleteApp fail:appId=" + applicationInfoDTO.getId(), t);
}
});
log.info(Thread.currentThread().getName() + ":Finished:sync app from cc");
Expand Down Expand Up @@ -427,7 +427,7 @@ private Pair<Long, Long> syncAppHostsAtOnce(ApplicationInfoDTO applicationInfoDT
appHostsUpdateHelper.waitAndStartAppHostsUpdating(appId);
return syncAppHostsIndeed(applicationInfoDTO);
} catch (Throwable t) {
log.error("Fail to syncAppHosts of appId {}", appId, t);
log.error("Fail to syncAppHosts of appId " + appId, t);
return null;
} finally {
appHostsUpdateHelper.endToUpdateAppHosts(appId);
Expand Down Expand Up @@ -549,7 +549,7 @@ public Long syncHost() {
writeToDBTimeConsuming += timeConsumingPair.getSecond();
} catch (Throwable t) {
appHostFailQueue.add(Pair.of(applicationInfoDTO, MAX_RETRY_COUNT));
log.error("syncHost of app fail:appId={}", applicationInfoDTO.getId(), t);
log.error("syncHost of app fail:appId=" + applicationInfoDTO.getId(), t);
}
}
log.info(Thread.currentThread().getName() + ":Finished:sync host from cc," +
Expand Down Expand Up @@ -739,7 +739,7 @@ public Long syncAgentStatus() {
gseInterfaceTimeConsuming += timeConsumingPair.getFirst();
writeToDBTimeConsuming += timeConsumingPair.getSecond();
} catch (Throwable t) {
log.error("syncAgentStatus of app fail:appId={}", applicationInfoDTO.getId(), t);
log.error("syncAgentStatus of app fail:appId=" + applicationInfoDTO.getId(), t);
}
}
log.info(Thread.currentThread().getName() + ":Finished:sync agentStatus from GSE," +
Expand Down Expand Up @@ -835,7 +835,7 @@ private void updateHostsInApp(Long appId, List<ApplicationHostInfoDTO> updateLis
notChangeCount += 1;
}
} catch (Throwable t) {
log.error("updateHost fail:appId={},hostInfo={}", appId, hostInfoDTO, t);
log.error(String.format("updateHost fail:appId=%d,hostInfo=%s", appId, hostInfoDTO), t);
errorCount += 1;
errorHostIds.add(hostInfoDTO.getHostId());
}
Expand Down Expand Up @@ -900,22 +900,24 @@ private boolean insertOrUpdateOneAppHost(Long appId, ApplicationHostInfoDTO info
} catch (DataAccessException e) {
String errorMessage = e.getMessage();
if (errorMessage.contains("Duplicate entry") && errorMessage.contains("PRIMARY")) {
log.warn("insertHost fail, try to update:Duplicate entry:appId={},insert hostInfo={}, old " +
"hostInfo={}", appId, infoDTO,
applicationHostDAO.getHostById(infoDTO.getHostId()), e);
log.warn(String.format(
"insertHost fail, try to update:Duplicate entry:appId=%d," +
"insert hostInfo=%s, old " +
"hostInfo=%s", appId, infoDTO,
applicationHostDAO.getHostById(infoDTO.getHostId())), e);
try {
// 插入失败了就应当更新，以后来的数据为准
applicationHostDAO.updateAppHostInfoByHostId(dslContext, appId, infoDTO);
} catch (Throwable t) {
log.error("update after insert fail:appId={},hostInfo={}", appId, infoDTO, t);
log.error(String.format("update after insert fail:appId=%d,hostInfo=%s", appId, infoDTO), t);
return false;
}
} else {
log.error("insertHost fail:appId={},hostInfo={}", appId, infoDTO, e);
log.error(String.format("insertHost fail:appId=%d,hostInfo=%s", appId, infoDTO), e);
return false;
}
} catch (Throwable t) {
log.error("insertHost fail:appId={},hostInfo={}", appId, infoDTO, t);
log.error(String.format("insertHost fail:appId=%d,hostInfo=%s", appId, infoDTO), t);
return false;
}
return true;
Expand Down

0 comments on commit dd7ad1a

Please sign in to comment.