Skip to content

Commit

Permalink
HPCC-32847 Require permission to include logs in ZAP
Browse files Browse the repository at this point in the history 
* Require the same permission to include log files in a ZAP as would be
  required to view them.
* Vary the permission feature required to access logs to match the one
  used for the cluster deployment- containerized or bare metal.
* The WsLogAccess permission is used when pulling all containerized log
  files, including eclcc, because the remote log accessor is required to
  pull them all.
* The ClusterTopologyAccess permission is used when pulling bare metal
  log files.
* Because users are used to seeing errors in log file stubs instead of
  actual log entries, continue that behavior here. When access to log
  files is denied, insert a message into each (otherwise empty) log file
  included in the ZAP.
* Some code paths don't currently replace files with error messages, so
  to maintain current behavior and user expectations don't insert error
  messages into any file other than a log file.

Signed-off-by: Terrence Asselin <[email protected]>
  • Loading branch information
@asselitx
asselitx committed Jan 21, 2025
1 parent d897955 commit 168ddbb
Show file tree
Hide file tree
Showing 2 changed files with 68 additions and 34 deletions.
92 changes: 63 additions & 29 deletions esp/services/ws_workunits/ws_workunitsHelpers.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,12 @@

namespace ws_workunits {

#ifdef _CONTAINERIZED
static const char* LOG_ACCESS_FEATURE = "WsLogAccess";
#else
static const char* LOG_ACCESS_FEATURE = "ClusterTopologyAccess";
#endif

const char * const timerFilterText = "measure[time],source[global],depth[1,]"; // Does not include hthor subgraph timings
const char* zipFolder = "tempzipfiles" PATHSEPSTR;

Expand Down Expand Up @@ -3992,13 +3998,12 @@ void CWsWuFileHelper::cleanFolder(IFile* folder, bool removeFolder)
}

#ifndef _CONTAINERIZED
void CWsWuFileHelper::createProcessLogfile(IConstWorkUnit* cwu, WsWuInfo& winfo, const char* process, const char* path)
void CWsWuFileHelper::createProcessLogfile(IConstWorkUnit* cwu, WsWuInfo& winfo, const char* process, const char* path, bool hasLogsAccess)
{
BoolHash uniqueProcesses;
Owned<IPropertyTreeIterator> procs = cwu->getProcesses(process, NULL);
ForEach (*procs)
{
StringBuffer logSpec;
IPropertyTree& proc = procs->query();
const char* processName = proc.queryName();
if (isEmpty(processName))
Expand All @@ -4022,26 +4027,32 @@ void CWsWuFileHelper::createProcessLogfile(IConstWorkUnit* cwu, WsWuInfo& winfo,
pid.appendf("%d", proc.getPropInt("@pid"));

fileName.append("_eclagent.log");
if (!hasLogsAccess)
throw makeStringExceptionV(ECLWATCH_ACCESS_TO_FILE_DENIED, "Access to log files is denied. You are missing permission %s:READ - check with your system admin.", LOG_ACCESS_FEATURE);

winfo.getWorkunitEclAgentLog(processName, nullptr, pid.str(), mb, fileName.str());
}
else if (strieq(process, "Thor"))
{
fileName.append("_thormaster.log");
if (!hasLogsAccess)
throw makeStringExceptionV(ECLWATCH_ACCESS_TO_FILE_DENIED, "Access to log files is denied. You are missing permission %s:READ - check with your system admin.", LOG_ACCESS_FEATURE);

winfo.getWorkunitThorMasterLog(processName, nullptr, mb, fileName.str());
}
}
catch(IException* e)
{
StringBuffer s;
e->errorMessage(s);
IERRLOG("Error accessing Process Log file %s: %s", logSpec.str(), s.str());
IERRLOG("Error accessing Process Log file %s: %s", processName, s.str());
writeToFile(fileName.str(), s.length(), s.str());
e->Release();
}
}
}

void CWsWuFileHelper::createThorSlaveLogfile(IConstWorkUnit* cwu, WsWuInfo& winfo, const char* path)
void CWsWuFileHelper::createThorSlaveLogfile(IConstWorkUnit* cwu, WsWuInfo& winfo, const char* path, bool hasLogsAccess)
{
if (cwu->getWuidVersion() == 0)
return;
Expand All @@ -4056,6 +4067,18 @@ void CWsWuFileHelper::createThorSlaveLogfile(IConstWorkUnit* cwu, WsWuInfo& winf
return;
}

// Since there is not already a handler inserting exceptions into the thor slave log,
// make a simple case here to insert a message about permission failure and write it to
// a file that could be recognized as a thor slave log.
if (!hasLogsAccess)
{
StringBuffer msg;
msg.appendf("Access to log files is denied. You are missing permission %s:READ - check with your system admin.", LOG_ACCESS_FEATURE);
VStringBuffer fileName("%s%cthorslave.log", path, PATHSEPCHAR);
writeToFile(fileName.str(), msg.length(), msg.str());
return;
}

Owned<IThreadFactory> threadFactory = new CGetThorSlaveLogToFileThreadFactory();
Owned<IThreadPool> threadPool = createThreadPool("WsWuFileHelper GetThorSlaveLogToFile Thread Pool",
threadFactory, false, nullptr, thorSlaveLogThreadPoolSize, INFINITE);
Expand Down Expand Up @@ -4097,8 +4120,7 @@ void CWsWuFileHelper::createThorSlaveLogfile(IConstWorkUnit* cwu, WsWuInfo& winf
}
#endif

void CWsWuFileHelper::createZAPInfoFile(const char* url, const char* esp, const char* thor, const char* problemDesc,
const char* whatChanged, const char* timing, IConstWorkUnit* cwu, const char* tempDirName)
void CWsWuFileHelper::createZAPInfoFile(CWsWuZAPInfoReq &request, IConstWorkUnit* cwu, const char* tempDirName)
{
VStringBuffer fileName("%s%c%s.txt", tempDirName, PATHSEPCHAR, cwu->queryWuid());
Owned<IFileIOStream> outFile = createBufferedIOStreamFromFile(fileName.str(), IFOcreate);
Expand All @@ -4110,11 +4132,11 @@ void CWsWuFileHelper::createZAPInfoFile(const char* url, const char* esp, const
sb.append("User: ").append(cwu->queryUser()).append("\r\n");
sb.append("Build Version:").append(getBuildVersion()).append("\r\n");
sb.append("Cluster: ").append(cwu->queryClusterName()).append("\r\n");
sb.append("ESP: ").append(esp).append("\r\n");
if (!isEmptyString(url))
sb.append("URL: ").append(url).append("\r\n");
if (!isEmptyString(thor))
sb.append("Thor: ").append(thor).append("\r\n");
sb.append("ESP: ").append(request.esp).append("\r\n");
if (!isEmptyString(request.url))
sb.append("URL: ").append(request.url).append("\r\n");
if (!isEmptyString(request.thor))
sb.append("Thor: ").append(request.thor).append("\r\n");
outFile->write(sb.length(), sb.str());

//Exceptions/Warnings/Info
Expand Down Expand Up @@ -4148,9 +4170,9 @@ void CWsWuFileHelper::createZAPInfoFile(const char* url, const char* esp, const
}

//User provided Information
writeZAPWUInfoToIOStream(outFile, "Problem: ", problemDesc);
writeZAPWUInfoToIOStream(outFile, "What Changed: ", whatChanged);
writeZAPWUInfoToIOStream(outFile, "Timing: ", timing);
writeZAPWUInfoToIOStream(outFile, "Problem: ", request.problemDesc);
writeZAPWUInfoToIOStream(outFile, "What Changed: ", request.whatChanged);
writeZAPWUInfoToIOStream(outFile, "Timing: ", request.whereSlow);
}

void CWsWuFileHelper::writeZAPWUInfoToIOStream(IFileIOStream* outFile, const char* name, SCMStringBuffer& value)
Expand Down Expand Up @@ -4282,6 +4304,8 @@ void CWsWuFileHelper::readWULogToFile(const char *logFileName, WsWuInfo &winfo,
{
try
{
if (!zapLogFilterOptions.hasLogsAccess)
throw makeStringExceptionV(ECLWATCH_ACCESS_TO_FILE_DENIED, "Access to log files is denied. You are missing permission %s:READ - check with your system admin.", LOG_ACCESS_FEATURE);
winfo.readWorkunitComponentLogs(logFileName, zapLogFilterOptions);
}
catch(IException* e)
Expand Down Expand Up @@ -4522,36 +4546,36 @@ int CWsWuFileHelper::zipAFolder(const char* folder, bool gzip, const char* zipFi
void CWsWuFileHelper::createWUZAPFile(IEspContext& context, IConstWorkUnit* cwu, CWsWuZAPInfoReq& request,
const char* tempDirName, StringBuffer& zapFileName, StringBuffer& zipFileNameWithFullPath, unsigned _thorSlaveLogThreadPoolSize)
{
request.hasLogsAccess = context.validateFeatureAccess(LOG_ACCESS_FEATURE, SecAccess_Read, false);
setZAPReportName(request.zapFileName, cwu->queryWuid(), zapFileName);

zipFileNameWithFullPath.set(tempDirName).append(PATHSEPCHAR).append("zapreport.zip");
thorSlaveLogThreadPoolSize = _thorSlaveLogThreadPoolSize;

//create WU ZAP files
createZAPInfoFile(request.url.str(), request.esp.str(), request.thor.str(), request.problemDesc.str(), request.whatChanged.str(),
request.whereSlow.str(), cwu, tempDirName);
//create WU ZAP files
createZAPInfoFile(request, cwu, tempDirName);
createZAPECLQueryArchiveFiles(cwu, tempDirName);

WsWuInfo winfo(context, cwu);
createZAPWUXMLFile(winfo, tempDirName);
createZAPWUGraphProgressFile(request.wuid.str(), tempDirName);

StringArray localFiles;
createZAPWUQueryAssociatedFiles(cwu, tempDirName, localFiles);
createZAPWUQueryAssociatedFiles(cwu, tempDirName, localFiles, request.hasLogsAccess);
#ifndef _CONTAINERIZED
createProcessLogfile(cwu, winfo, "EclAgent", tempDirName);
createProcessLogfile(cwu, winfo, "Thor", tempDirName);
if (request.includeThorSlaveLog.isEmpty() || strieq(request.includeThorSlaveLog.str(), "on"))
createThorSlaveLogfile(cwu, winfo, tempDirName);
createProcessLogfile(cwu, winfo, "EclAgent", tempDirName, request.hasLogsAccess);
createProcessLogfile(cwu, winfo, "Thor", tempDirName, request.hasLogsAccess);
if (request.includeThorSlaveLog.isEmpty() || strieq(request.includeThorSlaveLog.str(), "on"))
createThorSlaveLogfile(cwu, winfo, tempDirName, request.hasLogsAccess);
#else
readWULogToFiles(cwu, winfo, tempDirName, request);
readWULogToFiles(cwu, winfo, tempDirName, request);
#endif

//Write out to ZIP file
zipAllZAPFiles(tempDirName, localFiles, request.password, zipFileNameWithFullPath);
}

void CWsWuFileHelper::createZAPWUQueryAssociatedFiles(IConstWorkUnit* cwu, const char* tempDirName, StringArray& localFiles)
void CWsWuFileHelper::createZAPWUQueryAssociatedFiles(IConstWorkUnit* cwu, const char* tempDirName, StringArray& localFiles, bool hasLogsAccess)
{
Owned<IConstWUQuery> query = cwu->getQuery();
if (!query)
Expand All @@ -4571,6 +4595,21 @@ void CWsWuFileHelper::createZAPWUQueryAssociatedFiles(IConstWorkUnit* cwu, const
RemoteFilename rfn;
SocketEndpoint ep(ip.str());
rfn.setPath(ep, name.str());
StringBuffer fileName(name.str());
getFileNameOnly(fileName, false);

VStringBuffer outFileName("%s%c%s", tempDirName, PATHSEPCHAR, fileName.str());

// Since users are only accustomed to seeing error messages replacing log file contents,
// special case an access denied error message for the one log file processed here.
if (!hasLogsAccess && endsWith(name.str(), ".eclcc.log"))
{
StringBuffer msg;
msg.appendf("Access to log files is denied. You are missing permission %s:READ - check with your system admin.", LOG_ACCESS_FEATURE);
writeToFile(outFileName.str(), msg.length(), msg.str());
continue;
}

if (rfn.isLocal())
{
localFiles.append(name.str());
Expand All @@ -4584,11 +4623,6 @@ void CWsWuFileHelper::createZAPWUQueryAssociatedFiles(IConstWorkUnit* cwu, const
continue;
}

StringBuffer fileName(name.str());
getFileNameOnly(fileName, false);

VStringBuffer outFileName("%s%c%s", tempDirName, PATHSEPCHAR, fileName.str());

OwnedIFile outFile = createIFile(outFileName);
if (!outFile)
{
Expand Down
10 changes: 5 additions & 5 deletions esp/services/ws_workunits/ws_workunitsHelpers.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -367,6 +367,7 @@ struct CWsWuZAPInfoReq
bool sendEmail, attachZAPReportToEmail;
bool includeRelatedLogs = true, includePerComponentLogs = false;
unsigned maxAttachmentSize, port;
bool hasLogsAccess = false;

WUComponentLogOptions logFilter;

Expand Down Expand Up @@ -888,15 +889,14 @@ class CWsWuFileHelper
IFile *createWorkingFolder(IEspContext &context, const char *wuid, const char *namePrefix,
StringBuffer &namePrefixStr, StringBuffer &folderName);

void createZAPInfoFile(const char *url, const char *espIP, const char *thorIP, const char *problemDesc,
const char *whatChanged, const char *timing, IConstWorkUnit *cwu, const char *pathNameStr);
void createZAPInfoFile(CWsWuZAPInfoReq &request, IConstWorkUnit *cwu, const char *pathNameStr);
void createZAPWUXMLFile(WsWuInfo &winfo, const char *pathNameStr);
void createZAPECLQueryArchiveFiles(IConstWorkUnit *cwu, const char *pathNameStr);
void createZAPWUQueryAssociatedFiles(IConstWorkUnit *cwu, const char *pathToCreate, StringArray &localFiles);
void createZAPWUQueryAssociatedFiles(IConstWorkUnit *cwu, const char *pathToCreate, StringArray &localFiles, bool hasLogsAccess);
void createZAPWUGraphProgressFile(const char *wuid, const char *pathNameStr);
#ifndef _CONTAINERIZED
void createProcessLogfile(IConstWorkUnit *cwu, WsWuInfo &winfo, const char *process, const char *path);
void createThorSlaveLogfile(IConstWorkUnit *cwu, WsWuInfo &winfo, const char *path);
void createProcessLogfile(IConstWorkUnit *cwu, WsWuInfo &winfo, const char *process, const char *path, bool hasLogsAccess);
void createThorSlaveLogfile(IConstWorkUnit *cwu, WsWuInfo &winfo, const char *path, bool hasLogsAccess);
#endif
LogAccessLogFormat getComponentLogFormatFromLogName(const char *log);
void readWULogToFiles(IConstWorkUnit *cwu, WsWuInfo &winfo, const char *path, CWsWuZAPInfoReq &zapLogFilterOptions);
Expand Down

0 comments on commit 168ddbb

Please sign in to comment.