Skip to content

Commit

Permalink
HPCC-31028 Use cost_type internally for consistency
Browse files Browse the repository at this point in the history
Signed-off-by: Shamser Ahmed <shamser.ahmed@lexisnexis.com>
  • Loading branch information
shamser committed Dec 18, 2023
1 parent 597fd83 commit 302e29c
Show file tree
Hide file tree
Showing 14 changed files with 46 additions and 46 deletions.
32 changes: 16 additions & 16 deletions dali/base/dadfs.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -191,18 +191,18 @@ static IPropertyTree *getCostPropTree(const char *cluster)
}
}

extern da_decl double calcFileAtRestCost(const char * cluster, double sizeGB, double fileAgeDays)
extern da_decl cost_type calcFileAtRestCost(const char * cluster, double sizeGB, double fileAgeDays)
{
Owned<const IPropertyTree> costPT = getCostPropTree(cluster);

if (costPT==nullptr)
return 0.0;
double atRestPrice = costPT->getPropReal("@storageAtRest", 0.0);
double storageCostDaily = atRestPrice * 12 / 365;
return storageCostDaily * sizeGB * fileAgeDays;
return money2cost_type(storageCostDaily * sizeGB * fileAgeDays);
}

extern da_decl double calcFileAccessCost(const char * cluster, __int64 numDiskWrites, __int64 numDiskReads)
extern da_decl cost_type calcFileAccessCost(const char * cluster, __int64 numDiskWrites, __int64 numDiskReads)
{
Owned<const IPropertyTree> costPT = getCostPropTree(cluster);

Expand All @@ -211,10 +211,10 @@ extern da_decl double calcFileAccessCost(const char * cluster, __int64 numDiskWr
constexpr int accessPriceScalingFactor = 10000; // read/write pricing based on 10,000 operations
double readPrice = costPT->getPropReal("@storageReads", 0.0);
double writePrice = costPT->getPropReal("@storageWrites", 0.0);
return (readPrice * numDiskReads / accessPriceScalingFactor) + (writePrice * numDiskWrites / accessPriceScalingFactor);
return money2cost_type((readPrice * numDiskReads / accessPriceScalingFactor) + (writePrice * numDiskWrites / accessPriceScalingFactor));
}

extern da_decl double calcFileAccessCost(IDistributedFile *f, __int64 numDiskWrites, __int64 numDiskReads)
extern da_decl cost_type calcFileAccessCost(IDistributedFile *f, __int64 numDiskWrites, __int64 numDiskReads)
{
StringBuffer clusterName;
// Should really specify the cluster number too, but this is the best we can do for now
Expand Down Expand Up @@ -4942,7 +4942,7 @@ protected: friend class CDistributedFilePart;
else
return false;
}
virtual void getCost(const char * cluster, double & atRestCost, double & accessCost) override
virtual void getCost(const char * cluster, cost_type & atRestCost, cost_type & accessCost) override
{
CDateTime dt;
getModificationTime(dt);
Expand All @@ -4956,7 +4956,7 @@ protected: friend class CDistributedFilePart;
if (hasReadWriteCostFields(*attrs))
{
// Newer files have readCost and writeCost attributes
accessCost = cost_type2money(attrs->getPropInt64(getDFUQResultFieldName(DFUQRFreadCost)) + attrs->getPropInt64(getDFUQResultFieldName(DFUQRFwriteCost)));
accessCost = attrs->getPropInt64(getDFUQResultFieldName(DFUQRFreadCost)) + attrs->getPropInt64(getDFUQResultFieldName(DFUQRFwriteCost));
}
else
{
Expand Down Expand Up @@ -7041,12 +7041,12 @@ class CDistributedSuperFile: public CDistributedFileBase<IDistributedSuperFile>
return false;
}

virtual void getCost(const char * cluster, double & atRestCost, double & accessCost) override
virtual void getCost(const char * cluster, cost_type & atRestCost, cost_type & accessCost) override
{
CriticalBlock block (sect);
ForEachItemIn(i,subfiles)
{
double tmpAtRestcost, tmpAccessCost;
cost_type tmpAtRestcost, tmpAccessCost;
IDistributedFile &f = subfiles.item(i);
f.getCost(cluster, tmpAtRestcost, tmpAccessCost);
atRestCost += tmpAtRestcost;
Expand Down Expand Up @@ -13454,14 +13454,14 @@ IPropertyTreeIterator *deserializeFileAttrIterator(MemoryBuffer& mb, unsigned nu
else
sizeDiskSize = file->getPropInt64(getDFUQResultFieldName(DFUQRForigsize), 0);
double sizeGB = sizeDiskSize / ((double)1024 * 1024 * 1024);
cost_type atRestCost = money2cost_type(calcFileAtRestCost(nodeGroup, sizeGB, fileAgeDays));
file->setPropInt64(getDFUQResultFieldName(DFUQRFatRestCost), atRestCost);
cost_type atRestCostType = calcFileAtRestCost(nodeGroup, sizeGB, fileAgeDays);
file->setPropInt64(getDFUQResultFieldName(DFUQRFatRestCost), atRestCostType);

// Dyamically calc and set the access cost field and for legacy files set read/write cost fields
cost_type accessCost = 0;
cost_type accessCostType = 0;
if (hasReadWriteCostFields(*file))
{
accessCost = file->getPropInt64(getDFUQResultFieldName(DFUQRFreadCost)) + file->getPropInt64(getDFUQResultFieldName(DFUQRFwriteCost));
accessCostType = file->getPropInt64(getDFUQResultFieldName(DFUQRFreadCost)) + file->getPropInt64(getDFUQResultFieldName(DFUQRFwriteCost));
}
else // Calc access cost from numDiskRead & numDiskWrites for Legacy files
{
Expand All @@ -13471,12 +13471,12 @@ IPropertyTreeIterator *deserializeFileAttrIterator(MemoryBuffer& mb, unsigned nu
cost_type legacyWriteCost = getLegacyWriteCost(*file, nodeGroup);
file->setPropInt64(getDFUQResultFieldName(DFUQRFwriteCost), legacyWriteCost);

accessCost = legacyReadCost + legacyWriteCost;
accessCostType = legacyReadCost + legacyWriteCost;
}
file->setPropInt64(getDFUQResultFieldName(DFUQRFaccessCost), accessCost);
file->setPropInt64(getDFUQResultFieldName(DFUQRFaccessCost), accessCostType);

// Dymically calc and set the total cost field
file->setPropInt64(getDFUQResultFieldName(DFUQRFcost), atRestCost + accessCost);
file->setPropInt64(getDFUQResultFieldName(DFUQRFcost), atRestCostType + accessCostType);
}

IPropertyTree *deserializeFileAttr(MemoryBuffer &mb, StringArray& nodeGroupFilter)
Expand Down
12 changes: 6 additions & 6 deletions dali/base/dadfs.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -433,7 +433,7 @@ interface IDistributedFile: extends IInterface
virtual bool getSkewInfo(unsigned &maxSkew, unsigned &minSkew, unsigned &maxSkewPart, unsigned &minSkewPart, bool calculateIfMissing) = 0;
virtual int getExpire(StringBuffer *expirationDate) = 0;
virtual void setExpire(int expireDays) = 0;
virtual void getCost(const char * cluster, double & atRestCost, double & accessCost) = 0;
virtual void getCost(const char * cluster, cost_type & atRestCost, cost_type & accessCost) = 0;
};


Expand Down Expand Up @@ -889,9 +889,9 @@ extern da_decl void ensureFileScope(const CDfsLogicalFileName &dlfn, unsigned ti

extern da_decl bool checkLogicalName(const char *lfn,IUserDescriptor *user,bool readreq,bool createreq,bool allowquery,const char *specialnotallowedmsg);

extern da_decl double calcFileAtRestCost(const char * cluster, double sizeGB, double fileAgeDays);
extern da_decl double calcFileAccessCost(const char * cluster, __int64 numDiskWrites, __int64 numDiskReads);
extern da_decl double calcFileAccessCost(IDistributedFile *f, __int64 numDiskWrites, __int64 numDiskReads);
extern da_decl cost_type calcFileAtRestCost(const char * cluster, double sizeGB, double fileAgeDays);
extern da_decl cost_type calcFileAccessCost(const char * cluster, __int64 numDiskWrites, __int64 numDiskReads);
extern da_decl cost_type calcFileAccessCost(IDistributedFile *f, __int64 numDiskWrites, __int64 numDiskReads);
constexpr bool defaultPrivilegedUser = true;
constexpr bool defaultNonPrivilegedUser = false;

Expand All @@ -910,7 +910,7 @@ inline cost_type getLegacyReadCost(const IPropertyTree & fileAttr, Source source
&& !isFileKey(fileAttr))
{
stat_type prevDiskReads = fileAttr.getPropInt64(getDFUQResultFieldName(DFUQRFnumDiskReads), 0);
return money2cost_type(calcFileAccessCost(source, 0, prevDiskReads));
return calcFileAccessCost(source, 0, prevDiskReads);
}
else
return 0;
Expand All @@ -922,7 +922,7 @@ inline cost_type getLegacyWriteCost(const IPropertyTree & fileAttr, Source sourc
if (!hasReadWriteCostFields(fileAttr) && fileAttr.hasProp(getDFUQResultFieldName(DFUQRFnumDiskWrites)))
{
stat_type prevDiskWrites = fileAttr.getPropInt64(getDFUQResultFieldName(DFUQRFnumDiskWrites), 0);
return money2cost_type(calcFileAccessCost(source, prevDiskWrites, 0));
return calcFileAccessCost(source, prevDiskWrites, 0);
}
else
return 0;
Expand Down
6 changes: 3 additions & 3 deletions dali/daliadmin/daliadmin.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -726,10 +726,10 @@ static void testDFSFile(IDistributedFile *legacyDfsFile, const char *logicalName
PROGLOG("expire: %d", expire);
try
{
double atRestCost, accessCost;
cost_type atRestCost, accessCost;
legacyDfsFile->getCost(clusterName.str(), atRestCost, accessCost);
PROGLOG("accessCost: %f", accessCost);
PROGLOG("atRestCost: %f", atRestCost);
PROGLOG("accessCost: %f", cost_type2money(accessCost));
PROGLOG("atRestCost: %f", cost_type2money(atRestCost));
}
catch(IException *e)
{
Expand Down
2 changes: 1 addition & 1 deletion dali/dfu/dfurun.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -143,7 +143,7 @@ class CDFUengine: public CInterface, implements IDFUengine
DaftProgress::setRange(sizeReadBefore,totalSize,_totalNodes);
progress->setTotalNodes(_totalNodes);
}
void setFileAccessCost(double fileAccessCost)
void setFileAccessCost(cost_type fileAccessCost)
{
progress->setFileAccessCost(fileAccessCost);
}
Expand Down
4 changes: 2 additions & 2 deletions dali/dfu/dfuwu.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -795,10 +795,10 @@ class CDFUprogress: public CLinkedDFUWUchild, implements IDFUprogress
CriticalBlock block(parent->crit);
queryRoot()->setProp("@subdone",str);
}
void setFileAccessCost(double fileAccessCost)
void setFileAccessCost(cost_type fileAccessCost)
{
CriticalBlock block(parent->crit);
queryRoot()->setPropReal("@fileAccessCost", fileAccessCost);
queryRoot()->setPropInt64("@fileAccessCost", fileAccessCost);
}
unsigned incPublisherTaskCount()
{
Expand Down
2 changes: 1 addition & 1 deletion dali/dfu/dfuwu.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -333,7 +333,7 @@ interface IDFUprogress: extends IConstDFUprogress
virtual void setSubInProgress(const char *str) = 0; // set sub-DFUWUs in progress
virtual void setSubDone(const char *str) = 0; // set sub-DFUWUs done
virtual void clearProgress() = 0;
virtual void setFileAccessCost(double fileAccessCost) = 0;
virtual void setFileAccessCost(cost_type fileAccessCost) = 0;
virtual unsigned incPublisherTaskCount() = 0;
};

Expand Down
2 changes: 1 addition & 1 deletion dali/ft/daft.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ interface IDaftProgress
{
virtual void onProgress(unsigned __int64 sizeDone, unsigned __int64 totalSize, unsigned numNodes, unsigned __int64 numReads, unsigned __int64 numWrites) = 0; // how much has been done
virtual void setRange(unsigned __int64 sizeReadBefore, unsigned __int64 totalSize, unsigned totalNodes) = 0; // how much has been done
virtual void setFileAccessCost(double fileAccessCost) = 0;
virtual void setFileAccessCost(cost_type fileAccessCost) = 0;
};

interface IDaftCopyProgress
Expand Down
4 changes: 2 additions & 2 deletions dali/ft/daftprogress.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ class DALIFT_API DaftProgress : public IDaftProgress
unsigned numNodes, unsigned __int64 numReads, unsigned __int64 numWrites) = 0;
virtual void displaySummary(const char * timeTaken, unsigned kbPerSecond) = 0;
virtual void setRange(unsigned __int64 sizeReadBefore, unsigned __int64 totalSize, unsigned _totalNodes);
virtual void setFileAccessCost(double fileAccessCost) = 0;
virtual void setFileAccessCost(cost_type fileAccessCost) = 0;
protected:
void formatTime(char * buffer, unsigned secs);

Expand All @@ -59,7 +59,7 @@ class DALIFT_API DemoProgress : public DaftProgress
unsigned __int64 scaledDone, unsigned __int64 scaledTotal, const char * scale,
unsigned kbPerSecondAve, unsigned kbPerSecondRate, unsigned numNodes);
virtual void displaySummary(const char * timeTaken, unsigned kbPerSecond);
virtual void setFileAccessCost(double fileAccessCost) {};
virtual void setFileAccessCost(cost_type fileAccessCost) {};
};

#endif
Expand Down
6 changes: 3 additions & 3 deletions dali/ft/filecopy.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -3387,7 +3387,7 @@ void FileSprayer::spray()
updateTargetProperties();

//Calculate and store file access cost
double fileAccessCost = 0.0;
cost_type fileAccessCost = 0;
if (distributedTarget)
{
StringBuffer cluster;
Expand Down Expand Up @@ -3592,7 +3592,7 @@ void FileSprayer::updateTargetProperties()

DistributedFilePropertyLock lock(distributedTarget);
IPropertyTree &curProps = lock.queryAttributes();
cost_type writeCost = money2cost_type(calcFileAccessCost(distributedTarget, totalNumWrites, 0));
cost_type writeCost = calcFileAccessCost(distributedTarget, totalNumWrites, 0);
curProps.setPropInt64(getDFUQResultFieldName(DFUQRFwriteCost), writeCost);
curProps.setPropInt64(getDFUQResultFieldName(DFUQRFnumDiskWrites), totalNumWrites);

Expand Down Expand Up @@ -3777,7 +3777,7 @@ void FileSprayer::updateTargetProperties()
{
IPropertyTree & fileAttr = distributedSource->queryAttributes();
cost_type legacyReadCost = getLegacyReadCost(fileAttr, distributedSource);
cost_type curReadCost = money2cost_type(calcFileAccessCost(distributedSource, 0, totalNumReads));
cost_type curReadCost = calcFileAccessCost(distributedSource, 0, totalNumReads);
distributedSource->addAttrValue(getDFUQResultFieldName(DFUQRFreadCost), legacyReadCost+curReadCost);
distributedSource->addAttrValue(getDFUQResultFieldName(DFUQRFnumDiskReads), totalNumReads);
}
Expand Down
6 changes: 3 additions & 3 deletions ecl/hthor/hthor.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -789,7 +789,7 @@ void CHThorDiskWriteActivity::publish()
{
StringBuffer clusterName;
file->getClusterName(0, clusterName);
diskAccessCost = money2cost_type(calcFileAccessCost(clusterName, numDiskWrites, 0));
diskAccessCost = calcFileAccessCost(clusterName, numDiskWrites, 0);
properties.setPropInt64(getDFUQResultFieldName(DFUQRFwriteCost), diskAccessCost);
}
file->attach(logicalName.get(), agent.queryCodeContext()->queryUserDescriptor());
Expand Down Expand Up @@ -1437,7 +1437,7 @@ void CHThorIndexWriteActivity::execute()

StringBuffer clusterName;
dfile->getClusterName(0, clusterName);
diskAccessCost = money2cost_type(calcFileAccessCost(clusterName, numDiskWrites, 0));
diskAccessCost = calcFileAccessCost(clusterName, numDiskWrites, 0);
properties.setPropInt64(getDFUQResultFieldName(DFUQRFwriteCost), diskAccessCost);
}
else
Expand Down Expand Up @@ -8543,7 +8543,7 @@ void CHThorDiskReadBaseActivity::closepart()
}
IPropertyTree & fileAttr = dFile->queryAttributes();
cost_type legacyReadCost = getLegacyReadCost(fileAttr, dFile);
cost_type curReadCost = money2cost_type(calcFileAccessCost(dFile, 0, curDiskReads));
cost_type curReadCost = calcFileAccessCost(dFile, 0, curDiskReads);

dFile->addAttrValue(getDFUQResultFieldName(DFUQRFreadCost), legacyReadCost + curReadCost);
dFile->addAttrValue(getDFUQResultFieldName(DFUQRFnumDiskReads), curDiskReads);
Expand Down
2 changes: 1 addition & 1 deletion esp/clients/ws_dfsclient/ws_dfsclient.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -235,7 +235,7 @@ class CServiceDistributedFileBase : public CSimpleInterfaceOf<INTERFACE>
virtual bool isExternal() const override { return false; }
virtual bool getSkewInfo(unsigned &maxSkew, unsigned &minSkew, unsigned &maxSkewPart, unsigned &minSkewPart, bool calculateIfMissing) override { return legacyDFSFile->getSkewInfo(maxSkew, minSkew, maxSkewPart, minSkewPart, calculateIfMissing); }
virtual int getExpire(StringBuffer *expirationDate) override { return legacyDFSFile->getExpire(expirationDate); }
virtual void getCost(const char * cluster, double & atRestCost, double & accessCost) override { legacyDFSFile->getCost(cluster, atRestCost, accessCost); }
virtual void getCost(const char * cluster, cost_type & atRestCost, cost_type & accessCost) override { legacyDFSFile->getCost(cluster, atRestCost, accessCost); }

// setters (change file meta data)
virtual void setPreferredClusters(const char *clusters) override { legacyDFSFile->setPreferredClusters(clusters); }
Expand Down
8 changes: 4 additions & 4 deletions esp/services/ws_dfu/ws_dfuService.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -2667,15 +2667,15 @@ void CWsDfuEx::doGetFileDetails(IEspContext &context, IUserDescriptor *udesc, co
}
if (version >= 1.60)
{
double atRestCost, accessCost;
cost_type atRestCost, accessCost;
df->getCost(cluster, atRestCost, accessCost);

if (version <= 1.61)
FileDetails.setCost(atRestCost+accessCost);
FileDetails.setCost(cost_type2money(atRestCost+accessCost));
else
{
FileDetails.setAccessCost(accessCost);
FileDetails.setAtRestCost(atRestCost);
FileDetails.setAccessCost(cost_type2money(accessCost));
FileDetails.setAtRestCost(cost_type2money(atRestCost));
}
}
if (version >= 1.65)
Expand Down
2 changes: 1 addition & 1 deletion esp/services/ws_fs/ws_fsService.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -353,7 +353,7 @@ static void DeepAssign(IEspContext &context, IConstDFUWorkUnit *src, IEspDFUWork
if(secs > 0)
dest.setSecsLeft(secs);
dest.setPercentDone(prog->getPercentDone());
dest.setFileAccessCost(prog->getFileAccessCost());
dest.setFileAccessCost(cost_type2money(prog->getFileAccessCost()));
}

IConstDFUoptions *options = src->queryOptions();
Expand Down
4 changes: 2 additions & 2 deletions thorlcr/graph/thgraphmaster.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -655,7 +655,7 @@ void CMasterActivity::updateFileReadCostStats()
stat_type curDiskReads = stats.getStatisticSum(StNumDiskReads);
IPropertyTree & fileAttr = file->queryAttributes();
cost_type legacyReadCost = getLegacyReadCost(fileAttr, file);
cost_type curReadCost = money2cost_type(calcFileAccessCost(file, 0, curDiskReads));
cost_type curReadCost = calcFileAccessCost(file, 0, curDiskReads);
file->addAttrValue(getDFUQResultFieldName(DFUQRFreadCost), legacyReadCost + curReadCost);
file->addAttrValue(getDFUQResultFieldName(DFUQRFnumDiskReads), curDiskReads);
return curReadCost;
Expand Down Expand Up @@ -705,7 +705,7 @@ void CMasterActivity::updateFileWriteCostStats(IFileDescriptor & fileDesc, IProp
assertex(fileDesc.numClusters()>=1);
StringBuffer clusterName;
fileDesc.getClusterGroupName(0, clusterName);// Note: calculating for 1st cluster. (Future: calc for >1 clusters)
cost_type writeCost = money2cost_type(calcFileAccessCost(clusterName, numDiskWrites, 0));
cost_type writeCost = calcFileAccessCost(clusterName, numDiskWrites, 0);
props.setPropInt64(getDFUQResultFieldName(DFUQRFwriteCost), writeCost);
diskAccessCost = writeCost;
}
Expand Down

0 comments on commit 302e29c

Please sign in to comment.