Skip to content

Commit 841f6f2

Browse files
committed
Merge remote-tracking branch 'origin/candidate-9.6.x' into candidate-9.6.0
Signed-off-by: Gordon Smith <[email protected]>
2 parents c6827bc + 0c66669 commit 841f6f2

File tree

11 files changed

+1095
-923
lines changed

11 files changed

+1095
-923
lines changed

.github/workflows/build-assets.yml

+1-1
Original file line numberDiff line numberDiff line change
@@ -248,7 +248,7 @@ jobs:
248248
sudo rm -f ${{ needs.preamble.outputs.folder_build }}/CMakeCache.txt
249249
sudo rm -rf ${{ needs.preamble.outputs.folder_build }}/CMakeFiles
250250
docker run --rm --mount ${{ needs.preamble.outputs.mount_platform }} --mount ${{ needs.preamble.outputs.mount_build }} ${{ steps.vars.outputs.docker_tag_candidate_base }} "\
251-
cmake -S /hpcc-dev/HPCC-Platform -B /hpcc-dev/build -DMAKE_DOCS_ONLY=ON -DUSE_NATIVE_LIBRARIES=ON -DDOCS_AUTO=ON -DDOC_LANGS=ALL && \
251+
cmake -S /hpcc-dev/HPCC-Platform -B /hpcc-dev/build ${{ needs.preamble.outputs.cmake_docker_config }} -DSIGN_MODULES_PASSPHRASE=${{ secrets.SIGN_MODULES_PASSPHRASE }} -DSIGN_MODULES_KEYID=${{ secrets.SIGN_MODULES_KEYID }} -DMAKE_DOCS_ONLY=ON -DUSE_NATIVE_LIBRARIES=ON -DDOCS_AUTO=ON -DDOC_LANGS=ALL && \
252252
cmake --build /hpcc-dev/build --parallel $(nproc) --target all"
253253
docker run --rm --mount ${{ needs.preamble.outputs.mount_platform }} --mount ${{ needs.preamble.outputs.mount_build }} ${{ steps.vars.outputs.docker_tag_candidate_base }} "cd /hpcc-dev/build/Release/docs/EN_US && zip ALL_HPCC_DOCS_EN_US-${{ needs.preamble.outputs.community_tag }}.zip *.pdf"
254254
docker run --rm --mount ${{ needs.preamble.outputs.mount_platform }} --mount ${{ needs.preamble.outputs.mount_build }} ${{ steps.vars.outputs.docker_tag_candidate_base }} "cd /hpcc-dev/build/Release/docs/PT_BR && zip ALL_HPCC_DOCS_PT_BR-${{ needs.preamble.outputs.community_tag }}.zip *.pdf"

dali/server/daldap.cpp

+5-5
Original file line numberDiff line numberDiff line change
@@ -124,15 +124,14 @@ class CDaliLdapConnection: implements IDaliLdapConnection, public CInterface
124124
}
125125
else
126126
{
127-
DBGLOG("NULL UserDescriptor in daldap.cpp getPermissions('%s')", key ? key : "NULL");
127+
DBGLOG("NULL UserDescriptor in daldap.cpp getPermissions('%s')", key);
128128
}
129129

130130
if (0 == username.length())
131131
{
132132
username.append(filesdefaultuser);
133133
decrypt(password, filesdefaultpassword);
134-
OWARNLOG("Missing credentials, injecting deprecated filesdefaultuser for request %s %s", nullText(key),
135-
nullText(obj));
134+
OWARNLOG("Missing credentials, injecting deprecated filesdefaultuser for request %s %s", key, nullText(obj));
136135
logNullUser(nullptr);
137136
}
138137

@@ -148,15 +147,15 @@ class CDaliLdapConnection: implements IDaliLdapConnection, public CInterface
148147
if (perm == SecAccess_Unavailable)
149148
{
150149
OWARNLOG("LDAP: getPermissions(%s) Unable to get perms for scope=%s user=%s, setting 'SecAccess_None'",
151-
nullText(key), nullText(obj), username.str());
150+
key, nullText(obj), username.str());
152151
perm = SecAccess_None;
153152
}
154153
unsigned taken = msTick() - start;
155154
#ifndef _DEBUG
156155
if (taken>100)
157156
#endif
158157
{
159-
PROGLOG("LDAP: getPermissions(%s) scope=%s user=%s returns %d in %d ms", nullText(key), nullText(obj),
158+
PROGLOG("LDAP: getPermissions(%s) scope=%s user=%s returns %d in %d ms", key, nullText(obj),
160159
username.str(), perm, taken);
161160
}
162161
if (auditflags & DALI_LDAP_AUDIT_REPORT)
@@ -176,6 +175,7 @@ class CDaliLdapConnection: implements IDaliLdapConnection, public CInterface
176175
}
177176
return perm;
178177
}
178+
179179
bool clearPermissionsCache(IUserDescriptor *udesc)
180180
{
181181
if (!ldapsecurity || ((getLDAPflags() & DLF_ENABLED) == 0))

docs/EN_US/ContainerizedHPCC/ContainerizedMods/ContainerLogging.xml

+2-2
Original file line numberDiff line numberDiff line change
@@ -527,8 +527,8 @@ myelk-kibana LoadBalancer 10.110.129.199 localhost 5601:31465/TCP 68m
527527
<para>Example use:</para>
528528

529529
<programlisting>helm install myhpcc hpcc/hpcc \
530-
-f HPCC-Platform/helm/examples/azure/log-analytics/loganalytics-hpcc-logaccess.yaml \
531-
-f HPCC-Platform/helm/examples/az</programlisting>
530+
-f HPCC-Platform/helm/examples/azure/log-analytics/loganalytics-hpcc-logaccess.yaml
531+
</programlisting>
532532
</sect3>
533533
</sect2>
534534

docs/EN_US/ContainerizedHPCC/ContainerizedMods/CustomConfig.xml

+152
Original file line numberDiff line numberDiff line change
@@ -619,6 +619,158 @@ eclagent:
619619
</listitem>
620620
</orderedlist>
621621
</sect2>
622+
623+
<sect2 id="threeiThorsOneQueue" role="nobrk">
624+
<title>Create a Custom Configuration Chart for Multiple Thors Listening
625+
to a Common Queue</title>
626+
627+
<para>In this section, we will create three Thors that listen to a
628+
common queue (in addition to their own queue). This provides the ability
629+
to define distinct Thor cluster configurations but allow them to form a
630+
single target behind a single queue. These clusters can be bound to
631+
certain node pools in different availability zones, if desired. You can
632+
use this example as a starting point and adjust the number of Thor
633+
clusters you want.</para>
634+
635+
<para>This is accomplished by defining additional auxiliary target
636+
queues for each Thor definition and using a common name as an auxiliary
637+
queue.</para>
638+
639+
<orderedlist>
640+
<listitem>
641+
<para>If you have not already added the HPCC Systems repository to
642+
the helm repository list, add it now.</para>
643+
644+
<para><programlisting>helm repo add hpcc https://hpcc-systems.github.io/helm-chart/</programlisting></para>
645+
646+
<para>If you have added it, update to the latest charts:</para>
647+
648+
<para><programlisting>helm repo update</programlisting></para>
649+
</listitem>
650+
651+
<listitem>
652+
<para>Create a new text file and name it <emphasis
653+
role="bold">threethorsonequeue.yaml</emphasis> and open it in a text
654+
editor.</para>
655+
656+
<para>You can use any text editor.</para>
657+
</listitem>
658+
659+
<listitem>
660+
<para>Open the default values file that you saved earlier
661+
(myvalues.yaml) in a text editor.</para>
662+
</listitem>
663+
664+
<listitem>
665+
<para>Copy the entire <emphasis role="bold">thor:</emphasis> section
666+
and paste it into the new threethorsonequeue.yaml file.</para>
667+
</listitem>
668+
669+
<listitem>
670+
<para>Copy the entire contents of the new yaml file, except the
671+
first line (thor:), and paste it at the end of the file <emphasis
672+
role="bold">twice</emphasis>.</para>
673+
674+
<para>This creates three - name: sections.</para>
675+
</listitem>
676+
677+
<listitem>
678+
<para>Edit the file in the following manner:</para>
679+
680+
<para><orderedlist>
681+
<listitem>
682+
<para>Give each Thor a unique value for <emphasis
683+
role="bold">name:</emphasis>.</para>
684+
685+
<para>In this example, we use <emphasis role="bold">thor1,
686+
thor2, </emphasis>and <emphasis
687+
role="bold">thor3</emphasis>.</para>
688+
</listitem>
689+
690+
<listitem>
691+
<para>Add an <emphasis role="bold">auxQueues:</emphasis> entry
692+
to each Thor block using a common name</para>
693+
694+
<para>In this example, we are using</para>
695+
696+
<para><emphasis role="bold">auxQueues: [ thorQ
697+
]</emphasis></para>
698+
</listitem>
699+
700+
<listitem>
701+
<para>Make sure the <emphasis role="bold">prefix:</emphasis>
702+
is the same in each Thor block.</para>
703+
</listitem>
704+
</orderedlist></para>
705+
</listitem>
706+
707+
<listitem>
708+
<para>Save the file and close the text editor.</para>
709+
710+
<para>The resulting threethorsonequeue.yaml file should look like
711+
this:</para>
712+
713+
<para><emphasis role="bold">Note:</emphasis> The comments have been
714+
removed to simplify the example:</para>
715+
716+
<para><programlisting>thor:
717+
- name: thor1
718+
auxQueues: [ thorQ ]
719+
maxGraphs: 2
720+
maxJobs: 2
721+
numWorkers: 4
722+
numWorkersPerPod: 2
723+
prefix: thor
724+
- name: thor2
725+
maxGraphs: 2
726+
maxJobs: 2
727+
numWorkers: 4
728+
numWorkersPerPod: 2
729+
prefix: thor
730+
auxQueues: [ thorQ ]
731+
- name: thor3
732+
maxGraphs: 2
733+
maxJobs: 2
734+
numWorkers: 4
735+
numWorkersPerPod: 2
736+
prefix: thor
737+
auxQueues: [ thorQ ]</programlisting></para>
738+
</listitem>
739+
</orderedlist>
740+
741+
<para><emphasis role="bold">Deploy using the new custom configuration
742+
chart.</emphasis></para>
743+
744+
<orderedlist>
745+
<listitem>
746+
<para>Open a terminal and navigate to the folder where you saved the
747+
threethorsonequeue.yaml file.</para>
748+
</listitem>
749+
750+
<listitem>
751+
<para>Deploy your HPCC Systems Platform, adding the new
752+
configuration to your command:</para>
753+
754+
<para><programlisting># If you have previously stopped your cluster
755+
756+
helm install mycluster hpcc/hpcc -f threethorsonequeue.yaml
757+
758+
# To upgrade without stopping
759+
760+
helm upgrade mycluster hpcc/hpcc -f threethorsonequeue.yaml
761+
</programlisting></para>
762+
</listitem>
763+
764+
<listitem>
765+
<para>After you confirm that your deployment is running, open ECL
766+
Watch.</para>
767+
768+
<para>You should see four Thor clusters available as Targets --
769+
thor1, thor2, thor3, and a fourth queue that all three Thors listen
770+
to-- thorQ.</para>
771+
</listitem>
772+
</orderedlist>
773+
</sect2>
622774
</sect1>
623775

624776
<sect1 id="CostTracking1">

esp/scm/ws_topology.ecm

+16-2
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,20 @@ ESPenum RoxieQueueFilter : string
2424
WorkunitsOnly("WorkunitsOnly")
2525
};
2626

27+
ESPenum TpMachineType : string
28+
{
29+
All("ALLNODES"),
30+
Thor("THORMACHINES"),
31+
Hole("HOLEMACHINES"),
32+
Roxie("ROXIEMACHINES"),
33+
Machines("MACHINES"),
34+
Available("AVAILABLEMACHINES"),
35+
DropZone("DROPZONE"),
36+
StandBy("STANDBYNNODE"),
37+
ThorSpare("THORSPARENODES"),
38+
HoleStandby("HOLESTANDBYNODES")
39+
};
40+
2741
// ===========================================================================
2842

2943
ESPStruct TpMachine
@@ -418,7 +432,7 @@ ESPresponse [exceptions_inline,encode(0)] TpClusterInfoResponse
418432
ESPrequest [] TpMachineQueryRequest
419433
{
420434
//Type of Node the parent is....
421-
string Type;
435+
ESPenum TpMachineType Type;
422436
string Cluster;
423437
string OldIP;
424438
//Path to Parent within Dali(Encoded to prevent back-engineering)
@@ -433,7 +447,7 @@ ESPresponse [exceptions_inline] TpMachineQueryResponse
433447
{
434448
bool EnablePreflightInfo;
435449
[min_ver("1.15")] bool HasThorSpareProcess;
436-
string Type;
450+
ESPenum TpMachineType Type;
437451
string Cluster;
438452
string OldIP;
439453
string LogDirectory;

esp/services/ws_topology/ws_topologyService.cpp

+6-5
Original file line numberDiff line numberDiff line change
@@ -1536,12 +1536,13 @@ bool CWsTopologyEx::onTpMachineQuery(IEspContext &context, IEspTpMachineQueryReq
15361536
const char* directory = req.getDirectory();
15371537

15381538
bool hasThorSpareProcess = false;
1539-
const char* type = req.getType();
1540-
if (!type || !*type || (strcmp(eqAllNodes,type) == 0))
1539+
const CTpMachineType type = req.getType();
1540+
// if (!type || !*type || (strcmp(eqAllNodes,type) == 0))
1541+
if (type == TpMachineType_Undefined || type == CTpMachineType_All )
15411542
{
1542-
m_TpWrapper.getClusterMachineList(version, eqTHORMACHINES, path, directory, MachineList, hasThorSpareProcess);
1543-
m_TpWrapper.getClusterMachineList(version, eqHOLEMACHINES, path, directory, MachineList, hasThorSpareProcess);
1544-
m_TpWrapper.getClusterMachineList(version, eqROXIEMACHINES,path, directory, MachineList, hasThorSpareProcess);
1543+
m_TpWrapper.getClusterMachineList(version, CTpMachineType_Thor, path, directory, MachineList, hasThorSpareProcess);
1544+
m_TpWrapper.getClusterMachineList(version, CTpMachineType_Hole, path, directory, MachineList, hasThorSpareProcess);
1545+
m_TpWrapper.getClusterMachineList(version, CTpMachineType_Roxie,path, directory, MachineList, hasThorSpareProcess);
15451546
}
15461547
else
15471548
{

esp/smc/SMCLib/TpContainer.cpp

+1-1
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,7 @@ const char* MSG_FAILED_GET_ENVIRONMENT_INFO = "Failed to get environment informa
4141
//////////////////////////////////////////////////////////////////////
4242

4343
void CTpWrapper::getClusterMachineList(double clientVersion,
44-
const char* ClusterType,
44+
CTpMachineType ClusterType,
4545
const char* ClusterPath,
4646
const char* ClusterDirectory,
4747
IArrayOf<IEspTpMachine> &MachineList,

esp/smc/SMCLib/TpWrapper.cpp

+15-15
Original file line numberDiff line numberDiff line change
@@ -55,7 +55,7 @@ IPropertyTree* CTpWrapper::getEnvironment(const char* xpath)
5555
}
5656

5757
void CTpWrapper::getClusterMachineList(double clientVersion,
58-
const char* ClusterType,
58+
CTpMachineType ClusterType,
5959
const char* ClusterPath,
6060
const char* ClusterDirectory,
6161
IArrayOf<IEspTpMachine> &MachineList,
@@ -68,7 +68,7 @@ void CTpWrapper::getClusterMachineList(double clientVersion,
6868
getAttPath(ClusterPath,path);
6969
set<string> machineNames; //used for checking duplicates
7070

71-
if (strcmp(eqTHORMACHINES,ClusterType) == 0)
71+
if (ClusterType == CTpMachineType_Thor)
7272
{
7373
bool multiSlaves = false;
7474
getMachineList(clientVersion, eqThorMasterProcess, path.str(), "", ClusterDirectory, MachineList);
@@ -81,41 +81,41 @@ void CTpWrapper::getClusterMachineList(double clientVersion,
8181
if (!checkMultiSlavesFlag(ClusterName) &&(count < MachineList.length()))
8282
hasThorSpareProcess = true;
8383
}
84-
else if (strcmp(eqHOLEMACHINES,ClusterType) == 0)
84+
else if (ClusterType == CTpMachineType_Hole)
8585
{
8686
getMachineList(clientVersion, eqHoleSocketProcess, path.str(), "", ClusterDirectory, MachineList);
8787
getMachineList(clientVersion, eqHoleProcessorProcess, path.str(), "", ClusterDirectory, MachineList);
8888
getMachineList(clientVersion, eqHoleControlProcess, path.str(), "", ClusterDirectory, MachineList);
8989
getMachineList(clientVersion, eqHoleCollatorProcess, path.str(), "", ClusterDirectory, MachineList);
9090
getMachineList(clientVersion, eqHoleStandbyProcess, path.str(), "", ClusterDirectory, MachineList);
9191
}
92-
else if (strcmp(eqROXIEMACHINES,ClusterType) == 0)
92+
else if (ClusterType == CTpMachineType_Roxie)
9393
{
9494
getMachineList(clientVersion, "RoxieServerProcess", path.str(), "", ClusterDirectory, MachineList, &machineNames);
9595
}
96-
else if (strcmp(eqMACHINES,ClusterType) == 0)
96+
else if (ClusterType == CTpMachineType_Machines)
9797
{
9898
//load a list of available machines.......
9999
getMachineList(clientVersion, "Computer", "/Environment/Hardware", "", ClusterDirectory, MachineList);
100100
}
101-
else if (strcmp("AVAILABLEMACHINES",ClusterType) == 0)
101+
else if (ClusterType == CTpMachineType_Available)
102102
{
103103
getMachineList(clientVersion, "Computer", "/Environment/Hardware", eqMachineAvailablability, ClusterDirectory, MachineList);
104104
}
105-
else if (strcmp("DROPZONE",ClusterType) == 0)
105+
else if (ClusterType == CTpMachineType_DropZone)
106106
{
107107
getDropZoneMachineList(clientVersion, false, MachineList);
108108
}
109-
else if (strcmp("STANDBYNNODE",ClusterType) == 0)
109+
else if (ClusterType == CTpMachineType_StandBy)
110110
{
111111
getThorSpareMachineList(clientVersion, ClusterName, ClusterDirectory, MachineList);
112112
getMachineList(clientVersion, eqHoleStandbyProcess, path.str(), "", ClusterDirectory, MachineList);
113113
}
114-
else if (strcmp("THORSPARENODES",ClusterType) == 0)
114+
else if (ClusterType == CTpMachineType_ThorSpare)
115115
{
116116
getThorSpareMachineList(clientVersion, ClusterName, ClusterDirectory, MachineList);
117117
}
118-
else if (strcmp("HOLESTANDBYNODES",ClusterType) == 0)
118+
else if (ClusterType == CTpMachineType_HoleStandby)
119119
{
120120
getMachineList(clientVersion, eqHoleStandbyProcess, path.str(), "", ClusterDirectory, MachineList);
121121
}
@@ -756,23 +756,23 @@ void CTpWrapper::queryTargetClusterProcess(double version, const char* processNa
756756
OS_TYPE os = OS_WINDOWS;
757757
unsigned int clusterTypeLen = strlen(clusterType);
758758
const char* childType = NULL;
759-
const char* clusterType0 = NULL;
759+
CTpMachineType clusterType0 = TpMachineType_Undefined;
760760
if (clusterTypeLen > 4)
761761
{
762762
if (!strnicmp(clusterType, "roxie", 4))
763763
{
764764
childType = "RoxieServerProcess[1]";
765-
clusterType0 = eqROXIEMACHINES;
765+
clusterType0 = CTpMachineType_Roxie;
766766
}
767767
else if (!strnicmp(clusterType, "thor", 4))
768768
{
769769
childType = "ThorMasterProcess";
770-
clusterType0 = eqTHORMACHINES;
770+
clusterType0 = CTpMachineType_Thor;
771771
}
772772
else
773773
{
774774
childType = "HoleControlProcess";
775-
clusterType0 = eqHOLEMACHINES;
775+
clusterType0 = CTpMachineType_Hole;
776776
}
777777
}
778778

@@ -800,7 +800,7 @@ void CTpWrapper::queryTargetClusterProcess(double version, const char* processNa
800800
}
801801
clusterInfo->setOS(os);
802802

803-
if (clusterType0 && *clusterType0)
803+
if (clusterType0 != TpMachineType_Undefined)
804804
{
805805
bool hasThorSpareProcess = false;
806806
IArrayOf<IEspTpMachine> machineList;

0 commit comments

Comments
 (0)