Skip to content

Commit baba148

Browse files
committed
Merge remote-tracking branch 'origin/candidate-9.6.x' into candidate-9.6.0
Signed-off-by: Jake Smith <[email protected]>
2 parents a2349af + bc2cf78 commit baba148

37 files changed

+416
-146
lines changed

.github/workflows/jirabot.yml

+120-55
Original file line numberDiff line numberDiff line change
@@ -23,81 +23,146 @@ jobs:
2323
python -VV
2424
python -m site
2525
python -m pip install --upgrade pip setuptools wheel
26-
python -m pip install --upgrade jira
26+
python -m pip install --upgrade atlassian-python-api
27+
python -m pip --version
2728
- name: "Run"
28-
env:
29+
env:
2930
JIRABOT_USERNAME : ${{ secrets.JIRABOT_USERNAME }}
3031
JIRABOT_PASSWORD : ${{ secrets.JIRABOT_PASSWORD }}
31-
JIRA_URL : ${{ secrets.JIRA_URL }}
32+
JIRA_URL : ${{ vars.JIRA_URL }}
3233
PULL_REQUEST_NUMBER : ${{ github.event.pull_request.number }}
3334
PULL_REQUEST_TITLE : ${{ github.event.pull_request.title }}
3435
PULL_REQUEST_AUTHOR_NAME : ${{ github.event.pull_request.user.login }}
3536
PULL_URL: ${{ github.event.pull_request.html_url }}
3637
COMMENTS_URL: ${{ github.event.pull_request.comments_url }}
3738
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
38-
39+
GHUB_JIRA_USER_MAP: ${{ vars.GHUB_JIRA_USER_MAP }}
40+
JIRA_ISSUE_PROPERTY_MAP: ${{ vars.JIRA_ISSUE_PROPERTY_MAP }}
41+
JIRA_ISSUE_TRANSITION_MAP: ${{ vars.JIRA_ISSUE_TRANSITION_MAP }}
3942
run: |
4043
import os
4144
import re
42-
from jira.client import JIRA
43-
45+
import time
46+
import sys
47+
import json
48+
from atlassian.jira import Jira
49+
50+
def updateIssue(jira, issue, prAuthor : str, transitionMap: dict, propertyMap: dict, pull_url: str) -> str:
51+
result = ''
52+
53+
issueName = issue['key']
54+
issueFields = issue['fields']
55+
56+
statusName = str(issueFields['status']['name'])
57+
transition = transitionMap.get(statusName, None)
58+
59+
if transition == None:
60+
print('Error: Unable to find transition for status: ' + statusName)
61+
elif transition != '':
62+
try:
63+
jira.issue_transition(issueName, transition)
64+
result += 'Workflow Transition: ' + transition + '\n'
65+
except Exception as error:
66+
transitions = jira.get_issue_transitions(issueName)
67+
result += 'Error: Transition: "' + transition + '" failed with: "' + str(error) + '" Valid transitions=' + str(transitions) + '\n'
68+
69+
prFieldName = propertyMap.get('pullRequestFieldName', 'customfield_10010')
70+
71+
if prFieldName in issueFields:
72+
currentPR = issueFields[prFieldName]
73+
else:
74+
print('Error: Unable to find pull request field with field name: ' + prFieldName)
75+
currentPR = None
76+
77+
if currentPR is None:
78+
jira.update_issue_field(issueName, {prFieldName: pull_url})
79+
result += 'Updated PR\n'
80+
elif currentPR is not None and currentPR != pull_url:
81+
result += 'Additional PR: ' + pull_url + '\n'
82+
83+
if prAuthor:
84+
assignee = issueFields['assignee']
85+
if assignee is None:
86+
assigneeId = ''
87+
assigneeEmail = ''
88+
else:
89+
assigneeId = assignee['accountId']
90+
assigneeEmail = assignee["emailAddress"]
91+
92+
prAuthorId = prAuthor["accountId"]
93+
prAuthorEmail = prAuthor["emailAddress"]
94+
if assigneeId is None or assigneeId == '':
95+
jira.assign_issue(issueName, prAuthorId)
96+
result += 'Assigning user: ' + prAuthorEmail + '\n'
97+
elif assigneeId != prAuthorId:
98+
result += 'Changing assignee from: ' + assigneeEmail + ' to: ' + prAuthorEmail + '\n'
99+
jira.assign_issue(issueName, prAuthorId)
100+
101+
return result
102+
44103
jirabot_user = os.environ['JIRABOT_USERNAME']
45104
jirabot_pass = os.environ['JIRABOT_PASSWORD']
46105
jira_url = os.environ['JIRA_URL']
47106
pr = os.environ['PULL_REQUEST_NUMBER']
48107
title = os.environ['PULL_REQUEST_TITLE']
49-
user = os.environ['PULL_REQUEST_AUTHOR_NAME']
50-
comments_url = os.environ['COMMENTS_URL']
108+
prAuthor = os.environ['PULL_REQUEST_AUTHOR_NAME']
51109
pull_url = os.environ['PULL_URL']
52110
github_token = os.environ['GITHUB_TOKEN']
53-
54-
print("%s %s %s" % (title, user, comments_url))
55-
status = ''
56-
issuem = re.search("(HPCC|HH|IDE|EPE|ML|JAPI)-[0-9]+", title)
111+
comments_url = os.environ['COMMENTS_URL']
112+
113+
print("%s %s %s" % (title, prAuthor, comments_url))
114+
result = ''
115+
issuem = re.search("(HPCC|HH|IDE|EPE|ML|HPCC4J|JAPI)-[0-9]+", title)
57116
if issuem:
58117
issue_name = issuem.group()
59-
if user == 'dehilsterlexis':
60-
user = 'dehilster'
61-
if user == 'kunalaswani':
62-
user = 'kunal.aswani'
63-
if user == 'timothyklemm':
64-
user = 'klemti01'
65-
if user == 'jpmcmu':
66-
user = 'mcmuja01'
67-
if user == 'asselitx':
68-
user = 'terrenceasselin'
69-
if user == 'jeclrsg':
70-
user = 'clemje01'
71-
if user == 'jackdelv':
72-
user = 'delvecja'
73-
options = {
74-
'server': jira_url
75-
}
76-
jira = JIRA(options=options, basic_auth=(jirabot_user, jirabot_pass))
77-
issue = jira.issue(issue_name)
78-
status = jira_url + '/browse/' + issue_name + '\\n'
79-
if False and issue.fields.status.name != 'Active' and issue.fields.status.name != 'Open' and issue.fields.status.name != 'New' and issue.fields.status.name != 'Discussing' and issue.fields.status.name != 'Awaiting Information':
80-
status += 'Jira not updated (state was not active or new)'
81-
elif issue.fields.customfield_10010 != None:
82-
if issue.fields.customfield_10010 != pull_url:
83-
status += 'Jira not updated (pull request "%s" already registered)' % issue.fields.customfield_10010
84-
else:
85-
status += 'This pull request is already registered'
86-
elif issue.fields.assignee is not None and issue.fields.assignee.name.lower() != user.lower():
87-
status += 'Jira not updated (user does not match)'
118+
119+
userDict = json.loads(os.environ['GHUB_JIRA_USER_MAP'])
120+
if not isinstance(userDict, dict):
121+
userDict = {}
122+
123+
if prAuthor in userDict:
124+
prAuthor = userDict.get(prAuthor)
125+
print('Mapped Github user to Jira user: ' + prAuthor)
126+
127+
jira = Jira(url=jira_url, username= jirabot_user, password= jirabot_pass, cloud=True)
128+
129+
jiraUser = None
130+
userSearchResults = jira.user_find_by_user_string(query=prAuthor)
131+
if userSearchResults and len(userSearchResults) > 0:
132+
jiraUser = userSearchResults[0]
133+
else:
134+
print('Error: Unable to find Jira user: ' + prAuthor + ' continuing without assigning')
135+
136+
if not jira.issue_exists(issue_name):
137+
sys.exit('Error: Unable to find Jira issue: ' + issue_name)
88138
else:
89-
if issue.fields.assignee is None:
90-
jira.assign_issue(issue, user)
91-
issue.update(fields={'customfield_10010': pull_url})
92139
issue = jira.issue(issue_name)
93-
try:
94-
transitions = jira.transitions(issue)
95-
jira.transition_issue(issue, '291') # Attach Pull Request
96-
except:
97-
status += 'Failed to set to merge pending: transitions=%s' % transitions
98-
status += 'Jira updated'
99-
print('curl -X POST %s -H "Content-Type: application/json" -H "Authorization: token %s" --data \'{ "body": "%s" }\'' % ( comments_url, github_token, status ))
100-
os.system('curl -X POST %s -H "Content-Type: application/json" -H "Authorization: token %s" --data \'{ "body": "%s" }\'' % ( comments_url, github_token, status ))
101-
102-
print(status)
103-
shell: python
140+
141+
result = 'Jirabot Action Result:\n'
142+
143+
transitionMap = json.loads(os.environ['JIRA_ISSUE_TRANSITION_MAP'])
144+
if not isinstance(transitionMap, dict):
145+
print('Error: JIRA_ISSUE_TRANSITION_MAP is not a valid JSON object, ignoring.')
146+
transitionMap = {}
147+
148+
jiraIssuePropertyMap = json.loads(os.environ['JIRA_ISSUE_PROPERTY_MAP'])
149+
if not isinstance(jiraIssuePropertyMap, dict):
150+
print('Error: JIRA_ISSUE_PROPERTY_MAP is not a valid JSON object, ignoring.')
151+
jiraIssuePropertyMap = {}
152+
153+
result += updateIssue(jira, issue, jiraUser, transitionMap, jiraIssuePropertyMap, pull_url)
154+
jira.issue_add_comment(issue_name, result)
155+
156+
result = 'Jira Issue: ' + jira_url + '/browse/' + issue_name + '\n\n' + result
157+
158+
# Escape the result for JSON
159+
result = json.dumps(result)
160+
161+
curlCommand = 'curl -X POST %s -H "Content-Type: application/json" -H "Authorization: token %s" --data \'{ "body": %s }\'' % ( comments_url, github_token, result )
162+
print(curlCommand)
163+
os.system(curlCommand)
164+
else:
165+
print('Unable to find Jira issue name in title')
166+
167+
print(result)
168+
shell: python

common/thorhelper/thorcommon.cpp

+24-12
Original file line numberDiff line numberDiff line change
@@ -1621,25 +1621,33 @@ class CRowStreamWriter : private IRowSerializerTarget, implements IExtRowWriter,
16211621
--nested;
16221622
}
16231623

1624+
virtual unsigned __int64 getStatistic(StatisticKind kind) override
1625+
{
1626+
return stream->getStatistic(kind);
1627+
}
16241628
};
16251629

16261630
#ifdef TRACE_CREATE
16271631
unsigned CRowStreamWriter::wrnum=0;
16281632
#endif
16291633

1634+
template<typename T>
1635+
static IFileIO * createCompressedFileWriter(T file, IRowInterfaces *rowIf, unsigned flags, ICompressor *compressor, size32_t compressorBlkSz)
1636+
{
1637+
size32_t fixedSize = rowIf->queryRowMetaData()->querySerializedDiskMeta()->getFixedSize();
1638+
if (fixedSize && TestRwFlag(flags, rw_grouped))
1639+
++fixedSize; // row writer will include a grouping byte
1640+
ICompressedFileIO *compressedFileIO = createCompressedFileWriter(file, fixedSize, TestRwFlag(flags, rw_extend), TestRwFlag(flags, rw_compressblkcrc), compressor, getCompMethod(flags));
1641+
if (compressorBlkSz)
1642+
compressedFileIO->setBlockSize(compressorBlkSz);
1643+
return compressedFileIO;
1644+
}
1645+
16301646
IExtRowWriter *createRowWriter(IFile *iFile, IRowInterfaces *rowIf, unsigned flags, ICompressor *compressor, size32_t compressorBlkSz)
16311647
{
16321648
OwnedIFileIO iFileIO;
16331649
if (TestRwFlag(flags, rw_compress))
1634-
{
1635-
size32_t fixedSize = rowIf->queryRowMetaData()->querySerializedDiskMeta()->getFixedSize();
1636-
if (fixedSize && TestRwFlag(flags, rw_grouped))
1637-
++fixedSize; // row writer will include a grouping byte
1638-
ICompressedFileIO *compressedFileIO = createCompressedFileWriter(iFile, fixedSize, TestRwFlag(flags, rw_extend), TestRwFlag(flags, rw_compressblkcrc), compressor, getCompMethod(flags));
1639-
if (compressorBlkSz)
1640-
compressedFileIO->setBlockSize(compressorBlkSz);
1641-
iFileIO.setown(compressedFileIO);
1642-
}
1650+
iFileIO.setown(createCompressedFileWriter(iFile, rowIf, flags, compressor, compressorBlkSz));
16431651
else
16441652
iFileIO.setown(iFile->open((flags & rw_extend)?IFOwrite:IFOcreate));
16451653
if (!iFileIO)
@@ -1648,18 +1656,22 @@ IExtRowWriter *createRowWriter(IFile *iFile, IRowInterfaces *rowIf, unsigned fla
16481656
return createRowWriter(iFileIO, rowIf, flags);
16491657
}
16501658

1651-
IExtRowWriter *createRowWriter(IFileIO *iFileIO, IRowInterfaces *rowIf, unsigned flags, size32_t compressorBlkSz)
1659+
IExtRowWriter *createRowWriter(IFileIO *iFileIO, IRowInterfaces *rowIf, unsigned flags, ICompressor *compressor, size32_t compressorBlkSz)
16521660
{
1661+
Owned<IFileIO> compressedFileIO;
16531662
if (TestRwFlag(flags, rw_compress))
1654-
throw MakeStringException(0, "Unsupported createRowWriter flags");
1663+
{
1664+
compressedFileIO.setown(createCompressedFileWriter(iFileIO, rowIf, flags, compressor, compressorBlkSz));
1665+
iFileIO = compressedFileIO.get();
1666+
}
16551667
Owned<IFileIOStream> stream;
16561668
if (TestRwFlag(flags, rw_buffered))
16571669
stream.setown(createBufferedIOStream(iFileIO));
16581670
else
16591671
stream.setown(createIOStream(iFileIO));
16601672
if (flags & rw_extend)
16611673
stream->seek(0, IFSend);
1662-
flags &= ~((unsigned)(rw_extend|rw_buffered));
1674+
flags &= ~((unsigned)(rw_extend|rw_buffered|COMP_MASK));
16631675
return createRowWriter(stream, rowIf, flags);
16641676
}
16651677

common/thorhelper/thorcommon.hpp

+2-1
Original file line numberDiff line numberDiff line change
@@ -172,6 +172,7 @@ interface IExtRowWriter: extends IRowWriter
172172
virtual offset_t getPosition() = 0;
173173
using IRowWriter::flush;
174174
virtual void flush(CRC32 *crcout) = 0;
175+
virtual unsigned __int64 getStatistic(StatisticKind kind) = 0;
175176
};
176177

177178
enum EmptyRowSemantics { ers_forbidden, ers_allow, ers_eogonly };
@@ -210,7 +211,7 @@ extern THORHELPER_API IExtRowStream *createRowStream(IFile *file, IRowInterfaces
210211
extern THORHELPER_API IExtRowStream *createRowStreamEx(IFile *file, IRowInterfaces *rowif, offset_t offset=0, offset_t len=(offset_t)-1, unsigned __int64 maxrows=(unsigned __int64)-1, unsigned flags=DEFAULT_RWFLAGS, IExpander *eexp=nullptr, ITranslator *translatorContainer=nullptr, IVirtualFieldCallback * _fieldCallback = nullptr);
211212
interface ICompressor;
212213
extern THORHELPER_API IExtRowWriter *createRowWriter(IFile *file, IRowInterfaces *rowIf, unsigned flags=DEFAULT_RWFLAGS, ICompressor *compressor=NULL, size32_t compressorBlkSz=0);
213-
extern THORHELPER_API IExtRowWriter *createRowWriter(IFileIO *fileIO, IRowInterfaces *rowIf, unsigned flags=DEFAULT_RWFLAGS, size32_t compressorBlkSz=0);
214+
extern THORHELPER_API IExtRowWriter *createRowWriter(IFileIO *iFileIO, IRowInterfaces *rowIf, unsigned flags=DEFAULT_RWFLAGS, ICompressor *compressor=nullptr, size32_t compressorBlkSz=0);
214215
extern THORHELPER_API IExtRowWriter *createRowWriter(IFileIOStream *strm, IRowInterfaces *rowIf, unsigned flags=DEFAULT_RWFLAGS); // strm should be unbuffered
215216

216217
interface THORHELPER_API IDiskMerger : extends IInterface

common/thorhelper/thorread.cpp

+1-1
Original file line numberDiff line numberDiff line change
@@ -1248,7 +1248,7 @@ class MarkupDiskRowReader : public ExternalFormatDiskRowReader, implements IXMLS
12481248
StringBuffer rowTag;
12491249

12501250
ThorActivityKind kind;
1251-
IXmlToRowTransformer *xmlTransformer;
1251+
IXmlToRowTransformer *xmlTransformer = nullptr;
12521252
Linked<IColumnProvider> lastMatch;
12531253
Owned<IXMLParse> xmlParser;
12541254

0 commit comments

Comments
 (0)