Skip to content

Commit 732ab98

Browse files
xiazhveraxiazhvera
authored
Device Advisor CI automation (#407)
Description of changes: Add the device advisor scripts to enable GitHub Actions to automatically run device advisor test on push GitHub Setting Changes: Added Repository secrets: AWS_DATEST_ACCESS_KEY_ID, AWS_DATEST_SECRET_ACCESS_KEY The secrets are set to aws-sdk-common-runtime user: IotSDKDeviceAdvisorCIAutomation By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license.
1 parent 1f06463 commit 732ab98

File tree

15 files changed

+330
-57
lines changed

15 files changed

+330
-57
lines changed

‎.builder/actions/build_samples.py

+26-10
Original file line numberDiff line numberDiff line change
@@ -6,10 +6,6 @@
66

77
class BuildSamples(Builder.Action):
88
def run(self, env):
9-
if env.args.cli_config['variables'].get('skip_samples', "0") != "0":
10-
print('skip_samples is defined. Skipping samples...')
11-
return
12-
139
# parse extra cmake configs
1410
parser = argparse.ArgumentParser()
1511
parser.add_argument('--cmake-extra', action='append', default=[])
@@ -31,17 +27,37 @@ def run(self, env):
3127
'samples/secure_tunneling/secure_tunnel',
3228
'samples/secure_tunneling/tunnel_notification',
3329
]
30+
da_samples = [
31+
'deviceadvisor/tests/mqtt_connect',
32+
'deviceadvisor/tests/mqtt_publish',
33+
'deviceadvisor/tests/mqtt_subscribe',
34+
'deviceadvisor/tests/shadow_update'
35+
]
36+
3437
for sample_path in samples:
3538
build_path = os.path.join('build', sample_path)
3639
steps.append(['cmake',
37-
f'-B{build_path}',
38-
f'-H{sample_path}',
39-
f'-DCMAKE_PREFIX_PATH={env.install_dir}',
40-
'-DCMAKE_BUILD_TYPE=RelWithDebInfo'])
40+
f'-B{build_path}',
41+
f'-H{sample_path}',
42+
f'-DCMAKE_PREFIX_PATH={env.install_dir}',
43+
'-DCMAKE_BUILD_TYPE=RelWithDebInfo'])
44+
# append extra cmake configs
45+
steps[-1].extend(cmd_args.cmake_extra)
46+
steps.append(['cmake',
47+
'--build', build_path,
48+
'--config', 'RelWithDebInfo'])
49+
50+
for sample_path in da_samples:
51+
build_path = os.path.join('build', sample_path)
52+
steps.append(['cmake',
53+
f'-B{build_path}',
54+
f'-H{sample_path}',
55+
f'-DCMAKE_PREFIX_PATH={env.install_dir}',
56+
'-DCMAKE_BUILD_TYPE=RelWithDebInfo'])
4157
# append extra cmake configs
4258
steps[-1].extend(cmd_args.cmake_extra)
4359
steps.append(['cmake',
44-
'--build', build_path,
45-
'--config', 'RelWithDebInfo'])
60+
'--build', build_path,
61+
'--config', 'RelWithDebInfo'])
4662

4763
return Builder.Script(steps)

‎.github/workflows/ci.yml

+4-4
Original file line numberDiff line numberDiff line change
@@ -13,9 +13,9 @@ env:
1313
PACKAGE_NAME: aws-iot-device-sdk-cpp-v2
1414
LINUX_BASE_IMAGE: ubuntu-16-x64
1515
RUN: ${{ github.run_id }}-${{ github.run_number }}
16-
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
17-
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
18-
AWS_REGION: us-east-1
16+
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_DATEST_ACCESS_KEY_ID }}
17+
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_DATEST_SECRET_ACCESS_KEY }}
18+
AWS_DEFAULT_REGION: us-east-1
1919

2020
jobs:
2121
linux-compat:
@@ -64,7 +64,7 @@ jobs:
6464
- name: Build ${{ env.PACKAGE_NAME }}
6565
run: |
6666
aws s3 cp s3://aws-crt-test-stuff/ci/${{ env.BUILDER_VERSION }}/linux-container-ci.sh ./linux-container-ci.sh && chmod a+x ./linux-container-ci.sh
67-
./linux-container-ci.sh ${{ env.BUILDER_VERSION }} aws-crt-${{ env.LINUX_BASE_IMAGE }} build -p ${{ env.PACKAGE_NAME }} --cmake-extra=-DBYO_CRYPTO=ON skip_samples=1
67+
./linux-container-ci.sh ${{ env.BUILDER_VERSION }} aws-crt-${{ env.LINUX_BASE_IMAGE }} build -p ${{ env.PACKAGE_NAME }} --cmake-extra=-DBYO_CRYPTO=ON --variant=skip_sample
6868
6969
linux-no-cpu-extensions:
7070
runs-on: ubuntu-latest

‎builder.json

+18-1
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,12 @@
77
"search_dirs": [
88
"crt"
99
],
10+
"env": {
11+
"DA_TOPIC": "test/da",
12+
"DA_SHADOW_PROPERTY": "datest",
13+
"DA_SHADOW_VALUE_SET": "ON",
14+
"DA_SHADOW_VALUE_DEFAULT": "OFF"
15+
},
1016
"hosts": {
1117
"manylinux": {
1218
"architectures": {
@@ -20,5 +26,16 @@
2026
"build_steps": [
2127
"build",
2228
"build-samples"
23-
]
29+
],
30+
"test_steps": [
31+
"python3 -m pip install boto3",
32+
"python3 deviceadvisor/script/DATestRun.py"],
33+
"variants" : {
34+
"skip_sample": {
35+
"!test_steps": [],
36+
"!build_steps": [
37+
"build"
38+
]
39+
}
40+
}
2441
}

‎deviceadvisor/script/DATestConfig.json

+19
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
{
2+
"tests" :[ "MQTT Connect", "MQTT Publish", "MQTT Subscribe", "Shadow Publish", "Shadow Update"],
3+
"test_suite_ids" :
4+
{
5+
"MQTT Connect" : "ejbdzmo3hf3v",
6+
"MQTT Publish" : "euw7favf6an4",
7+
"MQTT Subscribe" : "01o8vo6no7sd",
8+
"Shadow Publish" : "elztm2jebc1q",
9+
"Shadow Update" : "vuydgrbbbfce"
10+
},
11+
"test_exe_path" :
12+
{
13+
"MQTT Connect" : "mqtt_connect",
14+
"MQTT Publish" : "mqtt_publish",
15+
"MQTT Subscribe" : "mqtt_subscribe",
16+
"Shadow Publish" : "shadow_update",
17+
"Shadow Update" : "shadow_update"
18+
}
19+
}

‎deviceadvisor/script/DATestRun.py

+224
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,224 @@
1+
import boto3
2+
import uuid
3+
import json
4+
import os
5+
import subprocess
6+
import platform
7+
from time import sleep
8+
9+
##############################################
10+
# Cleanup Certificates and Things and created certificate and private key file
11+
def delete_thing_with_certi(thingName, certiId, certiArn):
12+
client.detach_thing_principal(
13+
thingName = thingName,
14+
principal = certiArn)
15+
client.update_certificate(
16+
certificateId =certiId,
17+
newStatus ='INACTIVE')
18+
client.delete_certificate(certificateId = certiId, forceDelete = True)
19+
client.delete_thing(thingName = thingName)
20+
os.remove(os.environ["DA_CERTI"])
21+
os.remove(os.environ["DA_KEY"])
22+
23+
24+
##############################################
25+
# Initialize variables
26+
# create aws clients
27+
client = boto3.client('iot')
28+
dataClient = boto3.client('iot-data')
29+
deviceAdvisor = boto3.client('iotdeviceadvisor')
30+
31+
# load test config
32+
f = open('deviceadvisor/script/DATestConfig.json')
33+
DATestConfig = json.load(f)
34+
f.close()
35+
36+
# create an temporary certificate/key file path
37+
certificate_path = os.path.join(os.getcwd(), 'certificate.pem.crt')
38+
key_path = os.path.join(os.getcwd(), 'private.pem.key')
39+
40+
# load environment variables requried for testing
41+
shadowProperty = os.environ['DA_SHADOW_PROPERTY']
42+
shadowDefault = os.environ['DA_SHADOW_VALUE_DEFAULT']
43+
44+
# test result
45+
test_result = {}
46+
47+
for test_name in DATestConfig['tests']:
48+
##############################################
49+
# create a test thing
50+
thing_name = "DATest_" + str(uuid.uuid4())
51+
try:
52+
# create_thing_response:
53+
# {
54+
# 'thingName': 'string',
55+
# 'thingArn': 'string',
56+
# 'thingId': 'string'
57+
# }
58+
print("[Device Advisor]Info: Started to create thing...")
59+
create_thing_response = client.create_thing(
60+
thingName=thing_name
61+
)
62+
os.environ["DA_THING_NAME"] = thing_name
63+
64+
except Exception as e:
65+
print("[Device Advisor]Error: Failed to create thing: " + thing_name)
66+
exit(-1)
67+
68+
69+
##############################################
70+
# create certificate and keys used for testing
71+
try:
72+
print("[Device Advisor]Info: Started to create certificate...")
73+
# create_cert_response:
74+
# {
75+
# 'certificateArn': 'string',
76+
# 'certificateId': 'string',
77+
# 'certificatePem': 'string',
78+
# 'keyPair':
79+
# {
80+
# 'PublicKey': 'string',
81+
# 'PrivateKey': 'string'
82+
# }
83+
# }
84+
create_cert_response = client.create_keys_and_certificate(
85+
setAsActive=True
86+
)
87+
# write certificate to file
88+
f = open(certificate_path, "w")
89+
f.write(create_cert_response['certificatePem'])
90+
f.close()
91+
92+
# write private key to file
93+
f = open(key_path, "w")
94+
f.write(create_cert_response['keyPair']['PrivateKey'])
95+
f.close()
96+
97+
# setup environment variable
98+
os.environ["DA_CERTI"] = certificate_path
99+
os.environ["DA_KEY"] = key_path
100+
101+
except:
102+
client.delete_thing(thingName = thing_name)
103+
print("[Device Advisor]Error: Failed to create certificate.")
104+
exit(-1)
105+
106+
##############################################
107+
# attach certification to thing
108+
try:
109+
print("[Device Advisor]Info: Attach certificate to test thing...")
110+
# attache the certificate to thing
111+
client.attach_thing_principal(
112+
thingName = thing_name,
113+
principal = create_cert_response['certificateArn']
114+
)
115+
116+
certificate_arn = create_cert_response['certificateArn']
117+
certificate_id = create_cert_response['certificateId']
118+
119+
except:
120+
delete_thing_with_certi(thing_name, certificate_id ,certificate_arn )
121+
print("[Device Advisor]Error: Failed to attach certificate.")
122+
exit(-1)
123+
124+
125+
##############################################
126+
# Run device advisor
127+
try:
128+
######################################
129+
# set default shadow, for shadow update, if the
130+
# shadow does not exists, update will fail
131+
payload_shadow = json.dumps(
132+
{
133+
"state": {
134+
"desired": {
135+
shadowProperty: shadowDefault
136+
},
137+
"reported": {
138+
shadowProperty: shadowDefault
139+
}
140+
}
141+
})
142+
shadow_response = dataClient.update_thing_shadow(
143+
thingName = thing_name,
144+
payload = payload_shadow)
145+
get_shadow_response = dataClient.get_thing_shadow(thingName = thing_name)
146+
# make sure shadow is created before we go to next step
147+
while(get_shadow_response is None):
148+
get_shadow_response = dataClient.get_thing_shadow(thingName = thing_name)
149+
150+
# start device advisor test
151+
# test_start_response
152+
# {
153+
# 'suiteRunId': 'string',
154+
# 'suiteRunArn': 'string',
155+
# 'createdAt': datetime(2015, 1, 1)
156+
# }
157+
print("[Device Advisor]Info: Start device advisor test: " + test_name)
158+
test_start_response = deviceAdvisor.start_suite_run(
159+
suiteDefinitionId=DATestConfig['test_suite_ids'][test_name],
160+
suiteRunConfiguration={
161+
'primaryDevice': {
162+
'thingArn': create_thing_response['thingArn'],
163+
},
164+
'parallelRun': True
165+
})
166+
167+
# get DA endpoint
168+
endpoint_response = deviceAdvisor.get_endpoint(
169+
thingArn = create_thing_response['thingArn']
170+
)
171+
os.environ['DA_ENDPOINT'] = endpoint_response['endpoint']
172+
173+
while True:
174+
# sleep for 1s every loop to avoid TooManyRequestsException
175+
sleep(1)
176+
test_result_responds = deviceAdvisor.get_suite_run(
177+
suiteDefinitionId=DATestConfig['test_suite_ids'][test_name],
178+
suiteRunId=test_start_response['suiteRunId']
179+
)
180+
# If the status is PENDING or the responds does not loaded, the test suite is still loading
181+
if (test_result_responds['status'] == 'PENDING' or
182+
len(test_result_responds['testResult']['groups']) == 0 or # test group has not been loaded
183+
len(test_result_responds['testResult']['groups'][0]['tests']) == 0 or #test case has not been loaded
184+
test_result_responds['testResult']['groups'][0]['tests'][0]['status'] == 'PENDING'):
185+
continue
186+
187+
# Start to run the test sample after the status turns into RUNNING
188+
elif (test_result_responds['status'] == 'RUNNING' and
189+
test_result_responds['testResult']['groups'][0]['tests'][0]['status'] == 'RUNNING'):
190+
try:
191+
exe_path = os.path.join("build/deviceadvisor/tests/",DATestConfig['test_exe_path'][test_name])
192+
# Windows and MAC/LINUX has a different build folder structure
193+
if platform.system() == 'Windows':
194+
exe_path = os.path.join(exe_path, "RelWithDebInfo",DATestConfig['test_exe_path'][test_name])
195+
else:
196+
exe_path = os.path.join(exe_path, DATestConfig['test_exe_path'][test_name])
197+
print("start to run" + exe_path)
198+
result = subprocess.run(exe_path, timeout = 60*2)
199+
except:
200+
continue
201+
202+
# If the test finalizing or store the test result
203+
elif (test_result_responds['status'] != 'RUNNING'):
204+
test_result[test_name] = test_result_responds['status']
205+
if(test_result[test_name] == "PASS"):
206+
delete_thing_with_certi(thing_name, certificate_id ,certificate_arn )
207+
break
208+
except Exception as e:
209+
print("[Device Advisor]Error: Failed to test: "+ test_name)
210+
211+
##############################################
212+
# print result and cleanup things
213+
print(test_result)
214+
failed = False
215+
for test in test_result:
216+
if(test_result[test] != "PASS" and
217+
test_result[test] != "PASS_WITH_WARNINGS"):
218+
print("[Device Advisor]Error: Test \"" + test + "\" Failed with status:" + test_result[test])
219+
failed = True
220+
if failed:
221+
# if the test failed, we dont clean the Thing so that we can track the error
222+
exit(-1)
223+
224+
exit(0)

‎deviceadvisor/tests/mqtt_connect/CMakeLists.txt

+1
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ project(mqtt_connect CXX)
55
file(GLOB SRC_FILES
66
"*.cpp"
77
"../utils/*.cpp"
8+
"../utils/*.h"
89
)
910

1011
add_executable(${PROJECT_NAME} ${SRC_FILES})

‎deviceadvisor/tests/mqtt_connect/main.cpp

+2-3
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,6 @@ int main()
2222
* Do the global initialization for the API.
2323
*/
2424
ApiHandle apiHandle;
25-
2625
String clientId(String("test-") + Aws::Crt::UUID().ToString());
2726

2827
/*********************** Parse Arguments ***************************/
@@ -37,7 +36,6 @@ int main()
3736
/*
3837
* Setup client configuration with the MqttClientConnectionConfigBuilder.
3938
*/
40-
4139
Aws::Iot::MqttClientConnectionConfigBuilder builder =
4240
Aws::Iot::MqttClientConnectionConfigBuilder(daVars.certificatePath.c_str(), daVars.keyPath.c_str());
4341
builder.WithEndpoint(daVars.endpoint);
@@ -79,7 +77,8 @@ int main()
7977
/*
8078
* Actually perform the connect dance.
8179
*/
82-
if (!connection->Connect(clientId.c_str(), false /*cleanSession*/, 1000 /*keepAliveTimeSecs*/))
80+
if (!connection->Connect(
81+
clientId.c_str(), false /*cleanSession*/, 1000 /*keepAliveTimeSecs*/, 6000 /*pingTimeoutMs*/))
8382
{
8483
exit(-1);
8584
}

‎deviceadvisor/tests/mqtt_publish/CMakeLists.txt

+1
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ project(mqtt_publish CXX)
55
file(GLOB SRC_FILES
66
"*.cpp"
77
"../utils/*.cpp"
8+
"../utils/*.h"
89
)
910

1011
add_executable(${PROJECT_NAME} ${SRC_FILES})

0 commit comments

Comments
 (0)