Skip to content

Commit a56b663

Browse files
tomweber-sastomweber-sas
committed
enhance upload to validate file was uploaded after the fact
1 parent 3ff0ca4 commit a56b663

File tree

3 files changed

+147
-36
lines changed

3 files changed

+147
-36
lines changed

‎saspy/sasiohttp.py

Lines changed: 57 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1278,13 +1278,28 @@ def upload(self, localfile: str, remotefile: str, overwrite: bool = True, permis
12781278
"""
12791279
valid = self._sb.file_info(remotefile, quiet = True)
12801280

1281+
# check for non-exist, dir or existing file
12811282
if valid is None:
1282-
remf = remotefile
1283+
remf = remotefile
1284+
exist = False
12831285
else:
12841286
if valid == {}:
12851287
remf = remotefile + self._sb.hostsep + localfile.rpartition(os.sep)[2]
1288+
valid = self._sb.file_info(remf, quiet = True)
1289+
if valid is None:
1290+
exist = False
1291+
else:
1292+
if valid == {}:
1293+
return {'Success' : False,
1294+
'LOG' : "File "+str(remf)+" is an existing directory. Upload was stopped."}
1295+
else:
1296+
exist = True
1297+
if overwrite == False:
1298+
return {'Success' : False,
1299+
'LOG' : "File "+str(remf)+" exists and overwrite was set to False. Upload was stopped."}
12861300
else:
1287-
remf = remotefile
1301+
remf = remotefile
1302+
exist = True
12881303
if overwrite == False:
12891304
return {'Success' : False,
12901305
'LOG' : "File "+str(remotefile)+" exists and overwrite was set to False. Upload was stopped."}
@@ -1302,6 +1317,21 @@ def upload(self, localfile: str, remotefile: str, overwrite: bool = True, permis
13021317
ll = self.submit(code, 'text')
13031318
logf = ll['LOG']
13041319

1320+
# See if that worked cuz the next call will abend SAS if not; if lockdown/WatchDog ...
1321+
conn = self.sascfg.HTTPConn; conn.connect()
1322+
headers={"Accept":"application/vnd.sas.compute.fileref+json;application/json",
1323+
"Authorization":"Bearer "+self.sascfg._token}
1324+
conn.request('HEAD', self._uri_files+"/_sp_updn", headers=headers)
1325+
req = conn.getresponse()
1326+
status = req.status
1327+
resp = req.read()
1328+
conn.close()
1329+
1330+
if status > 299:
1331+
fd.close()
1332+
return {'Success' : False,
1333+
'LOG' : logf}
1334+
13051335
# GET Etag
13061336
conn = self.sascfg.HTTPConn; conn.connect()
13071337
headers={"Accept":"application/vnd.sas.compute.fileref+json;application/json",
@@ -1345,8 +1375,15 @@ def upload(self, localfile: str, remotefile: str, overwrite: bool = True, permis
13451375
status = req.status
13461376
resp = req.read()
13471377
conn.close()
1378+
fd.close()
13481379

1349-
code = "filename _sp_updn;"
1380+
ll = self.submit("filename _sp_updn;", 'text')
1381+
logf += ll['LOG']
1382+
1383+
if status > 299:
1384+
return {'Success' : False,
1385+
'LOG' : "Failure in upload. Status="+str(status)+"\nResponse="+str(resp.decode())+
1386+
"\n\n"+logf}
13501387
else:
13511388
logf = ''
13521389
code = """
@@ -1358,17 +1395,26 @@ def upload(self, localfile: str, remotefile: str, overwrite: bool = True, permis
13581395
run;
13591396
filename _sp_updn;
13601397
"""
1398+
ll = self.submit(code, 'text')
1399+
logf += ll['LOG']
1400+
fd.close()
13611401

1362-
ll = self.submit(code, 'text')
1363-
logf += ll['LOG']
1364-
fd.close()
1402+
valid2 = self._sb.file_info(remf, quiet = True)
13651403

1366-
if status > 299:
1367-
return {'Success' : False,
1368-
'LOG' : "Failure in upload. Status="+str(status)+"\nResponse="+str(resp.decode())}
1404+
if valid2 is not None:
1405+
if exist:
1406+
success = False
1407+
for key in valid.keys():
1408+
if valid[key] != valid2[key]:
1409+
success = True
1410+
break
1411+
else:
1412+
success = True
13691413
else:
1370-
return {'Success' : True,
1371-
'LOG' : logf}
1414+
success = False
1415+
1416+
return {'Success' : success,
1417+
'LOG' : logf}
13721418

13731419
def download(self, localfile: str, remotefile: str, overwrite: bool = True, **kwargs):
13741420
"""

‎saspy/sasioiom.py

Lines changed: 34 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1272,13 +1272,28 @@ def upload(self, localfile: str, remotefile: str, overwrite: bool = True, permis
12721272
"""
12731273
valid = self._sb.file_info(remotefile, quiet = True)
12741274

1275+
# check for non-exist, dir or existing file
12751276
if valid is None:
1276-
remf = remotefile
1277+
remf = remotefile
1278+
exist = False
12771279
else:
12781280
if valid == {}:
12791281
remf = remotefile + self._sb.hostsep + localfile.rpartition(os.sep)[2]
1282+
valid = self._sb.file_info(remf, quiet = True)
1283+
if valid is None:
1284+
exist = False
1285+
else:
1286+
if valid == {}:
1287+
return {'Success' : False,
1288+
'LOG' : "File "+str(remf)+" is an existing directory. Upload was stopped."}
1289+
else:
1290+
exist = True
1291+
if overwrite == False:
1292+
return {'Success' : False,
1293+
'LOG' : "File "+str(remf)+" exists and overwrite was set to False. Upload was stopped."}
12801294
else:
1281-
remf = remotefile
1295+
remf = remotefile
1296+
exist = True
12821297
if overwrite == False:
12831298
return {'Success' : False,
12841299
'LOG' : "File "+str(remotefile)+" exists and overwrite was set to False. Upload was stopped."}
@@ -1329,8 +1344,23 @@ def upload(self, localfile: str, remotefile: str, overwrite: bool = True, permis
13291344
ll2 = self.submit(code, 'text')
13301345
fd.close()
13311346

1332-
return {'Success' : True,
1333-
'LOG' : log1+ll2['LOG']}
1347+
logf = log1+ll2['LOG']
1348+
valid2 = self._sb.file_info(remf, quiet = True)
1349+
1350+
if valid2 is not None:
1351+
if exist:
1352+
success = False
1353+
for key in valid.keys():
1354+
if valid[key] != valid2[key]:
1355+
success = True
1356+
break
1357+
else:
1358+
success = True
1359+
else:
1360+
success = False
1361+
1362+
return {'Success' : success,
1363+
'LOG' : logf}
13341364

13351365
def download(self, localfile: str, remotefile: str, overwrite: bool = True, **kwargs):
13361366
"""

‎saspy/sasiostdio.py

Lines changed: 56 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -1352,13 +1352,28 @@ def upload(self, localfile: str, remotefile: str, overwrite: bool = True, permis
13521352
"""
13531353
valid = self._sb.file_info(remotefile, quiet = True)
13541354

1355+
# check for non-exist, dir or existing file
13551356
if valid is None:
1356-
remf = remotefile
1357+
remf = remotefile
1358+
exist = False
13571359
else:
13581360
if valid == {}:
13591361
remf = remotefile + self._sb.hostsep + localfile.rpartition(os.sep)[2]
1362+
valid = self._sb.file_info(remf, quiet = True)
1363+
if valid is None:
1364+
exist = False
1365+
else:
1366+
if valid == {}:
1367+
return {'Success' : False,
1368+
'LOG' : "File "+str(remf)+" is an existing directory. Upload was stopped."}
1369+
else:
1370+
exist = True
1371+
if overwrite == False:
1372+
return {'Success' : False,
1373+
'LOG' : "File "+str(remf)+" exists and overwrite was set to False. Upload was stopped."}
13601374
else:
1361-
remf = remotefile
1375+
remf = remotefile
1376+
exist = True
13621377
if overwrite == False:
13631378
return {'Success' : False,
13641379
'LOG' : "File "+str(remotefile)+" exists and overwrite was set to False. Upload was stopped."}
@@ -1370,7 +1385,7 @@ def upload(self, localfile: str, remotefile: str, overwrite: bool = True, permis
13701385
port = self.sascfg.rtunnel
13711386
host = 'localhost'
13721387
else:
1373-
return self._upload_client(localfile, remotefile, overwrite, permission, **kwargs)
1388+
return self._upload_client(localfile, remf, overwrite, permission, valid=valid, **kwargs)
13741389

13751390
try:
13761391
fd = open(localfile, 'rb')
@@ -1393,7 +1408,7 @@ def upload(self, localfile: str, remotefile: str, overwrite: bool = True, permis
13931408
filename sock;\n"""
13941409

13951410
self._asubmit(code, "text")
1396-
1411+
sleep(1)
13971412
sock = socks.socket()
13981413
sock.connect((host, port))
13991414

@@ -1438,31 +1453,36 @@ def upload(self, localfile: str, remotefile: str, overwrite: bool = True, permis
14381453
'LOG' : "Download was interrupted. Returning the SAS log:\n\n"+str(e)+"\n\n"+ll['LOG']}
14391454

14401455
ll = self.submit("", 'text')
1441-
return {'Success' : True,
1456+
1457+
valid2 = self._sb.file_info(remf, quiet = True)
1458+
1459+
if valid2 is not None:
1460+
if exist:
1461+
success = False
1462+
for key in valid.keys():
1463+
if valid[key] != valid2[key]:
1464+
success = True
1465+
break
1466+
else:
1467+
success = True
1468+
else:
1469+
success = False
1470+
1471+
return {'Success' : success,
14421472
'LOG' : ll['LOG']}
14431473

1444-
def _upload_client(self, localfile: str, remotefile: str, overwrite: bool = True, permission: str = '', **kwargs):
1474+
def _upload_client(self, localfile: str, remf: str, overwrite: bool = True, permission: str = '', **kwargs):
14451475
"""
14461476
This method uploads a local file to the SAS servers file system.
14471477
localfile - path to the local file to upload
1448-
remotefile - path to remote file to create or overwrite
1478+
remf - path to remote file to create or overwrite
14491479
overwrite - overwrite the output file if it exists?
14501480
permission - permissions to set on the new file. See SAS Filename Statement Doc for syntax
14511481
"""
1452-
valid = self._sb.file_info(remotefile, quiet = True)
1482+
valid = kwargs.pop('valid', None)
1483+
exist = False if valid is None else True
14531484

1454-
if valid is None:
1455-
remf = remotefile
1456-
else:
1457-
if valid == {}:
1458-
remf = remotefile + self._sb.hostsep + localfile.rpartition(os.sep)[2]
1459-
else:
1460-
remf = remotefile
1461-
if overwrite == False:
1462-
return {'Success' : False,
1463-
'LOG' : "File "+str(remotefile)+" exists and overwrite was set to False. Upload was stopped."}
1464-
1465-
port = kwargs.get('port', 0)
1485+
port = kwargs.get('port', 0)
14661486

14671487
if port==0 and self.sascfg.tunnel:
14681488
# we are using a tunnel; default to that port
@@ -1558,7 +1578,22 @@ def _upload_client(self, localfile: str, remotefile: str, overwrite: bool = True
15581578
'LOG' : "Download was interrupted. Returning the SAS log:\n\n"+str(e)+"\n\n"+ll['LOG']}
15591579

15601580
ll = self.submit("", 'text')
1561-
return {'Success' : True,
1581+
1582+
valid2 = self._sb.file_info(remf, quiet = True)
1583+
1584+
if valid2 is not None:
1585+
if exist:
1586+
success = False
1587+
for key in valid.keys():
1588+
if valid[key] != valid2[key]:
1589+
success = True
1590+
break
1591+
else:
1592+
success = True
1593+
else:
1594+
success = False
1595+
1596+
return {'Success' : success,
15621597
'LOG' : ll['LOG']}
15631598

15641599
def download(self, localfile: str, remotefile: str, overwrite: bool = True, **kwargs):

0 commit comments

Comments
 (0)