Skip to content

Commit

Permalink
enhance upload to validate file was uploaded after the fact
Browse files Browse the repository at this point in the history
  • Loading branch information
@tomweber-sas
tomweber-sas committed Oct 27, 2023
1 parent 3ff0ca4 commit a56b663
Show file tree
Hide file tree
Showing 3 changed files with 147 additions and 36 deletions.
68 changes: 57 additions & 11 deletions saspy/sasiohttp.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1278,13 +1278,28 @@ def upload(self, localfile: str, remotefile: str, overwrite: bool = True, permis
"""
valid = self._sb.file_info(remotefile, quiet = True)

# check for non-exist, dir or existing file
if valid is None:
remf = remotefile
remf = remotefile
exist = False
else:
if valid == {}:
remf = remotefile + self._sb.hostsep + localfile.rpartition(os.sep)[2]
valid = self._sb.file_info(remf, quiet = True)
if valid is None:
exist = False
else:
if valid == {}:
return {'Success' : False,
'LOG' : "File "+str(remf)+" is an existing directory. Upload was stopped."}
else:
exist = True
if overwrite == False:
return {'Success' : False,
'LOG' : "File "+str(remf)+" exists and overwrite was set to False. Upload was stopped."}
else:
remf = remotefile
remf = remotefile
exist = True
if overwrite == False:
return {'Success' : False,
'LOG' : "File "+str(remotefile)+" exists and overwrite was set to False. Upload was stopped."}
Expand All @@ -1302,6 +1317,21 @@ def upload(self, localfile: str, remotefile: str, overwrite: bool = True, permis
ll = self.submit(code, 'text')
logf = ll['LOG']

# See if that worked cuz the next call will abend SAS if not; if lockdown/WatchDog ...
conn = self.sascfg.HTTPConn; conn.connect()
headers={"Accept":"application/vnd.sas.compute.fileref+json;application/json",
"Authorization":"Bearer "+self.sascfg._token}
conn.request('HEAD', self._uri_files+"/_sp_updn", headers=headers)
req = conn.getresponse()
status = req.status
resp = req.read()
conn.close()

if status > 299:
fd.close()
return {'Success' : False,
'LOG' : logf}

# GET Etag
conn = self.sascfg.HTTPConn; conn.connect()
headers={"Accept":"application/vnd.sas.compute.fileref+json;application/json",
Expand Down Expand Up @@ -1345,8 +1375,15 @@ def upload(self, localfile: str, remotefile: str, overwrite: bool = True, permis
status = req.status
resp = req.read()
conn.close()
fd.close()

code = "filename _sp_updn;"
ll = self.submit("filename _sp_updn;", 'text')
logf += ll['LOG']

if status > 299:
return {'Success' : False,
'LOG' : "Failure in upload. Status="+str(status)+"\nResponse="+str(resp.decode())+
"\n\n"+logf}
else:
logf = ''
code = """
Expand All @@ -1358,17 +1395,26 @@ def upload(self, localfile: str, remotefile: str, overwrite: bool = True, permis
run;
filename _sp_updn;
"""
ll = self.submit(code, 'text')
logf += ll['LOG']
fd.close()

ll = self.submit(code, 'text')
logf += ll['LOG']
fd.close()
valid2 = self._sb.file_info(remf, quiet = True)

if status > 299:
return {'Success' : False,
'LOG' : "Failure in upload. Status="+str(status)+"\nResponse="+str(resp.decode())}
if valid2 is not None:
if exist:
success = False
for key in valid.keys():
if valid[key] != valid2[key]:
success = True
break
else:
success = True
else:
return {'Success' : True,
'LOG' : logf}
success = False

return {'Success' : success,
'LOG' : logf}

def download(self, localfile: str, remotefile: str, overwrite: bool = True, **kwargs):
"""
Expand Down
38 changes: 34 additions & 4 deletions saspy/sasioiom.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1272,13 +1272,28 @@ def upload(self, localfile: str, remotefile: str, overwrite: bool = True, permis
"""
valid = self._sb.file_info(remotefile, quiet = True)

# check for non-exist, dir or existing file
if valid is None:
remf = remotefile
remf = remotefile
exist = False
else:
if valid == {}:
remf = remotefile + self._sb.hostsep + localfile.rpartition(os.sep)[2]
valid = self._sb.file_info(remf, quiet = True)
if valid is None:
exist = False
else:
if valid == {}:
return {'Success' : False,
'LOG' : "File "+str(remf)+" is an existing directory. Upload was stopped."}
else:
exist = True
if overwrite == False:
return {'Success' : False,
'LOG' : "File "+str(remf)+" exists and overwrite was set to False. Upload was stopped."}
else:
remf = remotefile
remf = remotefile
exist = True
if overwrite == False:
return {'Success' : False,
'LOG' : "File "+str(remotefile)+" exists and overwrite was set to False. Upload was stopped."}
Expand Down Expand Up @@ -1329,8 +1344,23 @@ def upload(self, localfile: str, remotefile: str, overwrite: bool = True, permis
ll2 = self.submit(code, 'text')
fd.close()

return {'Success' : True,
'LOG' : log1+ll2['LOG']}
logf = log1+ll2['LOG']
valid2 = self._sb.file_info(remf, quiet = True)

if valid2 is not None:
if exist:
success = False
for key in valid.keys():
if valid[key] != valid2[key]:
success = True
break
else:
success = True
else:
success = False

return {'Success' : success,
'LOG' : logf}

def download(self, localfile: str, remotefile: str, overwrite: bool = True, **kwargs):
"""
Expand Down
77 changes: 56 additions & 21 deletions saspy/sasiostdio.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1352,13 +1352,28 @@ def upload(self, localfile: str, remotefile: str, overwrite: bool = True, permis
"""
valid = self._sb.file_info(remotefile, quiet = True)

# check for non-exist, dir or existing file
if valid is None:
remf = remotefile
remf = remotefile
exist = False
else:
if valid == {}:
remf = remotefile + self._sb.hostsep + localfile.rpartition(os.sep)[2]
valid = self._sb.file_info(remf, quiet = True)
if valid is None:
exist = False
else:
if valid == {}:
return {'Success' : False,
'LOG' : "File "+str(remf)+" is an existing directory. Upload was stopped."}
else:
exist = True
if overwrite == False:
return {'Success' : False,
'LOG' : "File "+str(remf)+" exists and overwrite was set to False. Upload was stopped."}
else:
remf = remotefile
remf = remotefile
exist = True
if overwrite == False:
return {'Success' : False,
'LOG' : "File "+str(remotefile)+" exists and overwrite was set to False. Upload was stopped."}
Expand All @@ -1370,7 +1385,7 @@ def upload(self, localfile: str, remotefile: str, overwrite: bool = True, permis
port = self.sascfg.rtunnel
host = 'localhost'
else:
return self._upload_client(localfile, remotefile, overwrite, permission, **kwargs)
return self._upload_client(localfile, remf, overwrite, permission, valid=valid, **kwargs)

try:
fd = open(localfile, 'rb')
Expand All @@ -1393,7 +1408,7 @@ def upload(self, localfile: str, remotefile: str, overwrite: bool = True, permis
filename sock;\n"""

self._asubmit(code, "text")

sleep(1)
sock = socks.socket()
sock.connect((host, port))

Expand Down Expand Up @@ -1438,31 +1453,36 @@ def upload(self, localfile: str, remotefile: str, overwrite: bool = True, permis
'LOG' : "Download was interrupted. Returning the SAS log:\n\n"+str(e)+"\n\n"+ll['LOG']}

ll = self.submit("", 'text')
return {'Success' : True,

valid2 = self._sb.file_info(remf, quiet = True)

if valid2 is not None:
if exist:
success = False
for key in valid.keys():
if valid[key] != valid2[key]:
success = True
break
else:
success = True
else:
success = False

return {'Success' : success,
'LOG' : ll['LOG']}

def _upload_client(self, localfile: str, remotefile: str, overwrite: bool = True, permission: str = '', **kwargs):
def _upload_client(self, localfile: str, remf: str, overwrite: bool = True, permission: str = '', **kwargs):
"""
This method uploads a local file to the SAS servers file system.
localfile - path to the local file to upload
remotefile - path to remote file to create or overwrite
remf - path to remote file to create or overwrite
overwrite - overwrite the output file if it exists?
permission - permissions to set on the new file. See SAS Filename Statement Doc for syntax
"""
valid = self._sb.file_info(remotefile, quiet = True)
valid = kwargs.pop('valid', None)
exist = False if valid is None else True

if valid is None:
remf = remotefile
else:
if valid == {}:
remf = remotefile + self._sb.hostsep + localfile.rpartition(os.sep)[2]
else:
remf = remotefile
if overwrite == False:
return {'Success' : False,
'LOG' : "File "+str(remotefile)+" exists and overwrite was set to False. Upload was stopped."}

port = kwargs.get('port', 0)
port = kwargs.get('port', 0)

if port==0 and self.sascfg.tunnel:
# we are using a tunnel; default to that port
Expand Down Expand Up @@ -1558,7 +1578,22 @@ def _upload_client(self, localfile: str, remotefile: str, overwrite: bool = True
'LOG' : "Download was interrupted. Returning the SAS log:\n\n"+str(e)+"\n\n"+ll['LOG']}

ll = self.submit("", 'text')
return {'Success' : True,

valid2 = self._sb.file_info(remf, quiet = True)

if valid2 is not None:
if exist:
success = False
for key in valid.keys():
if valid[key] != valid2[key]:
success = True
break
else:
success = True
else:
success = False

return {'Success' : success,
'LOG' : ll['LOG']}

def download(self, localfile: str, remotefile: str, overwrite: bool = True, **kwargs):
Expand Down

0 comments on commit a56b663

Please sign in to comment.