Skip to content

Commit

Permalink
get s3 file information in python
Browse files Browse the repository at this point in the history
  • Loading branch information
@sunyeongchoi
sunyeongchoi committed Dec 5, 2020
1 parent 855a210 commit 3633559
Show file tree
Hide file tree
Showing 3 changed files with 360 additions and 0 deletions.
108 changes: 108 additions & 0 deletions jupyter_test/.ipynb_checkpoints/S3 + python-checkpoint.ipynb
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,108 @@
{
"cells": [
{
"cell_type": "code",
"execution_count": 18,
"metadata": {},
"outputs": [],
"source": [
"import boto3"
]
},
{
"cell_type": "code",
"execution_count": 19,
"metadata": {},
"outputs": [],
"source": [
"s3 = boto3.resource('s3',\n",
" region_name = 'us-east-1',\n",
" aws_access_key_id = 'AKIAS4NQIUJ4WC3TUPHZ',\n",
" aws_secret_access_key = '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')"
]
},
{
"cell_type": "code",
"execution_count": 20,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"amplify-amplifytest-dev-103811-deployment\n",
"themightiestkpk\n"
]
}
],
"source": [
"for bucket in s3.buckets.all():\n",
" print(bucket.name)"
]
},
{
"cell_type": "code",
"execution_count": 24,
"metadata": {},
"outputs": [
{
"ename": "AttributeError",
"evalue": "'s3.ServiceResource' object has no attribute 'list_buckets'",
"output_type": "error",
"traceback": [
"\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
"\u001b[1;31mAttributeError\u001b[0m Traceback (most recent call last)",
"\u001b[1;32m<ipython-input-24-478a06e36e78>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[1;32mfor\u001b[0m \u001b[0mb\u001b[0m \u001b[1;32min\u001b[0m \u001b[0ms3\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mlist_buckets\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 2\u001b[0m \u001b[0mprint\u001b[0m \u001b[1;33m(\u001b[0m\u001b[0mb\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
"\u001b[1;31mAttributeError\u001b[0m: 's3.ServiceResource' object has no attribute 'list_buckets'"
]
}
],
"source": [
"for b in s3.list_buckets():\n",
" print (b)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.8.3"
}
},
"nbformat": 4,
"nbformat_minor": 4
}
251 changes: 251 additions & 0 deletions jupyter_test/S3 + python.ipynb
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,251 @@
{
"cells": [
{
"cell_type": "code",
"execution_count": 1,
"metadata": {},
"outputs": [],
"source": [
"import boto3"
]
},
{
"cell_type": "code",
"execution_count": 24,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"2020-12-05 12:07:06+00:00\n",
"제목 없음.png\n",
"2020-12-05 09:29:24+00:00\n",
"파일.PNG\n"
]
}
],
"source": [
"import boto3\n",
"from datetime import datetime, timezone\n",
"\n",
"today = datetime.now(timezone.utc)\n",
"\n",
"s3 = boto3.client('s3',\n",
" region_name = 'us-east-1',\n",
" aws_access_key_id = 'AKIAS4NQIUJ4WC3TUPHZ',\n",
" aws_secret_access_key = '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')\n",
"\n",
"objects = s3.list_objects(Bucket='themightiestkpk1')\n",
"\n",
"for o in objects[\"Contents\"]:\n",
" # 최초 수정 시간\n",
" print(o[\"LastModified\"])\n",
" # 파일 이름\n",
" print(o[\"Key\"])"
]
},
{
"cell_type": "code",
"execution_count": 32,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"https://themightiestkpk1.s3.amazonaws.com/%EC%A0%9C%EB%AA%A9%20%EC%97%86%EC%9D%8C.png?AWSAccessKeyId=AKIAS4NQIUJ4WC3TUPHZ&Signature=j4TwlZk%2BswkkWEXasboC0vzEa9M%3D&Expires=1607174691\n",
"제목 없음.png\n",
"2020-12-05 12:07:06+00:00\n",
"https://themightiestkpk1.s3.amazonaws.com/%ED%8C%8C%EC%9D%BC.PNG?AWSAccessKeyId=AKIAS4NQIUJ4WC3TUPHZ&Signature=Vpy1rxPL0pFMJDe04KO4S%2Be%2FCKE%3D&Expires=1607174691\n",
"파일.PNG\n",
"2020-12-05 09:29:24+00:00\n"
]
}
],
"source": [
"import boto3\n",
"\n",
"s3_client = boto3.client('s3',\n",
" region_name = 'us-east-1',\n",
" aws_access_key_id = 'AKIAS4NQIUJ4WC3TUPHZ',\n",
" aws_secret_access_key = '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')\n",
"s3_resource = boto3.resource('s3',\n",
" region_name = 'us-east-1',\n",
" aws_access_key_id = 'AKIAS4NQIUJ4WC3TUPHZ',\n",
" aws_secret_access_key = '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')\n",
"\n",
"bucket_name = 'themightiestkpk1'\n",
"my_bucket = s3_resource.Bucket(bucket_name)\n",
"\n",
"for file in my_bucket.objects.all():\n",
" params = {'Bucket': bucket_name, 'Key': file.key}\n",
" url = s3_client.generate_presigned_url('get_object', params)\n",
" # 사진 URL\n",
" print(url)\n",
" # 파일 이름\n",
" print(file.key)\n",
" # 생성 시간\n",
" print(file.last_modified)\n",
" \n",
"# objects = s3.list_objects(Bucket='themightiestkpk1')\n",
"\n",
"# for o in objects[\"Contents\"]:\n",
"# # 최초 수정 시간\n",
"# print(o[\"LastModified\"])\n",
"# # 파일 이름\n",
"# print(o[\"Key\"])"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {},
"outputs": [],
"source": [
"s3 = boto3.resource('s3',\n",
" region_name = 'us-east-1',\n",
" aws_access_key_id = 'AKIAS4NQIUJ4WC3TUPHZ',\n",
" aws_secret_access_key = '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')"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"amplify-amplifytest-dev-103811-deployment\n",
"themightiestkpk\n",
"themightiestkpk1\n"
]
}
],
"source": [
"for bucket in s3.buckets.all():\n",
" print(bucket.name)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"s3.download_file('themightiestkpk1', 'OBJECT_NAME', 'FILE_NAME')"
]
},
{
"cell_type": "code",
"execution_count": 13,
"metadata": {},
"outputs": [
{
"ename": "AttributeError",
"evalue": "'s3.ServiceResource' object has no attribute 'lookup'",
"output_type": "error",
"traceback": [
"\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
"\u001b[1;31mAttributeError\u001b[0m Traceback (most recent call last)",
"\u001b[1;32m<ipython-input-13-051c4251e0c7>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m 1\u001b[0m \u001b[1;31m#bucket = s3.Bucket('themightiestkpk1')\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 2\u001b[1;33m \u001b[0mbucket\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0ms3\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mlookup\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'themightiestkpk1'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 3\u001b[0m \u001b[1;32mfor\u001b[0m \u001b[0mkey\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mbucket\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 4\u001b[0m \u001b[0mprint\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mname\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mkey\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msize\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mkey\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mlast_modified\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 5\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
"\u001b[1;31mAttributeError\u001b[0m: 's3.ServiceResource' object has no attribute 'lookup'"
]
}
],
"source": [
"bucket = s3.Bucket('themightiestkpk1')\n",
"#bucket = s3.lookup('themightiestkpk1')\n",
"for key in bucket:\n",
" print(key.name, key.size, key.last_modified)\n",
"\n",
"\n",
"# for obj in bucket.objects.all():\n",
"# contents=obj.get()['Body'].read().decode(encoding=\"utf-8\",errors=\"ignore\")\n",
"# for line in contents.splitlines():\n",
"# print(line)\n",
"\n",
" \n",
"# for obj in bucket.objects.all():\n",
"# key = obj.key\n",
"# body = obj.get()['Body'].read().decode(encoding=\"utf-8\",errors=\"ignore\")\n",
"# #print(body)\n",
"# print(key) #파일.PNG\n",
"# print(body)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": 24,
"metadata": {},
"outputs": [
{
"ename": "AttributeError",
"evalue": "'s3.ServiceResource' object has no attribute 'list_buckets'",
"output_type": "error",
"traceback": [
"\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
"\u001b[1;31mAttributeError\u001b[0m Traceback (most recent call last)",
"\u001b[1;32m<ipython-input-24-478a06e36e78>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[1;32mfor\u001b[0m \u001b[0mb\u001b[0m \u001b[1;32min\u001b[0m \u001b[0ms3\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mlist_buckets\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 2\u001b[0m \u001b[0mprint\u001b[0m \u001b[1;33m(\u001b[0m\u001b[0mb\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
"\u001b[1;31mAttributeError\u001b[0m: 's3.ServiceResource' object has no attribute 'list_buckets'"
]
}
],
"source": [
"for b in s3.list_buckets():\n",
" print (b)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.8.3"
}
},
"nbformat": 4,
"nbformat_minor": 4
}
1 change: 1 addition & 0 deletions jupyter_test/debug.log
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
[1205/180120.157:ERROR:directory_reader_win.cc(43)] FindFirstFile: ������ ��θ� ã�� �� �����ϴ�. (0x3)

0 comments on commit 3633559

Please sign in to comment.