From 363355918e63bcb83d6d6e4f6b2f9e1284d0223e Mon Sep 17 00:00:00 2001 From: sunyeongchoi Date: Sat, 5 Dec 2020 23:07:25 +0900 Subject: [PATCH] get s3 file information in python --- .../S3 + python-checkpoint.ipynb | 108 ++++++++ jupyter_test/S3 + python.ipynb | 251 ++++++++++++++++++ jupyter_test/debug.log | 1 + 3 files changed, 360 insertions(+) create mode 100644 jupyter_test/.ipynb_checkpoints/S3 + python-checkpoint.ipynb create mode 100644 jupyter_test/S3 + python.ipynb create mode 100644 jupyter_test/debug.log diff --git a/jupyter_test/.ipynb_checkpoints/S3 + python-checkpoint.ipynb b/jupyter_test/.ipynb_checkpoints/S3 + python-checkpoint.ipynb new file mode 100644 index 0000000..9fe6c62 --- /dev/null +++ b/jupyter_test/.ipynb_checkpoints/S3 + python-checkpoint.ipynb @@ -0,0 +1,108 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 18, + "metadata": {}, + "outputs": [], + "source": [ + "import boto3" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "metadata": {}, + "outputs": [], + "source": [ + "s3 = boto3.resource('s3',\n", + " region_name = 'us-east-1',\n", + " aws_access_key_id = 'AKIAS4NQIUJ4WC3TUPHZ',\n", + " aws_secret_access_key = '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')" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "amplify-amplifytest-dev-103811-deployment\n", + "themightiestkpk\n" + ] + } + ], + "source": [ + "for bucket in s3.buckets.all():\n", + " print(bucket.name)" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "metadata": {}, + "outputs": [ + { + "ename": "AttributeError", + "evalue": "'s3.ServiceResource' object has no attribute 'list_buckets'", + "output_type": "error", + "traceback": [ + "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[1;31mAttributeError\u001b[0m Traceback (most recent call last)", + "\u001b[1;32m\u001b[0m in \u001b[0;36m\u001b[1;34m\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[1;32mfor\u001b[0m \u001b[0mb\u001b[0m \u001b[1;32min\u001b[0m \u001b[0ms3\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mlist_buckets\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 2\u001b[0m \u001b[0mprint\u001b[0m \u001b[1;33m(\u001b[0m\u001b[0mb\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n", + "\u001b[1;31mAttributeError\u001b[0m: 's3.ServiceResource' object has no attribute 'list_buckets'" + ] + } + ], + "source": [ + "for b in s3.list_buckets():\n", + " print (b)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.3" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/jupyter_test/S3 + python.ipynb b/jupyter_test/S3 + python.ipynb new file mode 100644 index 0000000..8ca0833 --- /dev/null +++ b/jupyter_test/S3 + python.ipynb @@ -0,0 +1,251 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "import boto3" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2020-12-05 12:07:06+00:00\n", + "제목 없음.png\n", + "2020-12-05 09:29:24+00:00\n", + "파일.PNG\n" + ] + } + ], + "source": [ + "import boto3\n", + "from datetime import datetime, timezone\n", + "\n", + "today = datetime.now(timezone.utc)\n", + "\n", + "s3 = boto3.client('s3',\n", + " region_name = 'us-east-1',\n", + " aws_access_key_id = 'AKIAS4NQIUJ4WC3TUPHZ',\n", + " aws_secret_access_key = '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')\n", + "\n", + "objects = s3.list_objects(Bucket='themightiestkpk1')\n", + "\n", + "for o in objects[\"Contents\"]:\n", + " # 최초 수정 시간\n", + " print(o[\"LastModified\"])\n", + " # 파일 이름\n", + " print(o[\"Key\"])" + ] + }, + { + "cell_type": "code", + "execution_count": 32, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "https://themightiestkpk1.s3.amazonaws.com/%EC%A0%9C%EB%AA%A9%20%EC%97%86%EC%9D%8C.png?AWSAccessKeyId=AKIAS4NQIUJ4WC3TUPHZ&Signature=j4TwlZk%2BswkkWEXasboC0vzEa9M%3D&Expires=1607174691\n", + "제목 없음.png\n", + "2020-12-05 12:07:06+00:00\n", + "https://themightiestkpk1.s3.amazonaws.com/%ED%8C%8C%EC%9D%BC.PNG?AWSAccessKeyId=AKIAS4NQIUJ4WC3TUPHZ&Signature=Vpy1rxPL0pFMJDe04KO4S%2Be%2FCKE%3D&Expires=1607174691\n", + "파일.PNG\n", + "2020-12-05 09:29:24+00:00\n" + ] + } + ], + "source": [ + "import boto3\n", + "\n", + "s3_client = boto3.client('s3',\n", + " region_name = 'us-east-1',\n", + " aws_access_key_id = 'AKIAS4NQIUJ4WC3TUPHZ',\n", + " aws_secret_access_key = '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')\n", + "s3_resource = boto3.resource('s3',\n", + " region_name = 'us-east-1',\n", + " aws_access_key_id = 'AKIAS4NQIUJ4WC3TUPHZ',\n", + " aws_secret_access_key = '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')\n", + "\n", + "bucket_name = 'themightiestkpk1'\n", + "my_bucket = s3_resource.Bucket(bucket_name)\n", + "\n", + "for file in my_bucket.objects.all():\n", + " params = {'Bucket': bucket_name, 'Key': file.key}\n", + " url = s3_client.generate_presigned_url('get_object', params)\n", + " # 사진 URL\n", + " print(url)\n", + " # 파일 이름\n", + " print(file.key)\n", + " # 생성 시간\n", + " print(file.last_modified)\n", + " \n", + "# objects = s3.list_objects(Bucket='themightiestkpk1')\n", + "\n", + "# for o in objects[\"Contents\"]:\n", + "# # 최초 수정 시간\n", + "# print(o[\"LastModified\"])\n", + "# # 파일 이름\n", + "# print(o[\"Key\"])" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "s3 = boto3.resource('s3',\n", + " region_name = 'us-east-1',\n", + " aws_access_key_id = 'AKIAS4NQIUJ4WC3TUPHZ',\n", + " aws_secret_access_key = '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')" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "amplify-amplifytest-dev-103811-deployment\n", + "themightiestkpk\n", + "themightiestkpk1\n" + ] + } + ], + "source": [ + "for bucket in s3.buckets.all():\n", + " print(bucket.name)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "s3.download_file('themightiestkpk1', 'OBJECT_NAME', 'FILE_NAME')" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [ + { + "ename": "AttributeError", + "evalue": "'s3.ServiceResource' object has no attribute 'lookup'", + "output_type": "error", + "traceback": [ + "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[1;31mAttributeError\u001b[0m Traceback (most recent call last)", + "\u001b[1;32m\u001b[0m in \u001b[0;36m\u001b[1;34m\u001b[0m\n\u001b[0;32m 1\u001b[0m \u001b[1;31m#bucket = s3.Bucket('themightiestkpk1')\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 2\u001b[1;33m \u001b[0mbucket\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0ms3\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mlookup\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'themightiestkpk1'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 3\u001b[0m \u001b[1;32mfor\u001b[0m \u001b[0mkey\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mbucket\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 4\u001b[0m \u001b[0mprint\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mname\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mkey\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msize\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mkey\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mlast_modified\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 5\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n", + "\u001b[1;31mAttributeError\u001b[0m: 's3.ServiceResource' object has no attribute 'lookup'" + ] + } + ], + "source": [ + "bucket = s3.Bucket('themightiestkpk1')\n", + "#bucket = s3.lookup('themightiestkpk1')\n", + "for key in bucket:\n", + " print(key.name, key.size, key.last_modified)\n", + "\n", + "\n", + "# for obj in bucket.objects.all():\n", + "# contents=obj.get()['Body'].read().decode(encoding=\"utf-8\",errors=\"ignore\")\n", + "# for line in contents.splitlines():\n", + "# print(line)\n", + "\n", + " \n", + "# for obj in bucket.objects.all():\n", + "# key = obj.key\n", + "# body = obj.get()['Body'].read().decode(encoding=\"utf-8\",errors=\"ignore\")\n", + "# #print(body)\n", + "# print(key) #파일.PNG\n", + "# print(body)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": 24, + "metadata": {}, + "outputs": [ + { + "ename": "AttributeError", + "evalue": "'s3.ServiceResource' object has no attribute 'list_buckets'", + "output_type": "error", + "traceback": [ + "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[1;31mAttributeError\u001b[0m Traceback (most recent call last)", + "\u001b[1;32m\u001b[0m in \u001b[0;36m\u001b[1;34m\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[1;32mfor\u001b[0m \u001b[0mb\u001b[0m \u001b[1;32min\u001b[0m \u001b[0ms3\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mlist_buckets\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 2\u001b[0m \u001b[0mprint\u001b[0m \u001b[1;33m(\u001b[0m\u001b[0mb\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n", + "\u001b[1;31mAttributeError\u001b[0m: 's3.ServiceResource' object has no attribute 'list_buckets'" + ] + } + ], + "source": [ + "for b in s3.list_buckets():\n", + " print (b)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.3" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/jupyter_test/debug.log b/jupyter_test/debug.log new file mode 100644 index 0000000..36ca360 --- /dev/null +++ b/jupyter_test/debug.log @@ -0,0 +1 @@ +[1205/180120.157:ERROR:directory_reader_win.cc(43)] FindFirstFile: θ ã ϴ. (0x3)