idds/tests/os_boto3_test.py

0001 #!/usr/bin/env python
0002
0003 import json                        # noqa F401
0004 import math                        # noqa F401
0005 import sys                         # noqa F401
0006 import os                          # noqa F401
0007
0008 from urllib.parse import urlparse
0009
0010 import boto3
0011 import boto.s3.connection          # noqa F401
0012 from boto.s3.key import Key        # noqa F401
0013 # from filechunkio import FileChunkIO
0014
0015 # AWS_SHARED_CREDENTIALS_FILE = ~/.aws/credentials
0016 # AWS_CONFIG_FILE = ~/.aws/config
0017
0018 access_key = ""
0019 secret_key = ""
0020
0021 hostname = 's3dfrgw.slac.stanford.edu'
0022 port = 443
0023 is_secure = True
0024
0025
0026 def get_endpoint_bucket_key(surl):
0027     parsed = urlparse(surl)
0028     endpoint = parsed.scheme + '://' + parsed.netloc
0029     full_path = parsed.path
0030     while "//" in full_path:
0031         full_path = full_path.replace('//', '/')
0032
0033     parts = full_path.split('/')
0034     bucket = parts[1]
0035     key = '/'.join(parts[2:])
0036     return endpoint, bucket, key
0037
0038
0039 s3_resource = boto3.resource(service_name='s3',
0040                              endpoint_url='https://s3dfrgw.slac.stanford.edu',
0041                              aws_access_key_id=access_key,
0042                              aws_secret_access_key=secret_key
0043                              )
0044 print(s3_resource)
0045
0046 rs = s3_resource.buckets.all()
0047 for b in rs:
0048     print(b)
0049     print(b.name)
0050
0051 s3 = boto3.client(service_name='s3',
0052                   endpoint_url='https://s3dfrgw.slac.stanford.edu',
0053                   aws_access_key_id=access_key,
0054                   aws_secret_access_key=secret_key
0055                   )
0056
0057 print(s3)
0058
0059 bucket_name = 'rubin-usdf-panda-logs'
0060 # rt = s3.get_bucket_policy(Bucket='rubin-usdf-panda-logs')
0061 # print(rt)
0062 # print(rt['Policy'])
0063
0064
0065 rt = s3.upload_file("/bin/hostname", 'rubin-usdf-panda-logs', 'hostname')
0066 print(rt)
0067
0068 my_bucket = s3_resource.Bucket('rubin-usdf-panda-logs')
0069 for obj in my_bucket.objects.all():
0070     print(obj)
0071
0072 rt = s3.get_object_acl(Bucket=bucket_name, Key='hostname')
0073 print(rt)
0074
0075 full_url = 'https://s3dfrgw.slac.stanford.edu/rubin-usdf-panda-logs/SLAC_TEST/PandaJob_56998520/pilotlog.txt'
0076 endpoint, bucket_name, object_name = get_endpoint_bucket_key(full_url)
0077 print(endpoint, bucket_name, object_name)
0078
0079 response = s3.generate_presigned_url('get_object',
0080                                      Params={'Bucket': bucket_name,
0081                                              'Key': object_name},
0082                                      ExpiresIn=3600)
0083 print(response)
0084
0085 rt = s3.download_file(bucket_name, object_name, '/tmp/pilotlog.txt')
0086 print(rt)