OSS_Python_API.tar

oss/0000775000076400007650000000000011654134241010016 5ustar  leoleooss/oss_fs.py0000664000076400007650000001156711654134232011676 0ustar  leoleo#coding=utf8
from oss_api import *
from oss_xml_handler import *

'''
Mock a file system for testing purpose
'''
class OssFS:
    def __init__(self, host, id="", key=""):
        self.buckets = []
        self.host = host
        self.oss = OssAPI(host, id, key)

    def put_bucket(self, bucketname,acl='private'):
        if bucketname in self.buckets:
             return False
        res = self.oss.put_bucket(bucketname,acl)        
        if (res.status / 100) == 2:
            self.buckets.append(bucketname)
            return True
        else:
            return False

    def delete_bucket(self, bucketname):
        res = self.oss.delete_bucket(bucketname)
        if (res.status / 100) == 2:
            while bucketname in self.buckets:
                 del self.buckets[self.buckets.index(bucketname)]
                 return True
        else:
            return False

    def list_bucket(self):
        res = self.oss.get_service()
        if (res.status / 100) == 2:
            body = res.read()
            h = GetServiceXml(body)
            self.buckets = h.list()
        return self.buckets

    def get_bucket_acl(self, bucket):
        res = self.oss.get_bucket_acl(bucket)
        body = res.read()
        h = GetBucketAclXml(body)
        return h.grant


    def put_bucket_acl(self, bucket, acl=''):
        res = self.oss.put_bucket_acl(bucket,acl)
        if (res.status / 100) == 2:
            return True
        else:
            return False

    #fileobj is django defined uploaded file structure
    def upload_file(self, bucket, object, filename):
        res = self.oss.put_object_from_file(bucket, object, filename)
        if (res.status / 100) == 2:
            return True
        else:
            return False

    def make_dir(self, bucket, dirname):
        object = dirname + '/'
        res = self.oss.put_object_with_data(bucket, object, "")
        if (res.status / 100) == 2:
            return True
        else:
            return False

    def read_file(self, bucket, filename):
        res = self.oss.get_object(bucket, filename)
        return res.read()

    #return a list of file
    def list_file(self, bucket, prefix="", delim="", marker="", maxkeys=5):
        fl = []
        pl = []
        res = self.oss.get_bucket(bucket, prefix, delimiter=delim, marker=marker, maxkeys=maxkeys)
        if (res.status / 100) == 2:
            body = res.read()
            h = GetBucketXml(body)
            (fl, pl) = h.list()
        return (fl, pl)

    def delete_file(self, bucket, filename):
        res = self.oss.delete_object(bucket, filename)
        if (res.status / 100) == 2:
            return True
        else:
            return False

    def open_file_for_write(self, bucket, filename, filesize):
        conn = self.oss._open_conn_to_put_object(bucket, filename, filesize)
        return WriteFileObject(conn)

    def open_file_for_read(self, bucket, filename):
        res = self.oss.get_object(bucket, filename)
        return ReadFileObject(res)
            
class ReadFileObject:
    def __init__(self, res):
        self.res = res

    def read(self, buffer_size):
        if (self.res.status / 100) == 2:
            return self.res.read(buffer_size)
        else:
            return ""

    def close(self):
        if (self.res.status / 100) == 2:
            return True
        else:
            return False
        
class WriteFileObject:
    def __init__(self, conn):
        self.conn = conn

    def write(self, data):
        self.conn.send(data)
        
    def close(self):
        res = self.conn.getresponse()
        if (res.status / 100) == 2:
            return True
        else:
            return False

    
def test_open_file_for_write(bucket, object, filename):
    fp = file(filename, 'r')
    fp.seek(os.SEEK_SET, os.SEEK_END)
    filesize = fp.tell()
    fp.seek(os.SEEK_SET)
    wo = fs.open_file_for_write(bucket, object, filesize)
    l = fp.read(BufferSize)
    while len(l) > 0:
        print len(l)
        wo.write(l)
        l = fp.read(BufferSize)
    print wo.close()

def test_open_file_for_read(bucket, object, filename):
    fp = file(filename, 'w')
    ro = fs.open_file_for_read(bucket, object)
    buf = ro.read(1024)
    while len(buf) > 0:
        print len(buf)
        fp.write(buf)
        buf = ro.read(1024)
    print ro.close()

if __name__ == "__main__":
    host = '127.0.0.1:8080'
    id = ""
    key = ""
    BufferSize = 8000
    fs = OssFS(host, id, key)
    test_open_file_for_write('6uu', "a/b/c/d", "put.jpg")
    test_open_file_for_read('6uu', "a/b/c/d", "put.jpg")
    print fs.upload_file('6uu', "a/b/c", 'put.jpg')
    print fs.upload_file('6uu', "b/b/c", 'put.jpg')
    print fs.upload_file('6uu', "b/c", 'put.jpg')
    print fs.upload_file('6uu', "c", 'put.jpg')
    (fl, pl) = fs.list_file('bb1','',delim='',marker='')
    print fl
    print pl
    (fl, pl) = fs.list_file('bb1','',delim='',marker='')
    print fl
    print pl
oss/oss_cmd.py0000664000076400007650000004337411654134202012027 0ustar  leoleo#coding=utf8
from oss_api import *
from oss_xml_handler import *

msg_remind = " PLEASE USE # TO SEPERATE EACH INPUT PARAMETER"

msg_quit = "q or quit, will quit out this console"
msg_help = "h or help, will show the message again"

#msg_get_service = "gs means get service , it lists all buckets that user has"
msg_get_service = "gs means get service."
example_get_service = "Input: gs"
GET_SERVICE = "get service"

#msg_put_bucket = "pb means put bucket, it creates bucket"
msg_put_bucket = "pb means put bucket."
example_put_bucket = "Input: pb#bucket||pb#bucket#acl||pb#bucket#acl#headers. "
PUT_BUCKET = "put bucket"

#msg_delete_bucket = "db means delete bucket, it deletes the bucket that user created"
msg_delete_bucket = "db means delete bucket."
example_delete_bucket = "Input: db#bucket"
DELETE_BUCKET = "delete bucket"

#msg_get_bucket = "gb means get bucket, it lists objects in the bucket"
msg_get_bucket = "gb means get bucket."
example_get_bucket = "Input: gb#bucket||gb#bucket#prefix||gb#bucket#prefix#marker||gb#bucket#prefix#marker#delimiter||gb#bucket#prefix#marker#delimiter#maxkeys||gb#bucket#prefix#marker#delimiter#maxkeys#headers"
GET_BUCKET = "get bucket"

#msg_get_bucket_acl = "gba means get bucket acl, it lists bucket acl"
msg_get_bucket_acl = "gba means get bucket acl."
example_get_bucket_acl = "Input: gba#bucket"
GET_BUCKET_ACL = "get bucket acl"

#msg_put_bucket_acl = "pba means put bucket acl, it sets bucket acl"
msg_put_bucket_acl = "pba means put bucket acl."
example_put_bucket_acl = "Input: pba#bucket#acl,  Example: pba#mybucket#public-read"
PUT_BUCKET_ACL = "put bucket acl"

#msg_put_object_with_data = "powd means put object with data, it puts object, its content from string data"
msg_put_object_with_data = "powd means put object with data."
example_put_object_with_data = "Input: powd#bucket#object#string_data||powd#bucket#object#string_data#content_type||powd#bucket#object#string_data#content_type#headers,  Example: powd#mybucket#myobjcet#abcdefghijk#txt/plain"
PUT_OBJECT_WITH_DATA = "put object with data"

#msg_put_object_from_file = "poff means put object from file, it puts object , its content from file"
msg_put_object_from_file = "poff means put object from file."
example_put_object_from_file = "Input: poff#bucket#object#filename||poff#bucket#object#filename#content_type||poff#bucket#object#filename#content_type#headers,  Example: poff#mybucket#myobjcet#myfilename"
PUT_OBJECT_FROM_FILE = "put object from file"

#msg_get_object = "go means get object, it gets object and print its content"
msg_get_object = "go means get object."
example_get_object = "Input: go#bucket#object||go#bucket#object#headers,  Example: go#mybucket#myobjcet"
GET_OBJECT = "get object"

#msg_get_object_to_file = "gotf means get object to file, it gets object and save its content to file"
msg_get_object_to_file = "gotf means get object to file."
example_get_object_to_file = "Input: gotf#bucket#object#filename||go#bucket#object#filename#headers,  Example: goff#mybucket#myobjcet#tofilename"
GET_OBJECT_TO_FILE = "get object to file"

#msg_delete_object = "do means delete object, it deletes object"
msg_delete_object = "do means delete object."
example_delete_object = "Input: do#bucket#object||do#bucket#object#headers,  Example: do#mybucket#myobjcet"
DELETE_OBJECT = "delete object"

#msg_head_object = "ho means head object, it gets meta info of object"
msg_head_object = "ho means head object."
example_head_object = "Input: ho#bucket#object||ho#bucket#object#headers,  Example: ho#mybucket#myobject"
HEAD_OBJECT = "head object"

def usage():
    width = 40
    print "=> 1) ", msg_quit.ljust(width)
    print "=> 2) ", msg_help.ljust(width)
    print '***************************************'
    print "NOTE:acl in below means access control level, acl SHOULD be public-read, private, or public-read-write"
    print "headers and parameters below means dic, it SHOULD be dic like a=b c=d"
    print "content type means object type , it is used in headers, default is application/octet-stream"
    print '***************************************'
    print 'bucket operation'
    print "=> 3) ", msg_get_service.ljust(width), example_get_service
    print "=> 4) ", msg_put_bucket.ljust(width), example_put_bucket
    print "=> 5) ", msg_delete_bucket.ljust(width), example_delete_bucket
    print "=> 6) ", msg_get_bucket.ljust(width), example_get_bucket
    print "=> 7) ", msg_get_bucket_acl.ljust(width), example_get_bucket_acl
    print "=> 8) ", msg_put_bucket_acl.ljust(width), example_put_bucket_acl
    print ''
    print '***************************************'
    print 'objcet operation'
    print "=> 9) ", msg_put_object_with_data.ljust(width), example_put_object_with_data
    print "=>10) ", msg_put_object_from_file.ljust(width), example_put_object_from_file
    print "=>11) ", msg_get_object.ljust(width), example_get_object
    print "=>12) ", msg_get_object_to_file.ljust(width), example_get_object_to_file
    print "=>13) ", msg_delete_object.ljust(width), example_delete_object
    print "=>14) ", msg_head_object.ljust(width), example_head_object
    print ''

def print_result(cmd, res):
    if res.status / 100 == 2:
        print cmd, "OK"
    else:
        print "error headers", res.getheaders()
        print "error body", res.read()
        print cmd, "Failed!"

def check_input(cmd_str, cmd_list, min, max):
    if len(cmd_list) < min or len(cmd_list) > max: 
        print "ERROR! ", cmd_str, "needs ", min, "-", max, " paramters"
        if example_map.has_key(cmd_str):
            print example_map[cmd_str]
        return False 
    else:
        return True

def get_cmd(cmd):
    string = cmd 
    tmp_list = string.split('#')
    cmd_list = []
    for i in tmp_list:
        if len(i.strip()) != 0:
            cmd_list.append(i.strip()) 
    return cmd_list
    
def transfer_string_to_dic(string):
    list = string.split() 
    dic = {} 
    for entry in list: 
        key, val = entry.split('=') 
        dic[key.strip()] = val.strip()
    print dic
    return dic

if __name__ == "__main__":
    import sys
    host = ""
    user_name = ""
    access_id = ""
    secret_access_key = ""
    remind = "the method you input is not supported! please input h or help to check method supported"
 
    if len(sys.argv) != 4:
        print '***************************************'
        print 'Please input the parameters like below'
        print '***************************************'
        print 'python ', __file__,' host access_id access_key'
        print 'host is the ip address with port of oss apache server . For Example: 10.249.105.22:8080'
        print 'access_id is public key that oss server provided. For Example: 84792jahfdsah+='
        print 'access_key is private secret key that oss server provided. For Example: atdh+=flahmzhha=+'
        print '***************************************'
        print ''
        print ''
        exit()
    else:
        host = sys.argv[1]
        access_id = sys.argv[2]
        secret_access_key = sys.argv[3]

    oss = OssAPI(host, access_id, secret_access_key)
    usage()
    bucketname = ""

    example_map = {}
    example_map[PUT_BUCKET] = example_put_bucket
    example_map[PUT_BUCKET_ACL] = example_put_bucket_acl
    example_map[GET_SERVICE] = example_get_service
    example_map[GET_BUCKET] = example_get_bucket
    example_map[DELETE_BUCKET] = example_delete_bucket
    example_map[GET_BUCKET_ACL] = example_get_bucket_acl
    example_map[PUT_OBJECT_WITH_DATA] = example_put_object_with_data
    example_map[PUT_OBJECT_FROM_FILE] = example_put_object_from_file
    example_map[GET_OBJECT] = example_get_object
    example_map[GET_OBJECT_TO_FILE] = example_get_object_to_file
    example_map[DELETE_OBJECT] = example_delete_object
    example_map[HEAD_OBJECT] = example_head_object
    
    while True:
        cmd = raw_input(">>")
        cmd = cmd.strip()
        cmd_list = get_cmd(cmd) 
        if ("q" == cmd.lower() or "quit" == cmd.lower()):
            break
        elif ("h" == cmd.lower() or "help" == cmd.lower()):
            usage()
        elif len(cmd_list) > 0:
            if cmd_list[0].lower() == "pb":
                cmd_str = PUT_BUCKET
                min = 2
                max = 4
                if not check_input(cmd_str, cmd_list, min, max):
                    pass
                else:
                    print "cmd", cmd_list
                    bucketname = cmd_list[1]
                    if len(cmd_list) == 2:
                        res = oss.put_bucket(cmd_list[1])
                    elif len(cmd_list) == 3:
                        res = oss.put_bucket(cmd_list[1], cmd_list[2])
                    elif len(cmd_list) == 4:
                        dic = transfer_string_to_dic(cmd_list[3])
                        res = oss.put_bucket(cmd_list[1], cmd_list[2], dic)
                    print_result(cmd_str, res)

            elif cmd_list[0] == "pba":
                cmd_str = PUT_BUCKET_ACL
                min = 3
                max = 3
                if not check_input(cmd_str, cmd_list, min, max):
                    pass
                else:
                    print "cmd", cmd_list
                    bucketname = cmd_list[1]
                    res = oss.put_bucket(cmd_list[1], cmd_list[2])
                    print_result(cmd_str, res)

            elif cmd_list[0].lower() == "gs":
                cmd_str = GET_SERVICE
                min = 1
                max = 1
                if not check_input(cmd_str, cmd_list, min, max):
                    pass
                else:
                    print "cmd", cmd_list
                    if len(cmd_list) == 1:
                        res = oss.get_service()
                    print_result(cmd_str, res)
                    if (res.status / 100) == 2:
                        body = res.read()
                        h = GetServiceXml(body)
                        print "bucket list size is: ", len(h.list())
                        print "bucket list is: "
                        for i in h.list():
                            print i

            elif cmd_list[0].lower() == "gb":
                cmd_str = GET_BUCKET
                min = 2
                max = 7
                if not check_input(cmd_str, cmd_list, min, max):
                    pass
                else:
                    print "cmd", cmd_list
                    bucketname = cmd_list[1]
                    #get_bucket(bucket, prefix='', marker='', delimiter='', maxkeys='', headers = {}):
                    if len(cmd_list) == 2:
                        res = oss.get_bucket(cmd_list[1])
                    elif len(cmd_list) == 3:
                        res = oss.get_bucket(cmd_list[1], cmd_list[2])
                    elif len(cmd_list) == 4:
                        res = oss.get_bucket(cmd_list[1], cmd_list[2], cmd_list[3])
                    elif len(cmd_list) == 5:
                        res = oss.get_bucket(cmd_list[1], cmd_list[2], cmd_list[3], cmd_list[4])
                    elif len(cmd_list) == 6:
                        res = oss.get_bucket(cmd_list[1], cmd_list[2], cmd_list[3], cmd_list[4], cmd_list[5])
                    elif len(cmd_list) == 7:
                        dic = transfer_string_to_dic(cmd_list[6])
                        res = oss.get_bucket(cmd_list[1], cmd_list[2], cmd_list[3], cmd_list[4], cmd_list[5], dic)
                    print_result(cmd_str, res)
                    if (res.status / 100) == 2:
                        body = res.read()
                        hh = GetBucketXml(body)
                        (fl, pl) = hh.list()
                        print "prefix list size is: ", len(pl)
                        print "prefix listis: "
                        for i in pl:
                            print i
                        print "file list size is: ", len(fl)
                        print "file list is: "
                        for i in fl:
                            print i

            elif cmd_list[0].lower() == "db":
                cmd_str = DELETE_BUCKET
                print cmd_str
                min = 2 
                max = 2
                if not check_input(cmd_str, cmd_list, min, max):
                    pass
                else:
                    print "cmd", cmd_list
                    if len(cmd_list) == 2:
                        res = oss.delete_bucket(cmd_list[1])
                    print_result(cmd_str, res)

            elif cmd_list[0].lower() == "gba":
                cmd_str = GET_BUCKET_ACL
                print cmd_str
 
                min = 2
                max = 2
                if not check_input(cmd_str, cmd_list, min, max):
                    pass
                else:
                    print "cmd", cmd_list
                    bucketname = cmd_list[1]
                    if len(cmd_list) == 2:
                        res = oss.get_bucket_acl(cmd_list[1])
                    print_result(cmd_str, res)

                    if (res.status / 100) == 2:
                        body = res.read()
                        h = GetBucketAclXml(body)
                        print "bucket ", bucketname, " acl is: ", h.grant

            elif cmd_list[0].lower() == "powd" or cmd_list[0].lower() == "poff":
                if cmd_list[0].lower() == "powd":
                    cmd_str = PUT_OBJECT_WITH_DATA
                else:
                    cmd_str = PUT_OBJECT_FROM_FILE
                min = 4 
                max = 6
                if not check_input(cmd_str, cmd_list, min, max):
                    pass
                else:
                    print "cmd", cmd_list
                    bucketname = cmd_list[1]
                    if isinstance(cmd_list[2], str):
                        tmp = unicode(cmd_list[2], 'utf-8')
                        cmd_list[2] = tmp
                    if len(cmd_list) == 4:
                        if cmd_list[0].lower() == "powd":
                            res = oss.put_object_with_data(cmd_list[1], cmd_list[2], cmd_list[3])
                        else:
                            res = oss.put_object_from_file(cmd_list[1], cmd_list[2], cmd_list[3])
                    elif len(cmd_list) == 5:
                        if cmd_list[0].lower() == "powd":
                            res = oss.put_object_with_data(cmd_list[1], cmd_list[2], cmd_list[3], cmd_list[4])
                        else:
                            res = oss.put_object_from_file(cmd_list[1], cmd_list[2], cmd_list[3], cmd_list[4])
                    elif len(cmd_list) == 6:
                        if cmd_list[0].lower() == "powd":
                            dic = transfer_string_to_dic(cmd_list[5])
                            res = oss.put_object_with_data(cmd_list[1], cmd_list[2], cmd_list[3], cmd_list[4], dic)
                        else:
                            dic = transfer_string_to_dic(cmd_list[5])
                            res = oss.put_object_from_file(cmd_list[1], cmd_list[2], cmd_list[3], cmd_list[4], dic)
                    print_result(cmd_str, res)

            elif cmd_list[0].lower() == "go":
                cmd_str = GET_OBJECT
                min = 3
                max = 4
                if not check_input(cmd_str, cmd_list, min, max):
                    pass
                else:
                    print "cmd", cmd_list
                    bucketname = cmd_list[1]
                    if len(cmd_list) == 3:
                        res = oss.get_object(cmd_list[1], cmd_list[2])
                    elif len(cmd_list) == 4:
                        dic = transfer_string_to_dic(cmd_list[3])
                        res = oss.get_object(cmd_list[1], cmd_list[2], dic)
                    print_result(cmd_str, res)
                    if res.status / 100 == 2:
                        print res.read()
 
            elif cmd_list[0].lower() == "gotf":
                cmd_str = GET_OBJECT_TO_FILE
                min = 4
                max = 5
                if not check_input(cmd_str, cmd_list, min, max):
                    pass
                else:
                    print "cmd", cmd_list
                    bucketname = cmd_list[1]
                    if len(cmd_list) == 4:
                        res = oss.get_object_to_file(cmd_list[1], cmd_list[2], cmd_list[3])
                    elif len(cmd_list) == 5:
                        dic = transfer_string_to_dic(cmd_list[4])
                        res = oss.get_object_to_file(cmd_list[1], cmd_list[2], cmd_list[3], dic)
                    print_result(cmd_str, res)
 
            elif cmd_list[0].lower() == "do":
                cmd_str = DELETE_OBJECT
                min = 3
                max = 4
                if not check_input(cmd_str, cmd_list, min, max):
                    pass
                else:
                    print "cmd", cmd_list
                    bucketname = cmd_list[1]
                    if len(cmd_list) == 3:
                        res = oss.delete_object(cmd_list[1], cmd_list[2])
                    elif len(cmd_list) == 4:
                        dic = transfer_string_to_dic(cmd_list[3])
                        res = oss.delete_object(cmd_list[1], cmd_list[2], dic)
                    print_result(cmd_str, res)

            elif cmd_list[0].lower() == "ho":
                cmd_str = HEAD_OBJECT
                min = 3
                max = 4
                if not check_input(cmd_str, cmd_list, min, max):
                    pass
                else:
                    print "cmd", cmd_list
                    bucketname = cmd_list[1]
                    if len(cmd_list) == 3:
                        res = oss.head_object(cmd_list[1], cmd_list[2])
                    elif len(cmd_list) == 4:
                        dic = transfer_string_to_dic(cmd_list[3])
                        res = oss.head_object(cmd_list[1], cmd_list[2], dic)
                    print_result(cmd_str, res)
                    if res.status / 100 == 2:
                        print res.getheaders()
            else:
                print remind
        else:
            print remind
oss/oss_util.py0000664000076400007650000002564511654134213012244 0ustar  leoleo#coding=utf8
import urllib
import base64
import hmac
import time
from hashlib import sha1 as sha
import os
import md5
import StringIO
from oss_xml_handler import *
from threading import Thread

DEBUG = False 

self_define_header_prefix = "x-oss-"

class User:
    def __init__(self, user_name = "", access_id = "", secret_access_key = ""):
        self.user_name = user_name
        self.access_id = access_id
        self.secret_access_key = secret_access_key

    def show(self):
        print (('user_name:(%s) access_id:(%s) secret_access_key(%s)')) % (self.user_name,
                self.access_id, self.secret_access_key)


def _format_header(headers = {}):
    '''
    format the headers that self define
    convert the self define headers to lower.
    '''
    tmp_headers = {}
    for k in headers.keys():
        if k.lower().startswith(self_define_header_prefix):
            k_lower = k.lower()
            tmp_headers[k_lower] = headers[k]
        else:
            tmp_headers[k] = headers[k]
    return tmp_headers

def get_assign(secret_access_key, method, headers = {}, resource="/", result = []):
    '''
    Create the authorization for OSS based on header input.
    You should put it into "Authorization" parameter of header.
    '''
    content_md5 = ""
    content_type = ""
    date = ""
    canonicalized_oss_headers = ""
    if DEBUG:
        print "secret_access_key", secret_access_key
    content_md5 = safe_get_element('Content-Md5', headers)
    content_type = safe_get_element('Content-Type', headers)
    date = safe_get_element('Date', headers)
    canonicalized_resource = resource
    tmp_headers = _format_header(headers)
    if len(tmp_headers) > 0:
        x_header_list = tmp_headers.keys() 
        x_header_list.sort()
        for k in x_header_list: 
            if k.startswith(self_define_header_prefix):
                canonicalized_oss_headers += k + ":" + tmp_headers[k] + "\n"
    
    string_to_sign = method + "\n" + content_md5 + "\n" + content_type + "\n" + date + "\n" + canonicalized_oss_headers + canonicalized_resource;
    result.append(string_to_sign)
    if DEBUG:
        print "string_to_sign", string_to_sign, "string_to_sign_size", len(string_to_sign)
    
    h = hmac.new(secret_access_key, string_to_sign, sha)

    return base64.encodestring(h.digest()).strip()

def convert_header2map(header_list):
    header_map = {}
    for (a, b) in header_list:
        header_map[a] = b
    return header_map

def safe_get_element(name, container):
    if name in container:
        return container[name]
    else:
        return ""

def append_param(url, params):
    l = []
    for k,v in params.items():
        k = k.replace('_', '-')
        if  k == 'maxkeys':
            k = 'max-keys'
        if isinstance(v, unicode):
            v = v.encode('utf-8')
        if v is not None and v != '':
            l.append('%s=%s' % (urllib.quote(k), urllib.quote(str(v))))
        elif k == 'acl':
            l.append('%s' % (urllib.quote(k)))
    if len(l):
        url = url + '?' + '&'.join(l)
    return url

def create_object_group_msg_xml(part_msg_list = []):
    xml_string = r'<CreateFileGroup>'
    for part in part_msg_list:
        if len(part) >= 3:
            if isinstance(part[1], unicode):
                file_path = part[1].encode('utf-8')
            else:
                file_path = part[1]
            xml_string += r'<Part>'
            xml_string += r'<PartNumber>' + str(part[0]) + r'</PartNumber>'
            xml_string += r'<PartName>' + str(file_path) + r'</PartName>'
            xml_string += r'<ETag>"' + str(part[2]).upper() + r'"</ETag>'
            xml_string += r'</Part>'
        else:
            print "the ", part, " in part_msg_list is not as expected!"
            return ""
    xml_string += r'</CreateFileGroup>'

    return xml_string

def delete_all_parts_of_object_group(oss, bucket, object_group_name):
    res = oss.get_object_group_index(bucket, object_group_name)
    if res.status == 200:
        body = res.read()
        h = GetObjectGroupIndexXml(body)
        object_group_index = h.list()
        for i in object_group_index:
            if len(i) == 4 and len(i[1]) > 0:
                part_name = i[1].strip()
                res = oss.delete_object(bucket, part_name)
                if res.status != 204:
                    print "delete part ", part_name, " in bucket:", bucket, " failed!"
                    return False
                else:
                    print "delete part ", part_name, " in bucket:", bucket, " ok"
    else:
        return False

    return True;

def split_large_file(file_path, object_prefix = "", max_part_num = 1000, part_size = 10 * 1024 * 1024, buffer_size = 10 * 1024):
    parts_list = []

    if os.path.isfile(file_path):
        file_size = os.path.getsize(file_path)

        if file_size > part_size * max_part_num:
            part_size = (file_size + max_part_num - file_size % max_part_num) / max_part_num
        
        part_order = 1
        fp = open(file_path, 'rb')
        fp.seek(os.SEEK_SET)

        total_split_len = 0
        part_num = file_size / part_size
        if file_size % part_size != 0:
            part_num += 1

        for i in range(0, part_num):
            left_len = part_size
            real_part_size = 0 
            m = md5.new()
            offset = part_size * i
            while True:
                read_size = 0
                if left_len <= 0:
                    break
                elif left_len < buffer_size:
                    read_size = left_len
                else:
                    read_size = buffer_size

                buffer_content = fp.read(read_size)
                m.update(buffer_content)
                real_part_size += len(buffer_content)

                left_len = left_len - read_size 

            md5sum = m.hexdigest()

            temp_file_name = os.path.basename(file_path) + "_" + str(part_order)
            if len(object_prefix) == 0:
                file_name = sum_string(temp_file_name) + "_" + temp_file_name
            else:
                file_name = object_prefix + "/" + sum_string(temp_file_name) + "_" + temp_file_name
            part_msg = (part_order, file_name, md5sum, real_part_size, offset)
            total_split_len += real_part_size
            parts_list.append(part_msg)
            part_order += 1

        fp.close()
    else:
        print "ERROR! No file: ", file_path, ", please check."

    return parts_list

def sumfile(fobj):
    '''Returns an md5 hash for an object with read() method.'''
    m = md5.new()
    while True:
        d = fobj.read(8096)
        if not d:
            break
        m.update(d)
    return m.hexdigest()

def md5sum(fname):
    '''Returns an md5 hash for file fname, or stdin if fname is "-".'''
    if fname == '-':
        ret = sumfile(sys.stdin)
    else:
        try:
            f = file(fname, 'rb')
        except:
            return 'Failed to open file'
        ret = sumfile(f)
        f.close()
    return ret

def md5sum2(filename, offset = 0, partsize = 0):
    m = md5.new()
    fp = open(filename, 'rb')
    if offset > os.path.getsize(filename):
        fp.seek(os.SEEK_SET, os.SEEK_END)
    else:
        fp.seek(offset)

    left_len = partsize
    BufferSize = 8 * 1024
    while True:
        if left_len <= 0:
           break
        elif left_len < BufferSize:
           buffer_content = fp.read(left_len)
        else:
           buffer_content = fp.read(BufferSize)
        m.update(buffer_content)

        left_len = left_len - len(buffer_content) 
    md5sum = m.hexdigest() 
    return md5sum

def sum_string(content):
    f = StringIO.StringIO(content)
    md5sum = sumfile(f)
    f.close()
    return md5sum

class PutObjectGroupWorker(Thread):
    def __init__(self, oss, bucket, file_path, part_msg_list):
        Thread.__init__(self)
        self.oss = oss 
        self.bucket = bucket
        self.part_msg_list = part_msg_list
        self.file_path = file_path

    def run(self):
        for part in self.part_msg_list:
            if len(part) == 5:
                bucket = self.bucket
                file_name = part[1]
                object_name = file_name
                res = self.oss.head_object(bucket, object_name)
                if res.status == 200:
                    header_map = convert_header2map(res.getheaders())
                    etag = safe_get_element("etag", header_map)
                    md5 = part[2] 
                    if etag.replace('"', "").upper() == md5.upper():
                        continue

                partsize = part[3]
                offset = part[4]
                res = self.oss.put_object_from_file_given_pos(bucket, object_name, self.file_path, offset, partsize)
                if res.status != 200:
                    print "upload ", file_name, "failed!"," ret is:", res.status
                    print "headers", res.getheaders()      
            else:
                print "ERROR! part", part , " is not as expected!"

class GetAllObjects:
    def __init__(self):
        self.object_list = []

    def get_object_in_bucket(self, oss, bucket="", marker="", prefix=""):
        object_list = []
        res = oss.get_bucket(bucket, prefix, marker)
        body = res.read()
        hh = GetBucketXml(body)
        (fl, pl) = hh.list()
        if len(fl) != 0:
            for i in fl:
                if isinstance(i[0], unicode):
                    object = i[0].encode('utf-8')
                    object_list.append(object)

        if hh.is_truncated:
            marker = hh.nextmarker
        return (object_list, marker)


    def get_all_object_in_bucket(self, oss, bucket="", marker="", prefix=""):
        marker2 = ""
        while True:
            (object_list, marker) = self.get_object_in_bucket(oss, bucket, marker2, prefix)
            marker2 = marker
            if len(object_list) != 0:
                self.object_list.extend(object_list)

            if len(marker) == 0:
                break

def clear_all_objects_in_bucket(oss_instance, bucket):
    '''
    it will clean all objects in bucket, after that, it will delete this bucket.
    
    example:
    from oss_api import *
    host = ""
    id = ""
    key = ""
    oss_instance = OssAPI(host, id, key)
    bucket = "leopublicreadprivatewrite"
    if clear_all_objects_in_bucket(oss_instance, bucket):
        print "clean OK"
    else:
        print "clean Fail"
    '''
    b = GetAllObjects()
    b.get_all_object_in_bucket(oss_instance, bucket)
    for i in b.object_list:
        res = oss_instance.delete_object(bucket, i)
        if (res.status / 100 != 2):
            print "clear_all_objects_in_bucket: delete object fail, ret is:", res.status, "object is: ", i 
            return False
        else:
            print "clear_all_objects_in_bucket: delete object ok", "object is: ", i 
    res = oss_instance.delete_bucket(bucket)
    if (res.status / 100 != 2 and res.status != 404):
        print "clear_all_objects_in_bucket: delete bucket fail, ret is:", res.status
        return False
    return True

if __name__ == '__main__':
    pass
oss/oss_xml_handler.py0000664000076400007650000002652311654134241013561 0ustar  leoleo#coding=utf8
from xml.dom import minidom

def get_tag_text(element, tag):
    nodes = element.getElementsByTagName(tag)
    if len(nodes) == 0:
        return ""
    else:
        node = nodes[0]
    rc = ""
    for node in node.childNodes:
        if node.nodeType in ( node.TEXT_NODE, node.CDATA_SECTION_NODE):
            rc = rc + node.data
    return rc

class ErrorXml:
    def __init__(self, xml_string):
        self.xml = minidom.parseString(xml_string)
        self.code = get_tag_text(self.xml, 'Code')
        self.msg = get_tag_text(self.xml, 'Message')
        self.resource = get_tag_text(self.xml, 'Resource')
        self.request_id = get_tag_text(self.xml, 'RequestId')
        self.host_id = get_tag_text(self.xml, 'HostId')
    
    def show(self):
        print "Code: %s\nMessage: %s\nResource: %s\nRequestId: %s \nHostId: %s" % (self.code, self.msg, self.resource, self.request_id, self.host_id)

class Owner:
    def __init__(self, xml_element):
        self.element = xml_element
        self.id = get_tag_text(self.element, "ID")
        self.display_name = get_tag_text(self.element, "DisplayName")
    
    def show(self):
        print "ID: %s\nDisplayName: %s" % (self.id, self.display_name)

class Bucket:
    def __init__(self, xml_element):
        self.element = xml_element
        self.name = get_tag_text(self.element, "Name")
        self.creation_date = get_tag_text(self.element, "CreationDate")
    
    def show(self):
        print "Name: %s\nCreationDate: %s" % (self.name, self.creation_date)

class GetServiceXml:
    def __init__(self, xml_string):
        self.xml = minidom.parseString(xml_string)
        self.owner = Owner(self.xml.getElementsByTagName('Owner')[0])
        self.buckets = self.xml.getElementsByTagName('Bucket')
        self.bucket_list = []
        for b in self.buckets:
            self.bucket_list.append(Bucket(b))

    def show(self):
        print "Owner:"
        self.owner.show()
        print "\nBucket list:"
        for b in self.bucket_list:
            b.show()
            print ""

    def list(self):
        bl = []
        for b in self.bucket_list:
            bl.append((b.name, b.creation_date))
        return bl
    
class Content:
    def __init__(self, xml_element):
        self.element = xml_element
        self.key = get_tag_text(self.element, "Key")        
        self.last_modified = get_tag_text(self.element, "LastModified")        
        self.etag = get_tag_text(self.element, "ETag")        
        self.size = get_tag_text(self.element, "Size")        
        self.owner = Owner(self.element.getElementsByTagName('Owner')[0])
        self.storage_class = get_tag_text(self.element, "StorageClass")        

    def show(self):
        print "Key: %s\nLastModified: %s\nETag: %s\nSize: %s\nStorageClass: %s" % (self.key, self.last_modified, self.etag, self.size, self.storage_class)
        self.owner.show()

class Part:
    def __init__(self, xml_element):
        self.element = xml_element
        self.part_num = get_tag_text(self.element, "PartNumber")        
        self.object_name = get_tag_text(self.element, "PartName")
        self.object_size = get_tag_text(self.element, "PartSize")
        self.etag = get_tag_text(self.element, "ETag")

    def show(self):
        print "PartNumber: %s\nPartName: %s\nPartSize: %s\nETag: %s\n" % (self.part_num, self.object_name, self.object_size, self.etag)

class PostObjectGroupXml:
    def __init__(self, xml_string):
        self.xml = minidom.parseString(xml_string)
        self.bucket = get_tag_text(self.xml, 'Bucket')
        self.key = get_tag_text(self.xml, 'Key')
        self.size = get_tag_text(self.xml, 'Size')
        self.etag = get_tag_text(self.xml, "ETag")

    def show(self):
        print "Post Object Group, Bucket: %s\nKey: %s\nSize: %s\nETag: %s" % (self.bucket, self.key, self.size, self.etag)

class GetObjectGroupIndexXml:
    def __init__(self, xml_string):
        self.xml = minidom.parseString(xml_string)
        self.bucket = get_tag_text(self.xml, 'Bucket')
        self.key = get_tag_text(self.xml, 'Key')
        self.etag = get_tag_text(self.xml, 'Etag')
        self.file_length = get_tag_text(self.xml, 'FileLength')
        self.index_list = []
        index_lists = self.xml.getElementsByTagName('Part')
        for i in index_lists:
            self.index_list.append(Part(i))

    def list(self):
        index_list = []
        for i in self.index_list:
            index_list.append((i.part_num, i.object_name, i.object_size, i.etag))
        return index_list

    def show(self):
        print "Bucket: %s\nObject: %s\nEtag: %s\nObjectSize: %s" % (self.bucket, self.key, self.etag, self.file_length)
        print "\nPart list:"
        for p in self.index_list:
            p.show()

class GetBucketXml:
    def __init__(self, xml_string):
        self.xml = minidom.parseString(xml_string)
        self.name = get_tag_text(self.xml, 'Name')
        self.prefix = get_tag_text(self.xml, 'Prefix')
        self.marker = get_tag_text(self.xml, 'Marker')
        self.nextmarker = get_tag_text(self.xml, 'NextMarker')
        self.maxkeys = get_tag_text(self.xml, 'MaxKeys')
        self.delimiter = get_tag_text(self.xml, 'Delimiter')
        self.is_truncated = get_tag_text(self.xml, 'IsTruncated')

        self.prefix_list = []
        prefixes = self.xml.getElementsByTagName('CommonPrefixes')
        for p in prefixes:
            tag_txt = get_tag_text(p, "Prefix")
            self.prefix_list.append(tag_txt)

        self.content_list = []
        contents = self.xml.getElementsByTagName('Contents')
        for c in contents:
            self.content_list.append(Content(c))

    def show(self):
        print "Name: %s\nPrefix: %s\nMarker: %s\nNextMarker: %s\nMaxKeys: %s\nDelimiter: %s\nIsTruncated: %s" % (self.name, self.prefix, self.marker, self.nextmarker, self.maxkeys, self.delimiter, self.is_truncated)
        print "\nPrefix list:"
        for p in self.prefix_list:
            print p
        print "\nContent list:"
        for c in self.content_list:
            c.show()
            print ""

    def list(self):
        cl = []
        pl = []
        for c in self.content_list:
            cl.append((c.key, c.last_modified, c.etag, c.size, c.owner.id, c.owner.display_name, c.storage_class))
        for p in self.prefix_list:
            pl.append(p)

        return (cl, pl)
 
class GetBucketAclXml:
    def __init__(self, xml_string):
        self.xml = minidom.parseString(xml_string)
        if len(self.xml.getElementsByTagName('Owner')) != 0:
            self.owner = Owner(self.xml.getElementsByTagName('Owner')[0])
        else:
            self.owner = "" 
        self.grant = get_tag_text(self.xml, 'Grant')

    def show(self):
        print "Owner Name: %s\nOwner ID: %s\nGrant: %s" % (self.owner.id, self.owner.display_name, self.grant)
       
def test_get_bucket_xml():
    body = "<?xml version=\"1.0\" encoding=\"UTF-8\"?><ListBucketResult xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\"><Name>sweet-memory</Name><Prefix>IMG</Prefix><Marker>IMG_0</Marker><MaxKeys>1000</MaxKeys><IsTruncated>false</IsTruncated><Contents><Key>IMG_2744.JPG</Key><LastModified>2011-03-04T06:20:37.000Z</LastModified><ETag>&quot;a56047f218618a43a9b1c2dca2d8c592&quot;</ETag><Size>220778</Size><Owner><ID>2cfd76976de6f4c6f4e05fcd02680c4ca619428123681589efcb203f29dce924</ID><DisplayName>sanbo_ustc</DisplayName></Owner><StorageClass>STANDARD</StorageClass></Contents><Contents><Key>IMG_2745.JPG</Key><LastModified>2011-03-04T06:20:39.000Z</LastModified><ETag>&quot;511c0b52911bcd667338103c385741af&quot;</ETag><Size>244612</Size><Owner><ID>2cfd76976de6f4c6f4e05fcd02680c4ca619428123681589efcb203f29dce924</ID><DisplayName>sanbo_ustc</DisplayName></Owner><StorageClass>STANDARD</StorageClass></Contents><Contents><Key>IMG_3344.JPG</Key><LastModified>2011-03-04T06:20:48.000Z</LastModified><ETag>&quot;4ea11d796ecc742b216864dcf5dfd193&quot;</ETag><Size>229211</Size><Owner><ID>2cfd76976de6f4c6f4e05fcd02680c4ca619428123681589efcb203f29dce924</ID><DisplayName>sanbo_ustc</DisplayName></Owner><StorageClass>STANDARD</StorageClass></Contents><Contents><Key>IMG_3387.JPG</Key><LastModified>2011-03-04T06:20:53.000Z</LastModified><ETag>&quot;c32b5568ae4fb0a3421f0daba25ecfd4&quot;</ETag><Size>460062</Size><Owner><ID>2cfd76976de6f4c6f4e05fcd02680c4ca619428123681589efcb203f29dce924</ID><DisplayName>sanbo_ustc</DisplayName></Owner><StorageClass>STANDARD</StorageClass></Contents><Contents><Key>IMG_3420.JPG</Key><LastModified>2011-03-04T06:20:25.000Z</LastModified><ETag>&quot;edf010d2a8a4877ce0362b245fcc963b&quot;</ETag><Size>174973</Size><Owner><ID>2cfd76976de6f4c6f4e05fcd02680c4ca619428123681589efcb203f29dce924</ID><DisplayName>sanbo_ustc</DisplayName></Owner><StorageClass>STANDARD</StorageClass></Contents><Contents><Key>中文.case</Key><LastModified>2011-03-04T06:20:26.000Z</LastModified><ETag>&quot;7fd64eec21799ef048ed827cf6098f06&quot;</ETag><Size>208134</Size><Owner><ID>2cfd76976de6f4c6f4e05fcd02680c4ca619428123681589efcb203f29dce924</ID><DisplayName>sanbo_ustc</DisplayName></Owner><StorageClass>STANDARD</StorageClass></Contents></ListBucketResult>"
    h = GetBucketXml(body)
    h.show()
    (fl, pl) = h.list()
    print "\nfile_list: ", fl
    print "prefix list: ", pl

def test_get_service_xml():
    body = "<?xml version=\"1.0\" encoding=\"UTF-8\"?><ListAllMyBucketsResult xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\"><Owner><ID>2cfd76976de6f4c6f4e05fcd02680c4ca619428123681589efcb203f29dce924</ID><DisplayName>sanbo_ustc</DisplayName></Owner><Buckets><Bucket><Name>360buy</Name><CreationDate>2011-03-04T09:25:37.000Z</CreationDate></Bucket><Bucket><Name>aliyun-test-test</Name><CreationDate>2011-04-11T12:24:06.000Z</CreationDate></Bucket><Bucket><Name>irecoffee</Name><CreationDate>2011-03-04T06:14:56.000Z</CreationDate></Bucket><Bucket><Name>sweet-memory</Name><CreationDate>2011-04-12T11:56:04.000Z</CreationDate></Bucket></Buckets></ListAllMyBucketsResult>"
    h = GetServiceXml(body)
    h.show()
    print "\nbucket list: ", h.list()

def test_get_bucket_acl_xml():
    body = '<?xml version="1.0" ?><AccessControlPolicy><Owner><ID>61155b1e39dbca1d0d0f3c7faa32d9e8e9a90a9cd86edbd27d8eed5d0ad8ce82</ID><DisplayName>megjian</DisplayName></Owner><AccessControlList><Grant>public-read-write</Grant></AccessControlList></AccessControlPolicy>'
    h = GetBucketAclXml(body)
    h.show()

def test_get_object_group_xml():
    body = '<?xml version="1.0" encoding="UTF-8"?><FileGroup><Bucket>ut_test_post_object_group</Bucket> <Key>ut_test_post_object_group</Key> <Etag>&quot;91E8503F4DA1324E28434AA6B6E20D15&quot;</Etag><FileLength>1073741824</FileLength> <FilePart><Part> <PartNumber>1</PartNumber> <ObjectName>4d37380c7149508bedf78dc7c5c68f55_test_post_object_group.txt_1</ObjectName><ObjectSize>10485760</ObjectSize><ETag>&quot;A957A9F1EF44ED7D40CD5C738D113509&quot;</ETag></Part><Part><PartNumber>2</PartNumber><ObjectName>7aa26b8da263589e875d179b87642691_test_post_object_group.txt_2</ObjectName><ObjectSize>10485760</ObjectSize><ETag>&quot;A957A9F1EF44ED7D40CD5C738D113509&quot;</ETag></Part><Part><PartNumber>3</PartNumber><ObjectName>28b0c8a9bd69469f76cd102d6e1b0f03_test_post_object_group.txt_3</ObjectName><ObjectSize>10485760</ObjectSize><ETag>&quot;A957A9F1EF44ED7D40CD5C738D113509&quot;</ETag></Part></FilePart></FileGroup>'
    h = GetObjectGroupIndexXml(body)
    h.show()

if __name__ == "__main__":
    test_get_bucket_xml()
    test_get_service_xml()
    test_get_bucket_acl_xml()
    test_get_object_group_xml()
oss/oss_sample.py0000664000076400007650000001527411654134217012551 0ustar  leoleo#coding=utf8
import time
from oss_xml_handler import *
from oss_api import *

HOST="storage.aliyun.com"
ACCESS_ID = ""
SECRET_ACCESS_KEY = ""
#ACCESS_ID and SECRET_ACCESS_KEY should not be empty, please input correct one.
   
if __name__ == "__main__": 
    #init oss,  get instance of oss
    if len(ACCESS_ID) == 0 or len(SECRET_ACCESS_KEY) == 0:
        print "Please make sure ACCESS_ID and SECRET_ACCESS_KEY are correct in ", __file__ , ", init are empty!"
        exit(0) 
    oss = OssAPI(HOST, ACCESS_ID, SECRET_ACCESS_KEY)
    sep = "=============================="
   
    #sign_url_auth_with_expire_time(self, method, url, headers = {}, resource="/", timeout = 60):
    method = "GET"
    bucket = "test" + time.strftime("%Y-%b-%d%H-%M-%S").lower()
    object = "test_object" 
    url = "http://" + HOST + "/oss/" + bucket + "/" + object
    headers = {}
    resource = "/" + bucket + "/" + object

    timeout = 60
    url_with_auth = oss.sign_url_auth_with_expire_time(method, url, headers, resource, timeout)
    print "after signature url is: ", url_with_auth
    print sep
    #put_bucket(self, bucket, acl='', headers = {}):
    acl = 'private'
    headers = {}
    res = oss.put_bucket(bucket, acl, headers)
    if (res.status / 100) == 2:
        print "put bucket ", bucket, "OK"
    else:
        print "put bucket ", bucket, "ERROR"
    print sep

    #get_service(self):
    res = oss.get_service()
    if (res.status / 100) == 2:
        body = res.read()
        h = GetServiceXml(body)
        print "bucket list size is: ", len(h.list())
        print "bucket list is: "
        for i in h.list():
            print i
    else:
        print res.status
    print sep

    #put_object_from_string(self, bucket, object, input_content, content_type=DefaultContentType, headers = {}):
    object = "object_test"
    input_content = "hello, OSS"
    content_type = "text/HTML"
    headers = {}
    res = oss.put_object_from_string(bucket, object, input_content, content_type, headers)
    if (res.status / 100) == 2:
        print "put_object_from_string OK"
    else:
        print "put_object_from_string ERROR"
    print sep
    
    #put_object_from_file(self, bucket, object, filename, content_type=DefaultContentType, headers = {}):
    object = "object_test"
    filename = __file__ 
    content_type = "text/HTML"
    headers = {}
    res = oss.put_object_from_file(bucket, object, filename, content_type, headers)
    if (res.status / 100) == 2:
        print "put_object_from_file OK"
    else:
        print "put_object_from_file ERROR"
    print sep
 
    #put_object_from_fp(self, bucket, object, fp, content_type=DefaultContentType, headers = {}):
    object = "object_test"
    filename = __file__
    content_type = "text/HTML"
    headers = {}

    fp = open(filename, 'rb')
    res = oss.put_object_from_fp(bucket, object, fp, content_type, headers)
    fp.close()
    if (res.status / 100) == 2:
        print "put_object_from_fp OK"
    else:
        print "put_object_from_fp ERROR"
    print sep

    #get_object(self, bucket, object, headers = {}):
    object = "object_test"
    headers = {}

    res = oss.get_object(bucket, object, headers)
    if (res.status / 100) == 2:
        print "get_object OK"
    else:
        print "get_object ERROR"
    print sep

    #get_object_to_file(self, bucket, object, filename, headers = {}):
    object = "object_test"
    headers = {}
    filename = "get_object_test_file"

    res = oss.get_object_to_file(bucket, object, filename, headers)
    if (res.status / 100) == 2:
        print "get_object_to_file OK"
    else:
        print "get_object_to_file ERROR"
    print sep

    #head_object(self, bucket, object, headers = {}):
    object = "object_test"
    headers = {}
    res = oss.head_object(bucket, object, headers)
    if (res.status / 100) == 2:
         print "head_object OK"
         header_map = convert_header2map(res.getheaders())
         content_len = safe_get_element("content-length", header_map)
         etag = safe_get_element("etag", header_map).upper()
         print "content length is:", content_len
         print "ETag is: ", etag

    else:
        print "head_object ERROR"
    print sep
    
    #get_bucket_acl(self, bucket):
    res = oss.get_bucket_acl(bucket)
    if (res.status / 100) == 2:
        body = res.read()
        h = GetBucketAclXml(body)
        print "bucket acl is:", h.grant 
    else:
        print "get bucket acl ERROR"
    print sep

    #get_bucket(self, bucket, prefix='', marker='', delimiter='', maxkeys='', headers = {}):
    prefix = ""
    marker = ""
    delimiter = "/"
    maxkeys = "100"
    headers = {}
    res = oss.get_bucket(bucket, prefix, marker, delimiter, maxkeys, headers)
    if (res.status / 100) == 2:
        body = res.read()
        h = GetBucketXml(body)
        (file_list, common_list) = h.list()
        print "object list is:"
        for i in file_list:
            print i
        print "common list is:"
        for i in common_list:
            print i
    print sep
 
    #upload_large_file(self, bucket, object, filename, thread_num = 10, max_part_num = 1000):
    res = oss.upload_large_file(bucket, object, __file__)    
    if (res.status / 100) == 2:
        print "upload_large_file OK"
    else:
        print "upload_large_file ERROR"

    print sep

    #get_object_group_index(self, bucket, object, headers = {})
    res = oss.get_object_group_index(bucket, object)
    if (res.status / 100) == 2:
        print "get_object_group_index OK"
        body = res.read()
        h = GetObjectGroupIndexXml(body)
        for i in h.list():
            print "object group part msg:", i
    else:
        print "get_object_group_index ERROR"

    res = oss.get_object_group_index(bucket, object)
    if res.status == 200:
        body = res.read()
        h = GetObjectGroupIndexXml(body)
        object_group_index = h.list()
        for i in object_group_index:
            if len(i) == 4 and len(i[1]) > 0:
                part_name = i[1].strip()
                res = oss.delete_object(bucket, part_name)
                if res.status != 204:
                    print "delete part ", part_name, " in bucket:", bucket, " failed!"
                else:
                    print "delete part ", part_name, " in bucket:", bucket, " ok"
    print sep

    #delete_object(self, bucket, object, headers = {}):
    object = "object_test"
    headers = {}
    res = oss.delete_object(bucket, object, headers)
    if (res.status / 100) == 2:
        print "delete_object OK"
    else:
        print "delete_object ERROR"
    print sep

    #delete_bucket(self, bucket):
    res = oss.delete_bucket(bucket)
    if (res.status / 100) == 2:
        print "delete bucket ", bucket, "OK"
    else:
        print "delete bucket ", bucket, "ERROR"

    print sep


oss/oss_api.py0000664000076400007650000005031211654134206012027 0ustar  leoleo#coding=utf8

# Copyright (c) 2011, Alibaba Cloud Computing 
# All rights reserved.
#
# Permission is hereby granted, free of charge, to any person obtaining a
# copy of this software and associated documentation files (the
# "Software"), to deal in the Software without restriction, including
# without limitation the rights to use, copy, modify, merge, publish, dis-
# tribute, sublicense, and/or sell copies of the Software, and to permit
# persons to whom the Software is furnished to do so, subject to the fol-
# lowing conditions:
#
# The above copyright notice and this permission notice shall be included
# in all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL-
# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT
# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, 
# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
# IN THE SOFTWARE.

import httplib
import time
import base64
import hmac
import os
import urllib
import StringIO
from oss_util import *
from hashlib import sha1 as sha
        
class OssAPI:
    '''
    A simple OSS API 
    '''
    DefaultContentType = 'application/octet-stream'
    SendBufferSize = 8192
    GetBufferSize = 1024*1024*10

    def __init__(self, host, access_id, secret_access_key=""):
        self.host = host
        self.access_id = access_id
        self.secret_access_key = secret_access_key

    def sign_url_auth_with_expire_time(self, method, url, headers = {}, resource="/", timeout = 60):
        '''
        Create the authorization for OSS based on the input method, url, body and headers
        :type method: string
        :param method: one of PUT, GET, DELETE, HEAD 

        :type url: string 
        :param:HTTP address of bucket or object, eg: http://HOST/bucket/object

        :type headers: dict
        :param: HTTP header
        
        :type resource: string
        :param:path of bucket or object, eg: /bucket/ or /bucket/object 

        :type timeout: int
        :param

        Returns:
            signature url.
        '''

        send_time= safe_get_element('Date', headers)
        if len(send_time) == 0:
            send_time = str(int(time.time()) + timeout)
        headers['Date'] = send_time
        auth_value = get_assign(self.secret_access_key, method, headers, resource)
        params = {"OSSAccessKeyId":self.access_id, "Expires":str(send_time), "Signature":auth_value}
        return append_param(url, params)

    def _create_sign_for_normal_auth(self, method, headers = {}, resource="/"):
        '''
        NOT public API
        Create the authorization for OSS based on header input.
        it should be put into "Authorization" parameter of header.

        :type method: string
        :param:one of PUT, GET, DELETE, HEAD 
        
        :type headers: dict
        :param: HTTP header

        :type resource: string
        :param:path of bucket or object, eg: /bucket/ or /bucket/object

        Returns:
            signature string
        
        '''

        auth_value = "OSS " + self.access_id + ":" + get_assign(self.secret_access_key, method, headers, resource)
        return auth_value
       
    def bucket_operation(self, method, bucket, headers={}, params={}):
        '''
        Send bucket operation request 

        :type method: string
        :param method: one of PUT, GET, DELETE, HEAD

        :type bucket: string
        :param

        :type headers: dict
        :param: HTTP header

        :type params: dict
        :param: parameters that will be appeded at the end of resource 

        Returns:
            HTTP Response

        '''

        url = append_param("/" + bucket + "/", params)
        date = time.strftime("%a, %d %b %Y %H:%M:%S GMT", time.gmtime())
        # Create REST header
        headers['Date'] = date
        headers['Host'] = self.host
        if params.has_key('acl'):
            resource = "/" + bucket + "/?acl"
        else:
            resource = "/" + bucket + "/"
        if "" != self.secret_access_key and "" != self.access_id:
            headers['Authorization'] = self._create_sign_for_normal_auth(method, headers, resource) 
        elif "" != self.access_id:
            headers['Authorization'] = self.access_id 
            
        conn = httplib.HTTPConnection(self.host)
        conn.request(method, url, "", headers)
        return conn.getresponse()

    def object_operation(self, method, bucket, object, headers = {}, data=""):
        '''
        Send Object operation request
        
        :type method: string
        :param method: one of PUT, GET, DELETE, HEAD

        :type bucket: string
        :param
        
        :type object: string
        :param
        
        :type headers: dict
        :param: HTTP header

        :type data: string
        :param
        
        Returns:
            HTTP Response
        '''
        
        if isinstance(object, unicode):
            object = object.encode('utf-8')
        resource = "/" + bucket + "/"
        resource = resource.encode('utf-8') + object 
        object = urllib.quote(object)
        url = "/" + bucket + "/" + object
        date = time.strftime("%a, %d %b %Y %H:%M:%S GMT", time.gmtime())

         # Create REST header
        headers['Date'] = date
        headers['Host'] = self.host
        if "" != self.secret_access_key and "" != self.access_id:
            headers['Authorization'] = self._create_sign_for_normal_auth(method, headers, resource) 
        elif "" != self.access_id:
            headers['Authorization'] = self.access_id 

        conn = httplib.HTTPConnection(self.host)
        conn.request(method, url, data, headers)
        return conn.getresponse()

    def get_service(self):
        '''
        List all buckets of user
        '''
        return self.list_all_my_buckets()

    def list_all_my_buckets(self):
        '''
        List all buckets of user
        '''

        method = "GET"
        url = "/"
        date = time.strftime("%a, %d %b %Y %H:%M:%S GMT", time.gmtime())
        headers = {}

        headers['Date'] = date
        headers['Host'] = self.host
        resource = "/"
        if "" != self.secret_access_key and "" != self.access_id:
            headers['Authorization'] = self._create_sign_for_normal_auth(method, headers, resource) 
        elif "" != self.access_id:
            headers['Authorization'] = self.access_id 
     
        conn = httplib.HTTPConnection(self.host)
        conn.request(method, url, "", headers)
        return conn.getresponse()

    def get_bucket_acl(self, bucket):
        '''
        Get Access Control Level of bucket
        
        :type bucket: string
        :param

        Returns:
            HTTP Response
        '''
        headers = {}
        params = {}
        params['acl'] = ''
        return self.bucket_operation("GET", bucket, headers, params)

    def get_bucket(self, bucket, prefix='', marker='', delimiter='', maxkeys='', headers = {}):
        '''
        List object that in bucket
        '''
        return self.list_bucket(bucket, prefix, marker, delimiter, maxkeys, headers)

    def list_bucket(self, bucket, prefix='', marker='', delimiter='', maxkeys='', headers = {}):
        '''
        List object that in bucket

        :type bucket: string
        :param

        :type prefix: string
        :param  
        
        :type marker: string
        :param
        
        :type delimiter: string
        :param
        
        :type maxkeys: string
        :param

        :type headers: dict
        :param: HTTP header

        Returns:
            HTTP Response
        '''

        params = {}
        params['prefix'] = prefix
        params['marker'] = marker
        params['delimiter'] = delimiter
        params['max-keys'] = maxkeys
        return self.bucket_operation("GET", bucket, headers, params)

    def create_bucket(self, bucket, acl='', headers = {}):
        '''
        Create bucket
        '''
        return self.put_bucket(bucket, acl, headers)

    def put_bucket(self, bucket, acl='', headers = {}):
        '''
        Create bucket
        
        :type bucket: string
        :param

        :type acl: string
        :param: one of private
        
        :type headers: dict
        :param: HTTP header

        Returns:
            HTTP Response
        '''

        if acl != '':
            headers['x-oss-acl'] = acl
        return self.bucket_operation("PUT", bucket, headers)

    def delete_bucket(self, bucket):
        '''
        Delete bucket
        
        :type bucket: string
        :param
        
        Returns:
            HTTP Response
        '''

        return self.bucket_operation("DELETE", bucket)

    def put_object_with_data(self, bucket, object, input_content, content_type=DefaultContentType, headers = {}):
        '''
        Put object into bucket, the content of object is from input_content
        '''
        return self.put_object_from_string(bucket, object, input_content, content_type, headers)

    def put_object_from_string(self, bucket, object, input_content, content_type=DefaultContentType, headers = {}):
        '''
        Put object into bucket, the content of object is from input_content

        :type bucket: string
        :param
        
        :type object: string
        :param
        
        :type input_content: string
        :param

        :type content_type: string
        :param: the object content type that supported by HTTP

        :type headers: dict
        :param: HTTP header

        Returns:
            HTTP Response
        '''

        headers['Content-Type'] = content_type
        headers['Content-Length'] = str(len(input_content))
        fp = StringIO.StringIO(input_content)
        res = self.put_object_from_fp(bucket, object, fp, content_type, headers)
        fp.close()
        return res

    def _open_conn_to_put_object(self, bucket, object, filesize, content_type=DefaultContentType, headers = {}):
        '''
        NOT public API
        Open a connectioon to put object

        :type bucket: string
        :param

        :type filesize: int 
        :param
        
        :type object: string
        :param
        
        :type input_content: string
        :param

        :type content_type: string
        :param: the object content type that supported by HTTP

        :type headers: dict
        :param: HTTP header

        Returns:
            HTTP Response
        '''

        method = "PUT"
        if isinstance(object, unicode):
            object = object.encode('utf-8')
        resource = "/" + bucket + "/"
        resource = resource.encode('utf-8') + object 
        object = urllib.quote(object)
        url = "/" + bucket + "/" + object
        date = time.strftime("%a, %d %b %Y %H:%M:%S GMT", time.gmtime())

        conn = httplib.HTTPConnection(self.host)
        conn.putrequest(method, url)
        headers["Content-Type"] = content_type
        headers["Content-Length"] = filesize
        headers["Date"] = date
        headers["Host"] = self.host
        headers["Expect"] = "100-Continue"
        for k in headers.keys():
            conn.putheader(k, headers[k])
        if "" != self.secret_access_key and "" != self.access_id:
            auth = self._create_sign_for_normal_auth(method, headers, resource)
            conn.putheader("Authorization", auth)
        conn.endheaders()
        return conn

    def put_object_from_file(self, bucket, object, filename, content_type=DefaultContentType, headers = {}):
        '''
        put object into bucket, the content of object is read from file        

        :type bucket: string
        :param
        
        :type object: string
        :param
        
        :type fllename: string
        :param: the name of the read file 

        :type content_type: string
        :param: the object content type that supported by HTTP

        :type headers: dict
        :param: HTTP header

        Returns:
            HTTP Response
        '''

        fp = open(filename, 'rb')
        res = self.put_object_from_fp(bucket, object, fp, content_type, headers)
        fp.close()
        return res

    def put_object_from_fp(self, bucket, object, fp, content_type=DefaultContentType, headers = {}):
        '''
        Put object into bucket, the content of object is read from file pointer

        :type bucket: string
        :param
        
        :type object: string
        :param
        
        :type fp: file 
        :param: the pointer of the read file 

        :type content_type: string
        :param: the object content type that supported by HTTP

        :type headers: dict
        :param: HTTP header

        Returns:
            HTTP Response
        '''
        
        fp.seek(os.SEEK_SET, os.SEEK_END)
        filesize = fp.tell()
        fp.seek(os.SEEK_SET)

        conn = self._open_conn_to_put_object(bucket, object, filesize, content_type, headers)
        l = fp.read(self.SendBufferSize)
        while len(l) > 0:
            conn.send(l)
            l = fp.read(self.SendBufferSize)
        return conn.getresponse()

    def get_object(self, bucket, object, headers = {}):
        '''
        Get object

        :type bucket: string
        :param
        
        :type object: string
        :param

        :type headers: dict
        :param: HTTP header

        Returns:
            HTTP Response
        '''

        return self.object_operation("GET", bucket, object, headers)

    def get_object_to_file(self, bucket, object, filename, headers = {}):
        '''
        Get object and write the content of object into a file

        :type bucket: string
        :param
        
        :type object: string
        :param

        :type filename: string
        :param

        :type headers: dict
        :param: HTTP header

        Returns:
            HTTP Response
        '''

        res = self.get_object(bucket, object, headers)
        f = file(filename, 'wb')
        data = ""
        while True:
            data = res.read(self.GetBufferSize)
            if len(data) != 0:
                f.write(data)
            else:
                break
        f.close()
        # TODO: get object with flow
        return res

    def delete_object(self, bucket, object, headers = {}):
        '''
        Delete object
        
        :type bucket: string
        :param
        
        :type object: string
        :param

        :type headers: dict
        :param: HTTP header

        Returns:
            HTTP Response
        '''

        return self.object_operation("DELETE", bucket, object, headers)

    def head_object(self, bucket, object, headers = {}):
        '''
        Head object, to get the meta message of object without the content
        
        :type bucket: string
        :param
        
        :type object: string
        :param

        :type headers: dict
        :param: HTTP header

        Returns:
            HTTP Response
        '''

        return self.object_operation("HEAD", bucket, object, headers)

    def post_object_group(self, bucket, object, object_group_msg_xml, headers = {}, params = {}):
        '''
        Post object group, merge all objects in object_group_msg_xml into one object
        :type bucket: string
        :param

        :type object: string
        :param
        
        :type object_group_msg_xml: string
        :param: xml format string, like
                <CreateFileGroup>
                    <Part>
                        <PartNumber>N</PartNumber>
                        <FileName>objectN</FileName>
                        <Etag>"47BCE5C74F589F4867DBD57E9CA9F808"</Etag>
                    </Part>
                </CreateFileGroup>
        :type headers: dict
        :param: HTTP header

        :type params: dict
        :param: parameters
        
        Returns:
            HTTP Response
        '''
        method = "POST"
        url = "/" + bucket + "/" + object + "?group"
        date = time.strftime("%a, %d %b %Y %H:%M:%S GMT", time.gmtime())
        # Create REST header
        headers['Date'] = date
        headers['Host'] = self.host
        headers['Content-Type'] = 'text/xml'
        headers['Content-Length'] = len(object_group_msg_xml)
        resource = "/" + bucket + "/" + object + "?group"
        if "" != self.secret_access_key and "" != self.access_id:
            headers['Authorization'] = self._create_sign_for_normal_auth(method, headers, resource)
        elif "" != self.access_id:
            headers['Authorization'] = self.access_id 

        conn = httplib.HTTPConnection(self.host)
        conn.request(method, url, object_group_msg_xml, headers)
        return conn.getresponse()

    def get_object_group_index(self, bucket, object, headers = {}):
        '''
        Get object group_index 

        :type bucket: string
        :param
        
        :type object: string
        :param

        :type headers: dict
        :param: HTTP header

        Returns:
            HTTP Response
        '''

        headers["x-oss-file-group"] = ""
        return self.object_operation("GET", bucket, object, headers)


    def put_object_from_file_given_pos(self, bucket, object, filename, offset, partsize, content_type=DefaultContentType, headers = {}):
        '''
        Put object into bucket, the content of object is read from given posision of filename
        :type bucket: string
        :param
        
        :type object: string
        :param
        
        :type fllename: string
        :param: the name of the read file 
        
        :type offset: int
        :param: the given position of file

        :type partsize: int
        :param: the size of read content 
    
        :type content_type: string
        :param: the object content type that supported by HTTP

        :type headers: dict
        :param: HTTP header

        Returns:
            HTTP Response
        '''
        fp = open(filename, 'rb')
        if offset > os.path.getsize(filename):
            fp.seek(os.SEEK_SET, os.SEEK_END)
        else:
            fp.seek(offset)

        conn = self._open_conn_to_put_object(bucket, object, partsize, content_type, headers)

        left_len = partsize 
        while True:
            if left_len <= 0:
               break
            elif left_len < self.SendBufferSize:
               buffer_content = fp.read(left_len)
            else:
               buffer_content = fp.read(self.SendBufferSize)

            if len(buffer_content) > 0:
                conn.send(buffer_content)

            left_len = left_len - len(buffer_content)

        fp.close()
        return conn.getresponse()

    def upload_large_file(self, bucket, object, filename, thread_num = 10, max_part_num = 1000):
        '''
        Upload large file, the content is read from filename. The large file is splitted into many parts. It will        put the many parts into bucket and then merge all the parts into one object. 

        :type bucket: string
        :param
        
        :type object: string
        :param
        
        :type fllename: string
        :param: the name of the read file 
 
        '''
        #split the large file into 1000 parts or many parts
        #get part_msg_list
        if isinstance(filename, unicode):
            filename = filename.encode('utf-8')

        part_msg_list = split_large_file(filename, object, max_part_num)

        #make sure all the parts are put into same bucket
        if len(part_msg_list) < thread_num and len(part_msg_list) != 0:
            thread_num = len(part_msg_list)

        step = len(part_msg_list) / thread_num
        threadpool = []
        for i in range(0, thread_num):
            if i == thread_num - 1:
                end = len(part_msg_list)
            else:
                end = i * step + step

            begin = i * step
            oss = OssAPI(self.host, self.access_id, self.secret_access_key)
            current = PutObjectGroupWorker(oss, bucket, filename, part_msg_list[begin:end])
            threadpool.append(current)
            current.start()

        for item in threadpool:
            item.join()
       
        #get xml string that contains msg of object group
        object_group_msg_xml = create_object_group_msg_xml(part_msg_list)

        return self.post_object_group(bucket, object, object_group_msg_xml)