From 042c4caebf1dbd0cc7c2e971d5ef2b002e7ae707 Mon Sep 17 00:00:00 2001 From: Enrico Bocchi Date: Tue, 5 Jun 2018 10:19:54 +0200 Subject: [PATCH 01/10] Fix 'cboxshareadmin.ini' variable name from args --- cernbox-share | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cernbox-share b/cernbox-share index 4db29d6..ddaf698 100755 --- a/cernbox-share +++ b/cernbox-share @@ -110,7 +110,7 @@ def main(): global args args = parser.parse_args() - config = cernbox_utils.script.configure(args.config) + config = cernbox_utils.script.configure(args.configfile) logger = cernbox_utils.script.getLogger(level=args.loglevel) From f06e5ef739365f4392ec19c07dad16d9bbcafca4 Mon Sep 17 00:00:00 2001 From: Enrico Bocchi Date: Tue, 5 Jun 2018 11:17:27 +0200 Subject: [PATCH 02/10] Pass 'None' as dummy share type to list_share function --- cernbox-share | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cernbox-share b/cernbox-share index ddaf698..eb3b3e7 100755 --- a/cernbox-share +++ b/cernbox-share @@ -231,7 +231,7 @@ def cmd_list_shares(args,role): import cernbox_utils.sharing groups={} - retobj = cernbox_utils.sharing.list_shares(args.user,role,groups,None,args.flat_list,False,db,eos) + retobj = cernbox_utils.sharing.list_shares(args.user,role,groups,None,None,args.flat_list,False,db,eos) return {'shares':retobj} From 48c79091c7a6537fed0de1db17f3dd811f70ff50 Mon Sep 17 00:00:00 2001 From: Enrico Bocchi Date: Tue, 5 Jun 2018 11:19:40 +0200 Subject: [PATCH 03/10] Handle 'None' being returned when sharing-by-link --- python/cernbox_utils/sharing.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/python/cernbox_utils/sharing.py b/python/cernbox_utils/sharing.py index 4111eeb..8cba74b 100644 --- a/python/cernbox_utils/sharing.py +++ b/python/cernbox_utils/sharing.py @@ -16,9 +16,11 @@ def share2acl(s): # this is the expected ACL entry in the shared directory tree acl = eos.AclEntry(name=s.share_with) - if is_egroup(s.share_with): + if s.share_with is None: # Share by link + acl.entity = "-" + elif is_egroup(s.share_with): # Share with egroups acl.entity = "egroup" - else: + else: # Authenticated shares acl.entity = "u" if s.permissions == 1: From b912d68a70c122280398c17287dc773f672ad8b0 Mon Sep 17 00:00:00 2001 From: Enrico Bocchi Date: Tue, 5 Jun 2018 11:52:32 +0200 Subject: [PATCH 04/10] add_share function requires additional params --- cernbox-share | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cernbox-share b/cernbox-share index eb3b3e7..7c42671 100755 --- a/cernbox-share +++ b/cernbox-share @@ -225,7 +225,7 @@ def cmd_remove(args): def cmd_add(args): import cernbox_utils.sharing - return cernbox_utils.sharing.add_share(args.owner,args.path,args.sharee,args.acl) + return cernbox_utils.sharing.add_share(args.owner,args.path,args.sharee,args.acl,eos,db,config) def cmd_list_shares(args,role): From 9c1138adf334930e7fbb1c909dbdc0f5c064ffe3 Mon Sep 17 00:00:00 2001 From: Enrico Bocchi Date: Tue, 5 Jun 2018 15:26:46 +0200 Subject: [PATCH 05/10] Import check_share_target from utils --- cernbox-share | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cernbox-share b/cernbox-share index 7c42671..061539d 100755 --- a/cernbox-share +++ b/cernbox-share @@ -187,13 +187,13 @@ class CmdError(Exception): def cmd_remove(args): - from cernbox_utils.sharing import split_sharee + from cernbox_utils.sharing import split_sharee, check_share_target share_with_entity,share_with_who = split_sharee(args.sharee) path = args.path #split_path(args.path) - f = check_share_target(path, args.owner) + f = check_share_target(path, args.owner, eos, config) shares=db.get_share(sharee=share_with_who,owner=args.owner,fid=f.ino) From 0e01591fbf65cef03f2b329b59a9fa1994f38488 Mon Sep 17 00:00:00 2001 From: Enrico Bocchi Date: Tue, 5 Jun 2018 15:27:43 +0200 Subject: [PATCH 06/10] Support for federated sharing in db --- python/cernbox_utils/db.py | 11 +++++++---- python/cernbox_utils/sharing.py | 6 +++--- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/python/cernbox_utils/db.py b/python/cernbox_utils/db.py index e4236ea..a9dd4cc 100644 --- a/python/cernbox_utils/db.py +++ b/python/cernbox_utils/db.py @@ -93,7 +93,7 @@ def get_share(self,fid=None,sharee=None,owner=None,share_type=None,share_time_gr # TODO: https://its.cern.ch/jira/browse/CERNBOX-236 - def insert_folder_share(self,owner,sharee,fid,file_target,permissions,stime=None,initiator=None): + def insert_folder_share(self,owner,sharee_entity,sharee,fid,file_target,permissions,stime=None,initiator=None): cur = self.db.cursor() logger = cernbox_utils.script.getLogger('db') @@ -104,10 +104,13 @@ def insert_folder_share(self,owner,sharee,fid,file_target,permissions,stime=None assert(all(c.isalnum() for c in initiator)) assert(all(c.isalnum() or c=='-' for c in sharee)) # egroups may have dash in the name - if '-' in sharee: - share_type = 1 # group + if sharee_entity == 'fed': + share_type = 6 # federated else: - share_type = 0 # user + if '-' in sharee: + share_type = 1 # group + else: + share_type = 0 # user assert(fid>0) assert(permissions>=0) diff --git a/python/cernbox_utils/sharing.py b/python/cernbox_utils/sharing.py index 8cba74b..88bc641 100644 --- a/python/cernbox_utils/sharing.py +++ b/python/cernbox_utils/sharing.py @@ -69,7 +69,7 @@ def is_egroup(name): def split_sharee(sharee): entity,who = sharee.split(":") # this may also raise ValueError - if not entity in ['u','egroup']: + if not entity in ['u','egroup', 'fed']: raise ValueError() return entity,who @@ -395,7 +395,7 @@ def add_share(owner,path,sharee,acl,eos,db,config,storage_acl_update=True): # ... continue from common code above ACL = {'r':'read','rw':'read-write'} - ENTITY = {'u':'user','egroup':'egroup'} + ENTITY = {'u':'user','egroup':'egroup', 'fed':'federated'} logger.info("Add %s share for %s %s to tree %s",ACL[acl],ENTITY[share_with_entity],share_with_who,path) @@ -412,7 +412,7 @@ def add_share(owner,path,sharee,acl,eos,db,config,storage_acl_update=True): logger.error(msg) raise ValueError(msg) # TODO: BAD REQUEST else: - db.insert_folder_share(owner,share_with_who,int(f.ino),file_target,cernbox_utils.sharing.crud2db(acl)) + db.insert_folder_share(owner,share_with_entity,share_with_who,int(f.ino),file_target,cernbox_utils.sharing.crud2db(acl)) try: # modify storage ACL From 923aa2730375e32d2f3683e17387e3fda66a9e64 Mon Sep 17 00:00:00 2001 From: Enrico Bocchi Date: Mon, 25 Jun 2018 11:20:53 +0200 Subject: [PATCH 07/10] Capability to manage trusted servers on DB --- cernbox-share | 75 +++++++++++++++++++--- python/cernbox_utils/db.py | 108 ++++++++++++++++++++++++++++++-- python/cernbox_utils/sharing.py | 59 ++++++++++++++++- 3 files changed, 228 insertions(+), 14 deletions(-) diff --git a/cernbox-share b/cernbox-share index 061539d..e7a7198 100755 --- a/cernbox-share +++ b/cernbox-share @@ -79,6 +79,25 @@ def main(): # TODO: OC_INTEGRATION add update command to update permissions by share id + + # FEDERATED SHARING COMMANDS + + subcmd = subparser.add_parser('add-trusted-server', help="add a trusted server to the system") + subcmd.add_argument("url", help="server url") + # TODO: Check the following args in the upstream ownCloud + #subcmd.add_argument("token", help="server token") + #subcmd.add_argument("shared-secret", help="server shared secret") + #subcmd.add_argument("status", help="trusted server configuration status") + #subcmd.add_argument("sync-token", help="syncronization token") + + subcmd = subparser.add_parser('remove-trusted-server', help="remove a trusted server to the system") + subcmd.add_argument("url", help="server url") + #subcmd.add_argument("token", help="server token") + #subcmd.add_argument("shared-secred", help="server shared secred") + + subcmd = subparser.add_parser('list-trusted-servers', help="list all trusted servers for federated shares") + + # ADMIN COMMANDS subcmd = subparser.add_parser('acl_update', help="update the sharing ACL for a path and all subdirectories") @@ -127,10 +146,6 @@ def main(): import cernbox_utils.sharing - if args.cmd == "acl_update": - import cernbox_utils.cmd_share_admin - cernbox_utils.cmd_share_admin.acl_update(args,config,eos,db) - if args.cmd == "remove": try: @@ -159,6 +174,33 @@ def main(): except CmdError: sys.exit(2) + # Federated Sharing: Trusted servers + if args.cmd == "add-trusted-server": + + try: + print_json(cmd_add_trusted_server(args)) + except CmdError: + sys.exit(2) + + if args.cmd == "remove-trusted-server": + + try: + print_json(cmd_remove_trusted_server(args)) + except CmdError: + sys.exit(2) + + if args.cmd == "list-trusted-servers": + + try: + print_json(cmd_list_trusted_servers(args)) + except CmdError: + sys.exit(2) + + + # Admin commands + if args.cmd == "acl_update": + import cernbox_utils.cmd_share_admin + cernbox_utils.cmd_share_admin.acl_update(args,config,eos,db) if args.cmd == "show-other-acls": import cernbox_utils.cmd_share_admin @@ -166,18 +208,15 @@ def main(): if args.cmd == "remove-orphan-xbits": import cernbox_utils.cmd_share_admin - cernbox_utils.cmd_share_admin.remove_orphan_xbits(args,config,eos,db) - + if args.cmd == "summary": import cernbox_utils.cmd_share_admin - cernbox_utils.cmd_share_admin.summary(args,config,eos,db) if args.cmd == "verify": import cernbox_utils.cmd_share_admin - cernbox_utils.cmd_share_admin.verify(args,config,eos,db) @@ -227,13 +266,31 @@ def cmd_add(args): import cernbox_utils.sharing return cernbox_utils.sharing.add_share(args.owner,args.path,args.sharee,args.acl,eos,db,config) -def cmd_list_shares(args,role): +def cmd_list_shares(args,role): import cernbox_utils.sharing groups={} retobj = cernbox_utils.sharing.list_shares(args.user,role,groups,None,None,args.flat_list,False,db,eos) return {'shares':retobj} +# Federated Sharing: Trusted servers +def cmd_add_trusted_server(args): + import cernbox_utils.sharing + return cernbox_utils.sharing.add_trusted_server(args.url,db) + + +def cmd_remove_trusted_server(args): + import cernbox_utils.sharing + return cernbox_utils.sharing.remove_trusted_server(args.url,db) + + +def cmd_list_trusted_servers(args): + import cernbox_utils.sharing + retobj = cernbox_utils.sharing.list_trusted_servers(db) + return {'trusted_servers_list':retobj} + + + if __name__ == "__main__": sys.exit(main()) diff --git a/python/cernbox_utils/db.py b/python/cernbox_utils/db.py index a9dd4cc..f255c3b 100644 --- a/python/cernbox_utils/db.py +++ b/python/cernbox_utils/db.py @@ -3,10 +3,16 @@ class ShareInfo(cernbox_utils.script.Data): _names = ['id','share_type','share_with','uid_owner','uid_initiator','parent','item_type','item_source','item_target','file_source','file_target','permissions','stime','accepted','expiration','token','mail_send'] + def _check_consistency(self): + pass + +class TrustedServersInfo(cernbox_utils.script.Data): + _names = ['id', 'url', 'url_hash', 'token', 'shared_secret', 'status', 'sync_token'] def _check_consistency(self): pass + import MySQLdb @@ -15,6 +21,12 @@ def _check_consistency(self): +# quote strings to insert into MySQL DB +def quote(x): + return '"'+x+'"' + + + class ShareDB: def __init__(self): @@ -34,7 +46,9 @@ def __init__(self): db = MySQLdb.connect(host=host,user=config['dbuser'],passwd=config['dbpassword'],db=config['dbname']) self.db = db - + + + def get_share(self,fid=None,sharee=None,owner=None,share_type=None,share_time_greater_than=None,item_type=None,share_id=None): """ Get share information matchin target file id AND sharee name AND owner name AND share type ("link" or "regular"). """ @@ -93,6 +107,15 @@ def get_share(self,fid=None,sharee=None,owner=None,share_type=None,share_time_gr # TODO: https://its.cern.ch/jira/browse/CERNBOX-236 + + + def insert_file_share(self): + pass + #TODO: to be implemented + return + + + def insert_folder_share(self,owner,sharee_entity,sharee,fid,file_target,permissions,stime=None,initiator=None): cur = self.db.cursor() logger = cernbox_utils.script.getLogger('db') @@ -117,9 +140,6 @@ def insert_folder_share(self,owner,sharee_entity,sharee,fid,file_target,permissi assert(stime is None or stime>0) assert(file_target!="") - def quote(x): - return '"'+x+'"' - item_source=fid item_target=quote("/%d"%fid) file_source=fid @@ -137,6 +157,7 @@ def quote(x): self.db.commit() + def update_share(self,id,file_target=None): cur = self.db.cursor() @@ -177,3 +198,82 @@ def delete_share(self,id): # Check referential integrity. # insert into oc_share(share_type, share_with, uid_owner, parent, item_type, item_source, item_target, file_source, file_target, permissions, stime) values (0,"rosma","cmsgemhw",NULL, "folder",28284090, "/28284090", 28284090, "/GE11_Shared_Documents (#28284090)",1,1489496970); + + +# Federated Sharing: Trusted servers + def add_trusted_server(self,url): + """ Add a server to the list of trusted ones. + """ + + cur = self.db.cursor() + + logger = cernbox_utils.script.getLogger('db') + + #TODO: CRITICAL + #TODO: Understand how to set these values: + url_hash = "hash_%s"%(url) + token = "none" + shared_secret = "none" + status = 2 + sync_token = "none" + + #sql = 'INSERT INTO oc_trusted_servers(url, url_hash, token, shared_secret, status, sync_token) values (%s,%s,%s,%s,%d,%s)' % (url, url_hash, token, shared_secret, status, sync_token) + sql = 'INSERT INTO oc_trusted_servers(url, url_hash, token, shared_secret, status, sync_token) values (%s,%s,%s,NULL,%d,NULL);' % (quote(url), quote(url_hash), quote(token), status) + + logger.debug(sql) + cur.execute(sql) + self.db.commit() + + + + def remove_trusted_server(self,id): + """ Remove a server from the list of trusted ones represented by id. + """ + + cur = self.db.cursor() + + logger = cernbox_utils.script.getLogger('db') + + sql = 'DELETE FROM oc_trusted_servers WHERE id=%d;'%int(id) + + logger.debug(sql) + cur.execute(sql) + self.db.commit() + + + + def get_trusted_server(self,url=None): + """ Get detailed information on one trusted server + or the entire list list of trusted servers for federated shares. + """ + + cur = self.db.cursor() + + WHERE = [] + + if url: + WHERE.append('url = "%s"'%url) + + if WHERE: + WHERE = "WHERE " + (' and '.join(WHERE)) + else: + WHERE = "" + + logger = cernbox_utils.script.getLogger('db') + + sql = "SELECT * FROM oc_trusted_servers "+WHERE + logger.debug(sql) + + cur.execute(sql) + + trusted_servers = [] + for row in cur.fetchall(): + s = TrustedServersInfo() + for i,name in enumerate(TrustedServersInfo._names): + setattr(s,name,row[i]) + trusted_servers.append(s) + logger.debug("ROW: %s",row) + + return trusted_servers + + diff --git a/python/cernbox_utils/sharing.py b/python/cernbox_utils/sharing.py index 88bc641..794d60f 100644 --- a/python/cernbox_utils/sharing.py +++ b/python/cernbox_utils/sharing.py @@ -287,6 +287,7 @@ def is_descendant(p1,p2): class ShareNode(cernbox_utils.script.Data): _names = ['inode','owner','shares'] + def collapse_into_nodes(shares): """ Collapse flat share list into a list of nodes. @@ -302,6 +303,7 @@ def collapse_into_nodes(shares): return nodes + def list_shares(user,role,groups,fid,share_type,flat_list,include_broken,db,eos): """ Return JSON-style dictionary listing all shares for a user in a role of "owner" or "sharee". Each shared directory has one entry (and multuple shared_with entries if applicable). @@ -355,8 +357,8 @@ def dtisoformat(x): if share_path or include_broken: retobj[s.id] = {'uid_owner':s.uid_owner,'uid_initiator':s.uid_initiator,'share_id':s.id, 'share_with':s.share_with,'type':s.share_type,'target_inode':s.item_source,'target_name':s.file_target, 'permissions':s.permissions, 'created' : datetime.datetime.fromtimestamp(s.stime).isoformat(), 'expires' : dtisoformat(s.expiration), 'token':s.token, 'target_path':share_path } - return retobj + else: retobj = [] nodes = collapse_into_nodes(shares) @@ -379,6 +381,7 @@ def dtisoformat(x): return retobj + def add_share(owner,path,sharee,acl,eos,db,config,storage_acl_update=True): logger = cernbox_utils.script.getLogger('sharing') @@ -427,3 +430,57 @@ def add_share(owner,path,sharee,acl,eos,db,config,storage_acl_update=True): #rollback the insert? raise + +# Federated Sharing: Trusted servers +def add_trusted_server(url,db): + """ Add a trusted server for federated sharing + """ + + logger = cernbox_utils.script.getLogger('trusted_servers') + + logger.info("Add %s as trusted server",url) + + trusted_server=db.get_trusted_server(url) + if trusted_server: + msg="Trusted server already exists, server url: %s"%url + logger.error(msg) + raise ValueError(msg) # TODO: BAD REQUEST + else: + db.add_trusted_server(url) + + + +def remove_trusted_server(url,db): + """ Remove a trusted server for federated sharing + """ + + logger = cernbox_utils.script.getLogger('trusted_servers') + + logger.info("Removing %s from trusted server list",url) + + trusted_server=db.get_trusted_server(url) + if trusted_server: + db.remove_trusted_server(trusted_server[0].id) + else: + msg="Server is not part of the trusted server list, server url: %s"%url + logger.error(msg) + raise ValueError(msg) # TODO: BAD REQUEST + + + +def list_trusted_servers(db): + """ Return JSON-style dictionary listing all trusted servers for federated sharing. + """ + logger = cernbox_utils.script.getLogger('trusted_servers') + + trusted_servers=db.get_trusted_server() + + retobj = [] + + for ts in trusted_servers: + retobj.append({'url':ts.url, 'url_hash':ts.url_hash, 'token':ts.token, 'shared_secret':ts.shared_secret, 'status':ts.status, 'sync_token':ts.sync_token}) + pass + + return retobj + + From 2df2f95d34a6215ec72bc50c89c1169022038595 Mon Sep 17 00:00:00 2001 From: Enrico Bocchi Date: Mon, 25 Jun 2018 17:41:40 +0200 Subject: [PATCH 08/10] Capabilityo to manage external shares on DB --- cernbox-share | 117 +++++++++++++++++++++++++++++- python/cernbox_utils/db.py | 121 ++++++++++++++++++++++++++++++++ python/cernbox_utils/sharing.py | 84 ++++++++++++++++++++-- 3 files changed, 315 insertions(+), 7 deletions(-) diff --git a/cernbox-share b/cernbox-share index e7a7198..c77f2df 100755 --- a/cernbox-share +++ b/cernbox-share @@ -82,6 +82,46 @@ def main(): # FEDERATED SHARING COMMANDS + subcmd = subparser.add_parser('add-external-share', help="add a share from a federated server") + subcmd.add_argument("remote", help="remote server url") + subcmd.add_argument("remote_id", help="server url") + subcmd.add_argument("share_token", help="server url") + subcmd.add_argument("password", help="server url") + subcmd.add_argument("name", help="name of the shared resource") + subcmd.add_argument("owner", help="remote owner of the resource") + subcmd.add_argument("user", help="local sharee") + #subcmd.add_argument("mountpoint", help="local mountpoint where to show the shared resource") + #subcmd.add_argument("mountpoint-hash", help="hash of the local mountpoint") + #subcmd.add_argument("accepted", help="local sharee has accepted the share") + + subcmd = subparser.add_parser('accept-external-share', help="accept a share from a federated server") + subcmd.add_argument("remote", help="remote server url") + #subcmd.add_argument("remote_id", help="server url") + #subcmd.add_argument("share_token", help="server url") + subcmd.add_argument("name", help="name of the shared resource") + subcmd.add_argument("owner", help="remote owner of the resource") + subcmd.add_argument("user", help="local sharee") + subcmd.add_argument("mountpoint", help="local mountpoint where to show the shared resource") + + subcmd = subparser.add_parser('remove-external-share', help="remove a share from a federated server") + subcmd.add_argument("remote", help="remote server url") + #subcmd.add_argument("remote_id", help="server url") + #subcmd.add_argument("share_token", help="server url") + subcmd.add_argument("name", help="name of the shared resource") + subcmd.add_argument("owner", help="remote owner of the resource") + subcmd.add_argument("user", help="local sharee") + + subcmd = subparser.add_parser('list-external-shared-by', help="list all federated shares created by a remote user from a trusted server") + subcmd.add_argument("remote", help="remote server url") + subcmd.add_argument("owner", help="remote owner of the resource") + + subcmd = subparser.add_parser('list-external-shared-from', help="list all federated shares from a trusted server") + subcmd.add_argument("remote", help="remote server url") + + subcmd = subparser.add_parser('list-external-shared-with', help="list all federated shares given to a local user") + subcmd.add_argument("user", help="local sharee") + + subcmd = subparser.add_parser('add-trusted-server', help="add a trusted server to the system") subcmd.add_argument("url", help="server url") # TODO: Check the following args in the upstream ownCloud @@ -174,23 +214,59 @@ def main(): except CmdError: sys.exit(2) + + # Federated Sharing: External shares + if args.cmd == "add-external-share": + try: + print_json(cmd_add_external_share(args)) + except CmdError: + sys.exit(2) + + if args.cmd == "accept-external-share": + try: + print_json(cmd_accept_external_share(args)) + except CmdError: + sys.exit(2) + + if args.cmd == "remove-external-share": + try: + print_json(cmd_remove_external_share(args)) + except CmdError: + sys.exit(2) + + if args.cmd == "list-external-shared-by": + try: + print_json(cmd_list_external_shared(args)) + except CmdError: + sys.exit(2) + + if args.cmd == "list-external-shared-from": + try: + print_json(cmd_list_external_shared(args)) + except CmdError: + sys.exit(2) + + if args.cmd == "list-external-shared-with": + try: + print_json(cmd_list_external_shared(args)) + except CmdError: + sys.exit(2) + + # Federated Sharing: Trusted servers if args.cmd == "add-trusted-server": - try: print_json(cmd_add_trusted_server(args)) except CmdError: sys.exit(2) if args.cmd == "remove-trusted-server": - try: print_json(cmd_remove_trusted_server(args)) except CmdError: sys.exit(2) if args.cmd == "list-trusted-servers": - try: print_json(cmd_list_trusted_servers(args)) except CmdError: @@ -274,6 +350,41 @@ def cmd_list_shares(args,role): return {'shares':retobj} + +# Federated Sharing: External shares +def cmd_add_external_share(args): + import cernbox_utils.sharing + return cernbox_utils.sharing.add_external_share(args.remote,args.remote_id,args.share_token,args.password,args.name,args.owner,args.user,db) + + +def cmd_accept_external_share(args): + import cernbox_utils.sharing + #return cernbox_utils.sharing.accept_external_share(args.remote,args.remote_id,args.share_token,args.name,args.owner,args.user,args.mountpoint,db) + return cernbox_utils.sharing.accept_external_share(args.remote,args.name,args.owner,args.user,args.mountpoint,db) + + +def cmd_remove_external_share(args): + import cernbox_utils.sharing + return cernbox_utils.sharing.remove_external_share(args.remote,args.name,args.owner,args.user,db) + + +def cmd_list_external_shared(args): + if all('remote' and 'owner') in dir(args): + retobj = cernbox_utils.sharing.list_external_shares(db,remote=args.remote,owner=args.owner) + return {'external_shared_by':retobj} + elif 'remote' in dir(args): + retobj = cernbox_utils.sharing.list_external_shares(db,remote=args.remote) + return {'external_shared_from':retobj} + elif 'user' in dir(args): + retobj = cernbox_utils.sharing.list_external_shares(db,user=args.user) + return {'external_shares_with':retobj} + else: + msg="Unsopported type of listing external shares" + logger.error(msg) + raise ValueError(msg) # TODO: BAD REQUEST + + + # Federated Sharing: Trusted servers def cmd_add_trusted_server(args): import cernbox_utils.sharing diff --git a/python/cernbox_utils/db.py b/python/cernbox_utils/db.py index f255c3b..575b978 100644 --- a/python/cernbox_utils/db.py +++ b/python/cernbox_utils/db.py @@ -6,6 +6,12 @@ class ShareInfo(cernbox_utils.script.Data): def _check_consistency(self): pass +class ExternalSharesInfo(cernbox_utils.script.Data): + _names = ['id', 'remote', 'remote_id', 'share_token', 'password', 'name', 'owner', 'user', 'mountpoint', 'mountpoint_hash', 'accepted'] + + def _check_consistency(self): + pass + class TrustedServersInfo(cernbox_utils.script.Data): _names = ['id', 'url', 'url_hash', 'token', 'shared_secret', 'status', 'sync_token'] @@ -200,6 +206,120 @@ def delete_share(self,id): # insert into oc_share(share_type, share_with, uid_owner, parent, item_type, item_source, item_target, file_source, file_target, permissions, stime) values (0,"rosma","cmsgemhw",NULL, "folder",28284090, "/28284090", 28284090, "/GE11_Shared_Documents (#28284090)",1,1489496970); +# Federated Sharing: External shares + def insert_external_share(self,remote,remote_id,share_token,password,name,owner,user): + """ Add an external share. + """ + + cur = self.db.cursor() + + logger = cernbox_utils.script.getLogger('db') + + #TODO: CRITICAL + #TODO: Understand how to set these values: + mountpoint = "{{TemporaryMPName#/%s}}"%name + mountpoint_hash = "hash_%s"%mountpoint + + remote_id = int(remote_id) + + sql = 'INSERT INTO oc_share_external(remote, remote_id, share_token, password, name, owner, user, mountpoint, mountpoint_hash, accepted) values (%s,%d,%s,%s,%s,%s,%s,%s,%s,%d);' % (quote(remote), remote_id, quote(share_token), quote(password), quote(name), quote(owner), quote(user), quote(mountpoint), quote(mountpoint_hash), 0) + + logger.debug(sql) + cur.execute(sql) + self.db.commit() + + + + def accept_external_share(self,id,mountpoint=None): + """ Accept an external share and set its final mount point. + """ + + cur = self.db.cursor() + logger = cernbox_utils.script.getLogger('db') + + set_cmd = [] + + set_cmd.append("accepted = 1") + + if mountpoint: + #TODO: CRITICAL + #TODO: Understand how to set these values: + mountpoint_hash = "hash_%s"%mountpoint + set_cmd.append("mountpoint = '%s'"%mountpoint) + set_cmd.append("mountpoint_hash = '%s'"%mountpoint_hash) + + set_cmd = ",".join(set_cmd) + + sql="UPDATE oc_share_external SET %s WHERE id=%d;"%(set_cmd,id) + + logger.debug(sql) + cur.execute(sql) + self.db.commit() + + + + def delete_external_share(self,id): + """ Delete an external share. + """ + + cur = self.db.cursor() + logger = cernbox_utils.script.getLogger('db') + + sql = 'DELETE FROM oc_share_external WHERE id=%d;'%int(id) + + logger.debug(sql) + cur.execute(sql) + self.db.commit() + + + + def get_external_share(self,remote=None,name=None,owner=None,user=None,accepted=None): + """ Get detailed information on one share + or the entire list of shares for one local user or a remote server//owner + """ + cur = self.db.cursor() + + WHERE = [] + + if remote: + WHERE.append('remote = "%s"'%remote) + + if name: + WHERE.append('name = "%s"'%name) + + if owner: + WHERE.append('owner = "%s"'%owner) + + if user: + WHERE.append('user = "%s"'%user) + + if accepted: + WHERE.append('accepted = "%d"'%accepted) + + if WHERE: + WHERE = "WHERE " + (' and '.join(WHERE)) + else: + WHERE = "" + + logger = cernbox_utils.script.getLogger('db') + + sql = "SELECT * FROM oc_share_external "+WHERE + logger.debug(sql) + + cur.execute(sql) + + external_shares = [] + for row in cur.fetchall(): + s = ExternalSharesInfo() + for i,name in enumerate(ExternalSharesInfo._names): + setattr(s,name,row[i]) + external_shares.append(s) + logger.debug("ROW: %s",row) + + return external_shares + + + # Federated Sharing: Trusted servers def add_trusted_server(self,url): """ Add a server to the list of trusted ones. @@ -216,6 +336,7 @@ def add_trusted_server(self,url): shared_secret = "none" status = 2 sync_token = "none" + status = int(status) #sql = 'INSERT INTO oc_trusted_servers(url, url_hash, token, shared_secret, status, sync_token) values (%s,%s,%s,%s,%d,%s)' % (url, url_hash, token, shared_secret, status, sync_token) sql = 'INSERT INTO oc_trusted_servers(url, url_hash, token, shared_secret, status, sync_token) values (%s,%s,%s,NULL,%d,NULL);' % (quote(url), quote(url_hash), quote(token), status) diff --git a/python/cernbox_utils/sharing.py b/python/cernbox_utils/sharing.py index 794d60f..36b62be 100644 --- a/python/cernbox_utils/sharing.py +++ b/python/cernbox_utils/sharing.py @@ -352,7 +352,7 @@ def dtisoformat(x): # eos entry does not exist logger.warning("DANGLING_SHARE id=%d owner=%s sharee=%s target='%s' inode=%s",s.id,s.uid_owner,s.share_with,s.file_target,s.item_source) share_path=None - + if share_path or include_broken: retobj[s.id] = {'uid_owner':s.uid_owner,'uid_initiator':s.uid_initiator,'share_id':s.id, 'share_with':s.share_with,'type':s.share_type,'target_inode':s.item_source,'target_name':s.file_target, 'permissions':s.permissions, 'created' : datetime.datetime.fromtimestamp(s.stime).isoformat(), 'expires' : dtisoformat(s.expiration), 'token':s.token, 'target_path':share_path } @@ -371,7 +371,7 @@ def dtisoformat(x): # eos entry does not exist logger.warning("DANGLING_SHARE inode=%s",target_id) target_path,target_size=None,0 - + if target_path or include_broken: retobj.append({'path':target_path, 'inode':target_id, 'size':target_size, 'shared_by':nodes[target_id].owner, 'shared_with' : []}) @@ -431,6 +431,84 @@ def add_share(owner,path,sharee,acl,eos,db,config,storage_acl_update=True): raise +# Federated Sharing: External shares +def add_external_share(remote,remote_id,share_token,password,name,owner,user,db): + """ Add an external share + (to be confirmed by 'accept_external_share' once the user accepted it) + """ + + logger = cernbox_utils.script.getLogger('external_shares') + + logger.info("Add external share %s owned by %s from %s for user %s",name,owner,remote,user) + + ext_share=db.get_external_share(remote,name,owner,user) # TODO: Should we leverage on share_token instead? + + if ext_share: + msg="Share already exists, resource %s owned by %s from %s for user %s"%(ext_share[0].name,ext_share[0].owner,ext_share[0].remote,ext_share[0].user) + logger.error(msg) + raise ValueError(msg) # TODO: Bad request + else: + db.insert_external_share(remote,remote_id,share_token,password,name,owner,user) + + + +#def accept_exterinal_share(remote,remote_id,share_token,name,owner,user,mountpoint,db): +def accept_external_share(remote,name,owner,user,mountpoint,db): + """ Accept a (previously added) external share. + """ + + logger = cernbox_utils.script.getLogger('external_shares') + + logger.info("Accept external share %s owned by %s from %s for user %s",name,owner,remote,user) + + ext_share=db.get_external_share(remote,name,owner,user) + + if ext_share: + db.accept_external_share(ext_share[0].id,mountpoint) + else: + msg="Share does not exist, resource %s owned by %s from %s for user %s"%(ext_share[0].name,ext_share[0].owner,ext_share[0].remote,ext_share[0].user) + logger.error(msg) + raise ValueError(msg) # TODO: BAD REQUEST + + +def remove_external_share(remote,name,owner,user,db): + """ Delete an external share. + """ + + logger = cernbox_utils.script.getLogger('external_shares') + + logger.info("Remove external share %s owned by %s from %s for user %s",name,owner,remote,user) + + ext_share=db.get_external_share(remote,name,owner,user) + + if ext_share: + db.delete_external_share(ext_share[0].id) + else: + msg="Share does not exist, resource %s owned by %s from %s for user %s"%(ext_share[0].name,ext_share[0].owner,ext_share[0].remote,ext_share[0].user) + logger.error(msg) + raise ValueError(msg) # TODO: BAD REQUEST + + +def list_external_shares(db,remote=None,owner=None,user=None,accepted=None): + """ Return JSON-style dictionary listing all shares for + 1. a local user in a role of "sharee" ("list-external-shared-with") + 2a. a remote server in a role of host for the shared resources ("list-external-shared-by") + 2b. a remote user in a role of "owner" ("list-external-shared-by") + """ + + logger = cernbox_utils.script.getLogger('external_shares') + + ext_shares = db.get_external_share(remote=remote,owner=owner,user=user,accepted=accepted) + + retobj = [] + + for es in ext_shares: + retobj.append({'remote':es.remote, 'remote-id':es.remote_id, 'share_token':es.share_token, 'password':es.password, 'name':es.name, 'owner':es.owner, 'user':es.user, 'mountpoint':es.mountpoint, 'mountpoint-hash':es.mountpoint_hash, 'accepted':es.accepted}) + + return retobj + + + # Federated Sharing: Trusted servers def add_trusted_server(url,db): """ Add a trusted server for federated sharing @@ -479,8 +557,6 @@ def list_trusted_servers(db): for ts in trusted_servers: retobj.append({'url':ts.url, 'url_hash':ts.url_hash, 'token':ts.token, 'shared_secret':ts.shared_secret, 'status':ts.status, 'sync_token':ts.sync_token}) - pass return retobj - From 135ac8198ef15c96c46e0ce6fadd2a76c3abfbf0 Mon Sep 17 00:00:00 2001 From: Enrico Bocchi Date: Mon, 25 Jun 2018 18:17:14 +0200 Subject: [PATCH 09/10] md5 digest for mountpoint hash --- python/cernbox_utils/db.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/python/cernbox_utils/db.py b/python/cernbox_utils/db.py index 575b978..a51ab54 100644 --- a/python/cernbox_utils/db.py +++ b/python/cernbox_utils/db.py @@ -33,6 +33,13 @@ def quote(x): +# compute md5 digest +def md5_digest(x): + import hashlib + return hashlib.md5(x).hexdigest() + + + class ShareDB: def __init__(self): @@ -215,10 +222,8 @@ def insert_external_share(self,remote,remote_id,share_token,password,name,owner, logger = cernbox_utils.script.getLogger('db') - #TODO: CRITICAL - #TODO: Understand how to set these values: - mountpoint = "{{TemporaryMPName#/%s}}"%name - mountpoint_hash = "hash_%s"%mountpoint + mountpoint = "{{TemporaryMountPointName#%s}}"%name + mountpoint_hash = md5_digest(mountpoint) remote_id = int(remote_id) @@ -242,9 +247,7 @@ def accept_external_share(self,id,mountpoint=None): set_cmd.append("accepted = 1") if mountpoint: - #TODO: CRITICAL - #TODO: Understand how to set these values: - mountpoint_hash = "hash_%s"%mountpoint + mountpoint_hash = md5_digest(mountpoint) set_cmd.append("mountpoint = '%s'"%mountpoint) set_cmd.append("mountpoint_hash = '%s'"%mountpoint_hash) From 176a39ccff31dd1d225ae9ece048108103e819b3 Mon Sep 17 00:00:00 2001 From: Enrico Bocchi Date: Thu, 28 Jun 2018 12:31:18 +0200 Subject: [PATCH 10/10] Functions to set shared_secret and sync_token for trusted servers --- cernbox-share | 55 +++++++++++++++++++++--- python/cernbox_utils/db.py | 76 +++++++++++++++++++++++++++------ python/cernbox_utils/sharing.py | 48 +++++++++++++++++++-- 3 files changed, 155 insertions(+), 24 deletions(-) diff --git a/cernbox-share b/cernbox-share index c77f2df..a13eb9f 100755 --- a/cernbox-share +++ b/cernbox-share @@ -42,6 +42,14 @@ def print_json(obj): def print_json_error(msg): print_json({"error" : str(msg)}) +def secure_server_url(url): + if url.startswith('http://'): + return url.replace('http://', 'https://') + if url.startswith('https://'): + return url + return 'https://'+url + + import os, os.path, sys import subprocess @@ -122,13 +130,20 @@ def main(): subcmd.add_argument("user", help="local sharee") - subcmd = subparser.add_parser('add-trusted-server', help="add a trusted server to the system") + subcmd = subparser.add_parser('add-trusted-server', help="add a trusted server") subcmd.add_argument("url", help="server url") - # TODO: Check the following args in the upstream ownCloud - #subcmd.add_argument("token", help="server token") - #subcmd.add_argument("shared-secret", help="server shared secret") + subcmd.add_argument("token", help="server token") + #subcmd.add_argument("shared_secret", help="server shared secret") #subcmd.add_argument("status", help="trusted server configuration status") - #subcmd.add_argument("sync-token", help="syncronization token") + #subcmd.add_argument("sync_token", help="syncronization token") + + subcmd = subparser.add_parser('set-trusted-server-shared-secret', help="set the shared secret for trusted server") + subcmd.add_argument("url", help="server url") + subcmd.add_argument("shared_secret", help="server shared secret") + + subcmd = subparser.add_parser('set-trusted-server-sync-token', help="set the sync token for trusted server") + subcmd.add_argument("url", help="server url") + subcmd.add_argument("sync_token", help="syncronization token") subcmd = subparser.add_parser('remove-trusted-server', help="remove a trusted server to the system") subcmd.add_argument("url", help="server url") @@ -260,6 +275,18 @@ def main(): except CmdError: sys.exit(2) + if args.cmd == "set-trusted-server-shared-secret": + try: + print_json(cmd_set_trusted_server_shared_secret(args)) + except CmdError: + sys.exit(2) + + if args.cmd == "set-trusted-server-sync-token": + try: + print_json(cmd_set_trusted_server_sync_token(args)) + except CmdError: + sys.exit(2) + if args.cmd == "remove-trusted-server": try: print_json(cmd_remove_trusted_server(args)) @@ -388,12 +415,26 @@ def cmd_list_external_shared(args): # Federated Sharing: Trusted servers def cmd_add_trusted_server(args): import cernbox_utils.sharing - return cernbox_utils.sharing.add_trusted_server(args.url,db) + server_url = secure_server_url(args.url) + return cernbox_utils.sharing.add_trusted_server(server_url,args.token,db) + + +def cmd_set_trusted_server_shared_secret(args): + import cernbox_utils.sharing + server_url = secure_server_url(args.url) + return cernbox_utils.sharing.set_trusted_server_shared_secret(server_url,args.shared_secret,db) + + +def cmd_set_trusted_server_sync_token(args): + import cernbox_utils.sharing + server_url = secure_server_url(args.url) + return cernbox_utils.sharing.set_trusted_server_sync_token(server_url,args.sync_token,db) def cmd_remove_trusted_server(args): import cernbox_utils.sharing - return cernbox_utils.sharing.remove_trusted_server(args.url,db) + server_url = secure_server_url(args.url) + return cernbox_utils.sharing.remove_trusted_server(server_url,db) def cmd_list_trusted_servers(args): diff --git a/python/cernbox_utils/db.py b/python/cernbox_utils/db.py index a51ab54..49437f5 100644 --- a/python/cernbox_utils/db.py +++ b/python/cernbox_utils/db.py @@ -26,19 +26,23 @@ def _check_consistency(self): #oc_share = dict([(name,i) for i,name in enumerate(['id','share_type','share_with','uid_owner','parent','item_source','item_target','file_source','file_target','permissions','stime','accepted','expiration','token','mail_send'])]) - # quote strings to insert into MySQL DB def quote(x): return '"'+x+'"' - # compute md5 digest def md5_digest(x): import hashlib return hashlib.md5(x).hexdigest() +# compute sha1 digest +def sha1_digest(x): + import hashlib + return hashlib.sha1(x).hexdigest() + + class ShareDB: @@ -324,7 +328,7 @@ def get_external_share(self,remote=None,name=None,owner=None,user=None,accepted= # Federated Sharing: Trusted servers - def add_trusted_server(self,url): + def add_trusted_server(self,url,token): """ Add a server to the list of trusted ones. """ @@ -332,17 +336,61 @@ def add_trusted_server(self,url): logger = cernbox_utils.script.getLogger('db') - #TODO: CRITICAL - #TODO: Understand how to set these values: - url_hash = "hash_%s"%(url) - token = "none" - shared_secret = "none" - status = 2 - sync_token = "none" - status = int(status) - - #sql = 'INSERT INTO oc_trusted_servers(url, url_hash, token, shared_secret, status, sync_token) values (%s,%s,%s,%s,%d,%s)' % (url, url_hash, token, shared_secret, status, sync_token) - sql = 'INSERT INTO oc_trusted_servers(url, url_hash, token, shared_secret, status, sync_token) values (%s,%s,%s,NULL,%d,NULL);' % (quote(url), quote(url_hash), quote(token), status) + if url[0:7] == 'http://': + stripped_url = url[7:] + elif url[0:8] == 'https://': + stripped_url = url[8:] + url_hash = sha1_digest(stripped_url) + + sql = 'INSERT INTO oc_trusted_servers(url, url_hash, token, shared_secret, status, sync_token) values (%s,%s,%s,NULL,2,NULL);' % (quote(url), quote(url_hash), quote(token)) + + logger.debug(sql) + cur.execute(sql) + self.db.commit() + + + + def set_trusted_server_shared_secret(self,id,shared_secret): + """ Set shared secret for a trusted server + """ + + cur = self.db.cursor() + + logger = cernbox_utils.script.getLogger('db') + + set_cmd = [] + + set_cmd.append("shared_secret = '%s'"%shared_secret) + set_cmd.append("token = ''") # When setting the shared secret, the token should be removed + + set_cmd = ",".join(set_cmd) + + sql="UPDATE oc_trusted_servers SET %s WHERE id=%d;"%(set_cmd,id) + + logger.debug(sql) + cur.execute(sql) + self.db.commit() + + + + def set_trusted_server_sync_token(self,id,sync_token,new_status=None): + """ Set sync token for a trusted server + """ + + cur = self.db.cursor() + + logger = cernbox_utils.script.getLogger('db') + + set_cmd = [] + + set_cmd.append("sync_token = '%s'"%sync_token) + + if new_status: + set_cmd.append("status = '%d'"%new_status) + + set_cmd = ",".join(set_cmd) + + sql="UPDATE oc_trusted_servers SET %s WHERE id=%d;"%(set_cmd,id) logger.debug(sql) cur.execute(sql) diff --git a/python/cernbox_utils/sharing.py b/python/cernbox_utils/sharing.py index 36b62be..8970526 100644 --- a/python/cernbox_utils/sharing.py +++ b/python/cernbox_utils/sharing.py @@ -510,13 +510,13 @@ def list_external_shares(db,remote=None,owner=None,user=None,accepted=None): # Federated Sharing: Trusted servers -def add_trusted_server(url,db): +def add_trusted_server(url,token,db): """ Add a trusted server for federated sharing """ logger = cernbox_utils.script.getLogger('trusted_servers') - logger.info("Add %s as trusted server",url) + logger.info("Add %s as trusted server with token %s",url,token) trusted_server=db.get_trusted_server(url) if trusted_server: @@ -524,7 +524,49 @@ def add_trusted_server(url,db): logger.error(msg) raise ValueError(msg) # TODO: BAD REQUEST else: - db.add_trusted_server(url) + db.add_trusted_server(url,token) + + + +def set_trusted_server_shared_secret(url,shared_secret,db): + """ Set shared secret for a trusted server + """ + + logger = cernbox_utils.script.getLogger('trusted_servers') + + logger.info("Add shared secret %s for trusted server %s",shared_secret,url) + + trusted_server=db.get_trusted_server(url) + if trusted_server: + db.set_trusted_server_shared_secret(trusted_server[0].id,shared_secret) + else: + msg="Server is not part of the trusted server list, server url: %s"%url + logger.error(msg) + raise ValueError(msg) # TODO: BAD REQUEST + + + +def set_trusted_server_sync_token(url,sync_token,db): + """ Set sync token for a trusted server + """ + + logger = cernbox_utils.script.getLogger('trusted_servers') + + logger.info("Add sync token %s for trusted server %s",sync_token,url) + + trusted_server=db.get_trusted_server(url) + if trusted_server: + if trusted_server[0].shared_secret: + # When shared_secret and sync_token are set, status is 1 + new_status = 1 + logger.info("Shared secret is set for server %s. Updating status to %d",url,new_status) + db.set_trusted_server_sync_token(trusted_server[0].id,sync_token,new_status) + else: + db.set_trusted_server_sync_token(trusted_server[0].id,sync_token) + else: + msg="Server is not part of the trusted server list, server url: %s"%url + logger.error(msg) + raise ValueError(msg) # TODO: BAD REQUEST