Streamlining auth and fixing compatiblity

README.md

@@ -1,13 +1,13 @@
 # Migration utility for moving from Gitlab to Gogs / Gitea
 
-This tools provides an automated way to copy all repositories in a namespaces from Gitlab to Gogs / Gitea. 
-All tags and branches are copied. 
-Organizations in Gogs are supported as well. 
+This tools provides an automated way to copy all repositories in a namespaces from Gitlab to Gogs / Gitea.
+All tags and branches are copied.
+Organizations in Gogs are supported as well.
 
 ## Usage
 
-Type `python migrate_gitlab_to_gogs.py --help` for usage information. 
-It will print 
+Type `python migrate_gitlab_to_gogs.py --help` for usage information.
+It will print
 ```
 usage: migrate_gitlab_to_gogs.py [-h] --source_namespace SOURCE_NAMESPACE
                                  [--add_to_private]
@@ -45,7 +45,8 @@ optional arguments:
   --skip_existing       Skip repositories that already exist on remote without
                         asking the user
   --use_ssh             Use ssh to pull/push files to repos
+  --use_push_ssh        Use ssh only to push file to the repos
 ```
 
 ## Requirements
-This tools was written for Python 3 using the requests, json, subprocess, and argparse modules. 
+This tools was written for Python 3 using the requests, json, subprocess, and argparse modules.

migrate_gitlab_to_gogs.py

@@ -8,26 +8,29 @@
 import argparse
 
 parser = argparse.ArgumentParser()
-parser.add_argument('--source_namespace', 
+parser.add_argument('--source_namespace',
                     help='The namespace in gitlab as it appears in URLs. For example, given the repository address http://mygitlab.com/harry/my-awesome-repo.git, it shows that this repository lies within my personal namespace "harry". Hence I would pass harry as parameter.',
                     required=True)
 parser.add_argument('--add_to_private',default=None, action='store_true',help='If you want to add the repositories under your own name, ie. not in any organisation, use this flag.')
 parser.add_argument('--add_to_organization',default=None, metavar='organization_name', help='If you want to add all the repositories to an exisiting organisation, please pass the name to this parameter. Organizations correspond to groups in Gitlab. The name can be taken from the URL, for example, if your organization is http://mygogs-repo.com/org/my-awesome-organisation/dashboard then pass my-awesome-organisation here')
-parser.add_argument('--source_repo', 
+parser.add_argument('--source_repo',
                     help='URL to your gitlab repo in the format http://mygitlab.com/',
                     required=True)
-parser.add_argument('--target_repo', 
+parser.add_argument('--target_repo',
                     help='URL to your gogs / gitea repo in the format http://mygogs.com/',
                     required=True)
-parser.add_argument('--no_confirm', 
+parser.add_argument('--no_confirm',
                     help='Skip user confirmation of each single step',
                     action='store_true')
-parser.add_argument('--skip_existing', 
+parser.add_argument('--skip_existing',
                     help='Skip repositories that already exist on remote without asking the user',
                     action='store_true')
-parser.add_argument('--use_ssh', 
+parser.add_argument('--use_ssh',
                     help='Use ssh to pull/push files to repos',
                     action='store_true')
+parser.add_argument('--use_push_ssh',
+                    help='Use ssh to only to push files to repos',
+                    action='store_true')
 
 args = parser.parse_args()
 
@@ -62,9 +65,9 @@
     gitlab_token=os.environ['gitlab_token']
 else:
     gitlab_token = input(("\n\nToken to access your GITLAB account. This is NOT your password! Go to \n"
-                        "{}/profile/account \n"
-                        "and copy the value in section 'Private token'. It should \n"
-                        "look like du8dfsJlfEWFJAFhs\n"
+                        "{}/-/user_settings/personal_access_tokens \n"
+                        "and create a token with the \"read_repository\" and \"read_api\" scope \n"
+                        "look like glpat-FyfaVh2WwiRiZKxNOKnhGW86MQp1OjUH.01.0w0b3glkk\n"
                         "\ngitlab_token=").format(args.source_repo))
 assert len(gitlab_token)>0, 'The gitlab token cannot be empty!'
 
@@ -83,18 +86,21 @@
 
 print('Getting existing projects from namespace %s...'%args.source_namespace)
 s = requests.Session()
-page_id = 1
-finished = False
 project_list = []
-while not finished:
-    print('Getting page %s'%page_id)
-    res = s.get(gitlab_url + '/projects?private_token=%s&page=%s'%(gitlab_token,page_id))
+headers = {
+    "PRIVATE-TOKEN": gitlab_token
+}
+
+loaded = False
+page = 0
+while not loaded:
+    res = s.get(gitlab_url + f'/projects?page={page}', headers=headers)
     assert res.status_code == 200, 'Error when retrieving the projects. The returned html is %s'%res.text
     project_list += json.loads(res.text)
-    if len(json.loads(res.text)) < 1:
-        finished = True
-    else:
-        page_id += 1
+    if len(json.loads(res.text)) == 0:
+        loaded = True
+    page += 1
+
 
 filtered_projects = list(filter(lambda x: x['path_with_namespace'].split('/')[0]==args.source_namespace, project_list))
 
@@ -114,23 +120,25 @@
     else:
         src_url = filtered_projects[i]['http_url_to_repo']
     src_description = filtered_projects[i]['description']
+    is_private = filtered_projects[i]['pages_access_level'] != "enabled"
     dst_name = src_name.replace(' ','-')
 
     print('\n\nMigrating project %s to project %s now.'%(src_url,dst_name))
 
     if not args.no_confirm:
         if 'yes' != input('Do you want to continue? (please answer yes or no) '):
             print('\nYou decided to cancel...')
+            continue
 
-    # Create repo 
+    # Create repo
     if args.add_to_private:
         print('Posting to:' + gogs_url + '/user/repos')
-        create_repo = s.post(gogs_url+'/user/repos', data=dict(token=gogs_token, name=dst_name, private=True))
+        create_repo = s.post(gogs_url+'/user/repos', data=dict(token=gogs_token, name=dst_name, private=is_private))
 
     elif args.add_to_organization:
         print('Posting to:' + gogs_url + '/org/%s/repos')
-        create_repo = s.post(gogs_url+'/org/%s/repos'%args.add_to_organization, 
-                            data=dict(token=gogs_token, name=dst_name, private=True, description=src_description))
+        create_repo = s.post(gogs_url+'/org/%s/repos'%args.add_to_organization,
+                            data=dict(token=gogs_token, name=dst_name, private=is_private, description=src_description))
     if create_repo.status_code != 201:
         print('Could not create repo %s because of %s'%(src_name,json.loads(create_repo.text)['message']))
         if args.skip_existing:
@@ -140,27 +148,33 @@
                 print('\nYou decided to cancel...')
                 exit(1)
         continue
-    
+
     dst_info = json.loads(create_repo.text)
 
-    if args.use_ssh:
+    if args.use_ssh or args.use_push_ssh:
         dst_url = dst_info['ssh_url']
     else:
         dst_url = dst_info['html_url']
+
+    repo_src_authed = src_url
+    if not args.use_ssh:
+        #provide the token to skip login
+        repo_src_authed = src_url.replace("https://", "https://oauth2:" + gitlab_token + "@")
+
     # Git pull and push
-    subprocess.check_call(['git','clone','--bare',src_url])
+    subprocess.check_call(['git','clone','--bare', repo_src_authed])
     os.chdir(src_url.split('/')[-1])
     branches=subprocess.check_output(['git','branch','-a'])
     if len(branches) == 0:
         print('\n\nThis repository is empty - skipping push')
     else:
-        subprocess.check_call(['git','push','--mirror',dst_url])
+        subprocess.run(['git','push','--mirror',dst_url])
     os.chdir('..')
-    subprocess.check_call(['rm','-rf',src_url.split('/')[-1]]) 
+    subprocess.check_call(['rm','-rf',src_url.split('/')[-1]])
 
     print('\n\nFinished migration. New project URL is %s'%dst_info['html_url'])
     print('Please open the URL and check if everything is fine.')
     if not args.no_confirm:
         input('Hit any key to continue!')
-    
+
 print('\n\nEverything finished!\n')