misc/machine_readable/fetch-machine-readable_salsa.py

@@ -37,7 +37,7 @@ BLENDSGROUPS={ 'Debian 3D Printing Team'       : '3dprinter'
 # FIXME: Debian Perl Group is listed with only 2 projects which is definitely wrong
 
 # just start with one team
-BLENDSGROUPS={ 'Debian Med'                    : 'debian-med'}
+# BLENDSGROUPS={ 'Debian Med'                    : 'debian-med'}
 
 debianmetadata = [ 'changelog',
                    'control',
@@ -54,7 +54,7 @@ READMEDEBIANARCHIVE='/srv/blends.debian.org/www/_'+TDNAME+'/README.Debian.tar.xz
 TARGETDIR=os.path.join(os.environ['HOME'],TDNAME)
 
 SLEEPTIME=1
-SLEEPUNTILRETRY=3600
+SLEEPUNTILRETRY=1800
 debug=0
 
 # do not clean dirs.  FIXME: Find a method to detect removed repositories and clean this up here
@@ -63,6 +63,10 @@ try:
 except:
     pass
 
+def wait_for_salsa():
+    time.sleep(SLEEPUNTILRETRY)
+    print("Received GitlabGetError: 429 - Retry later ... waiting for %i seconds" % SLEEPUNTILRETRY )
+
 def get_blends_groups():
     blends_groups=[]
     groups = gl.groups.list(all=True, order_by='name', sort='asc')
@@ -87,8 +91,7 @@ def output_metadata(subdir, metadata):
             try:
                 file_info = project.repository_blob(item['id'])
             except gitlab.exceptions.GitlabGetError:
-                time.sleep(SLEEPUNTILRETRY)
-                print("Received GitlabGetError: 429 - Retry later ... waiting for %i seconds" % SLEEPUNTILRETRY )
+                wait_for_salsa()
                 file_info = project.repository_blob(item['id'])
             content = base64.b64decode(file_info['content'])
             if item['name'] == 'metadata':
@@ -117,7 +120,11 @@ for group in blends_groups:
     # projects = group.projects.list(page=1, per_page=10, order_by='name', sort='asc')
     print("%s has %i projects (Blend name %s)" % (group.attributes['name'], len(projects), blend)) #  group.attributes['id'], group.attributes['path']) # , group.attributes['description'], group.attributes['full_name'])
     for pr in projects:
-        project = gl.projects.get(pr.attributes['id']) # without this extra get repository_tree() fails
+        try:
+            project = gl.projects.get(pr.attributes['id']) # without this extra get repository_tree() fails
+        except gitlab.exceptions.GitlabGetError:
+            wait_for_salsa()
+            project = gl.projects.get(pr.attributes['id']) # without this extra get repository_tree() fails
         name = project.attributes['name']
         namedir = os.path.join(TARGETDIR, name[0])
         if not os.path.exists(namedir):