misc/machine_readable/fetch-machine-readable_salsa.py

@@ -54,6 +54,7 @@ READMEDEBIANARCHIVE='/srv/blends.debian.org/www/_'+TDNAME+'/README.Debian.tar.xz
 TARGETDIR=os.path.join(os.environ['HOME'],TDNAME)
 
 SLEEPTIME=1
+SLEEPUNTILRETRY=3600
 debug=0
 
 # do not clean dirs.  FIXME: Find a method to detect removed repositories and clean this up here
@@ -83,7 +84,12 @@ def output_metadata(subdir, metadata):
         return None
     for item in items:
         if item['name'] in metadata:
-            file_info = project.repository_blob(item['id'])
+            try:
+                file_info = project.repository_blob(item['id'])
+            except gitlab.exceptions.GitlabGetError:
+                time.sleep(SLEEPUNTILRETRY)
+                print("Received GitlabGetError: 429 - Retry later ... waiting for %i seconds" % SLEEPUNTILRETRY )
+                file_info = project.repository_blob(item['id'])
             content = base64.b64decode(file_info['content'])
             if item['name'] == 'metadata':
                 ext = '.upstream'
@@ -103,6 +109,7 @@ LAST_ACTIVITY_LENGTH=len(LAST_ACTIVITY)
 
 blends_groups = get_blends_groups()
 for group in blends_groups:
+    unchanged, changed, new = (0, 0, 0)
     gpath = group.attributes['path']
     blend = BLENDSGROUPS[group.attributes['name']]
     projects = group.projects.list(all=True, order_by='name', sort='asc')
@@ -110,11 +117,6 @@ for group in blends_groups:
     # projects = group.projects.list(page=1, per_page=10, order_by='name', sort='asc')
     print("%s has %i projects (Blend name %s)" % (group.attributes['name'], len(projects), blend)) #  group.attributes['id'], group.attributes['path']) # , group.attributes['description'], group.attributes['full_name'])
     for pr in projects:
-        # it seems that 1 second sleeping does not help anyway - there seems to be a block for about 1 hour
-        # FIXME: try adding sleep(60) statements in the main loop or so
-        # For the moment wait for responses to https://lists.debian.org/debian-devel/2018/07/msg00119.html
-        # before implementing anything based on wild guessing
-        time.sleep(SLEEPTIME)
         project = gl.projects.get(pr.attributes['id']) # without this extra get repository_tree() fails
         name = project.attributes['name']
         namedir = os.path.join(TARGETDIR, name[0])
@@ -129,13 +131,16 @@ for group in blends_groups:
                     last_activity_at = line[LAST_ACTIVITY_LENGTH:].strip()
                     if last_activity_at == project.attributes['last_activity_at']:
                         dosomething=False
+                        unchanged += 1
                         if debug > 0:
                             print("Do nothing in %s since found last_activity_at %s == repository %s" % (name, last_activity_at, project.attributes['last_activity_at']))
                     else:
                         if debug > 0:
+                            changed += 1
                             print("Continue with %s since found last_activity_at %s != repository %s\n" % (name, last_activity_at, project.attributes['last_activity_at']))
                 invcs.close()
         except FileNotFoundError:
+            new += 1
             pass
         if not dosomething:
             continue
@@ -147,8 +152,12 @@ for group in blends_groups:
             out.write("%s%s\n" % (LAST_ACTIVITY, project.attributes['last_activity_at']))
         out.close()
         # print(project)
+	# be friendly to Salsa and add some delay timr before real reading of data
+        # see thread around https://lists.debian.org/debian-devel/2018/07/msg00125.html
+        time.sleep(SLEEPTIME)
         if output_metadata('debian', debianmetadata):
             output_metadata('debian/upstream', upstreammetadata)
+    print("In %s %i repositories changed, %i remained unchanged and %i were new." % (blend, changed, unchanged, new))
 
 # os.system("tar --exclude=*README.Debian -caf %s %s" % (MACHINEREADABLEARCHIVE, TARGETDIR))
 p = subprocess.Popen(['tar', '--exclude=*README.Debian', '-caf', MACHINEREADABLEARCHIVE, TDNAME], cwd=os.environ['HOME'])