misc/machine_readable/fetch-machine-readable_salsa.py

@@ -63,9 +63,9 @@ MACHINEREADABLEARCHIVE='/srv/blends.debian.org/www/_'+TDNAME+'/'+TDNAME+'.tar.xz
 READMEDEBIANARCHIVE='/srv/blends.debian.org/www/_'+TDNAME+'/README.Debian.tar.xz'
 TARGETDIR=os.path.join(os.environ['HOME'],TDNAME)
 
-SLEEPTIMEFILES=5
-SLEEPTIMEPROJECTS=5
-SLEEPUNTILRETRY=3600
+SLEEPTIMEFILES=5	# Wait 5 seconds until next file is fetched
+SLEEPTIMEPROJECTS=60	# Wait 60 seconds until next group is queried for all projects
+SLEEPUNTILRETRY=600	# In case of overloaded Salsa wait for 10min
 debug=1
 
 # do not clean dirs.  FIXME: Find a method to detect removed repositories and clean this up here
@@ -74,10 +74,6 @@ try:
 except:
     pass
 
-def wait_for_salsa():
-    print("Received GitlabGetError: 429 - Retry later ... waiting for %i seconds" % SLEEPUNTILRETRY )
-    time.sleep(SLEEPUNTILRETRY)
-
 def output_metadata(project, subdir, metadata):
     try:
         items = project.repository_tree(path=subdir)
@@ -116,9 +112,15 @@ LAST_ACTIVITY_LENGTH=len(LAST_ACTIVITY)
 #blends_groups = get_blends_groups()
 for blend in BLENDSGROUPS:
     unchanged, changed, new, ignored = (0, 0, 0, 0)
-    group=gl.groups.get(BLENDSGROUPS[blend])
+    group=gl.groups.get(BLENDSGROUPS[blend], with_projects=False, simple=True)
     gpath = group.path
-    projects = group.projects.list(all=True, order_by='name', sort='asc', simple=True)
+    try:
+        projects = group.projects.list(all=True, order_by='name', sort='asc', simple=True)
+    except gitlab.exceptions.GitlabGetError as err:
+        print("GitlabGetError: %s - wait %s seconds", (str(err), SLEEPUNTILRETRY))
+        time.sleep(SLEEPUNTILRETRY)
+        projects = group.projects.list(all=True, order_by='name', sort='asc', simple=True)
+        # If Salsa is too busy do not try again
     # DEBUG : only few projects to be faster
     # projects = group.projects.list(page=1, per_page=10, order_by='name', sort='asc')
     print("%s has %i projects (Blend name %s)" % (group.name, len(projects), blend)) #  group.attributes['id'], group.attributes['path']) # , group.attributes['description'], group.attributes['full_name'])
@@ -176,6 +178,7 @@ for blend in BLENDSGROUPS:
         if output_metadata(pr, 'debian', debianmetadata):
             output_metadata(pr, 'debian/upstream', upstreammetadata)
     print("In %s %i repositories changed, %i remained unchanged, %i were new and %i were ignored." % (blend, changed, unchanged, new, ignored))
+    time.sleep(SLEEPTIMEPROJECTS)
 
 # os.system("tar --exclude=*README.Debian -caf %s %s" % (MACHINEREADABLEARCHIVE, TARGETDIR))
 p = subprocess.Popen(['tar', '--exclude=*README.Debian', '-caf', MACHINEREADABLEARCHIVE, TDNAME], cwd=os.environ['HOME'])