misc/machine_readable/fetch-machine-readable_salsa.py

@@ -31,6 +31,17 @@ BLENDSGROUPS={ 'Debian 3D Printing Team'       : '3dprinter'
              , 'Debichem'                      : 'debichem'
              }
 
+# Do not parse these projects which are no real packages
+BLACKLIST = [  'bio-linux'
+             , 'communication'
+             , 'helper-scripts'
+             , 'package_template'
+             , 'papers'
+             , 'policy'
+             , 'scripts'
+             , 'website'
+            ]
+
 # missing on Salsa yet:
 #            pkg-exppsy
 #            Python
@@ -53,7 +64,8 @@ MACHINEREADABLEARCHIVE='/srv/blends.debian.org/www/_'+TDNAME+'/'+TDNAME+'.tar.xz
 READMEDEBIANARCHIVE='/srv/blends.debian.org/www/_'+TDNAME+'/README.Debian.tar.xz'
 TARGETDIR=os.path.join(os.environ['HOME'],TDNAME)
 
-SLEEPTIME=1
+SLEEPTIMEFILES=3
+SLEEPTIMEPROJECTS=0.1
 SLEEPUNTILRETRY=2400
 debug=1
 
@@ -112,7 +124,7 @@ LAST_ACTIVITY_LENGTH=len(LAST_ACTIVITY)
 
 blends_groups = get_blends_groups()
 for group in blends_groups:
-    unchanged, changed, new = (0, 0, 0)
+    unchanged, changed, new, ignored = (0, 0, 0, 0)
     gpath = group.attributes['path']
     blend = BLENDSGROUPS[group.attributes['name']]
     projects = group.projects.list(all=True, order_by='name', sort='asc')
@@ -126,6 +138,12 @@ for group in blends_groups:
             wait_for_salsa()
             project = gl.projects.get(pr.attributes['id']) # without this extra get repository_tree() fails
         name = project.attributes['name']
+        time.sleep(SLEEPTIMEPROJECTS)
+        if name in BLACKLIST or name.endswith('.pages.debian.net'):
+            ignored += 1
+            if debug > 0:
+                print("Ignore project %s of blend %s." % (name, blend))
+            continue
         namedir = os.path.join(TARGETDIR, name[0])
         if not os.path.exists(namedir):
             os.makedirs(namedir)
@@ -139,11 +157,11 @@ for group in blends_groups:
                     if last_activity_at == project.attributes['last_activity_at']:
                         dosomething=False
                         unchanged += 1
-                        if debug > 0:
+                        if debug > 1:
                             print("Do nothing in %s since found last_activity_at %s == repository %s" % (name, last_activity_at, project.attributes['last_activity_at']))
                     else:
                         if debug > 0:
-                            print("Continue with %s since found last_activity_at %s != repository %s\n" % (name, last_activity_at, project.attributes['last_activity_at']))
+                            print("Continue with %s since found last_activity_at %s != repository %s" % (name, last_activity_at, project.attributes['last_activity_at']))
                         changed += 1
                 invcs.close()
         except FileNotFoundError:
@@ -151,8 +169,8 @@ for group in blends_groups:
             pass
         if not dosomething:
             continue
-        if debug > 0:
-            print("Opening %s failed thus writing." % os.path.join(namedir,name+'.vcs'))
+        if debug > 1:
+            print("Writing %s." % os.path.join(namedir,name+'.vcs'))
         with open(os.path.join(namedir,name+'.vcs'), 'w') as out:
             out.write("Vcs-Browser: https://salsa.debian.org/%s/%s\n" % (gpath, name))
             out.write("Vcs-Git: https://salsa.debian.org/%s/%s.git\n" % (gpath, name))
@@ -162,10 +180,10 @@ for group in blends_groups:
         # print(project)
 	# be friendly to Salsa and add some delay timr before real reading of data
         # see thread around https://lists.debian.org/debian-devel/2018/07/msg00125.html
-        time.sleep(SLEEPTIME)
+        time.sleep(SLEEPTIMEFILES)
         if output_metadata('debian', debianmetadata):
             output_metadata('debian/upstream', upstreammetadata)
-    print("In %s %i repositories changed, %i remained unchanged and %i were new." % (blend, changed, unchanged, new))
+    print("In %s %i repositories changed, %i remained unchanged, %i were new and %i were ignored." % (blend, changed, unchanged, new, ignored))
 
 # os.system("tar --exclude=*README.Debian -caf %s %s" % (MACHINEREADABLEARCHIVE, TARGETDIR))
 p = subprocess.Popen(['tar', '--exclude=*README.Debian', '-caf', MACHINEREADABLEARCHIVE, TDNAME], cwd=os.environ['HOME'])