misc/machine_readable/fetch-machine-readable_salsa.py

@@ -37,6 +37,7 @@ BLENDSGROUPS={ '3dprinter'		: '3dprinting-team'
 BLACKLIST = [  'bio-linux'
              , 'communication'
              , 'helper-scripts'
+             , 'maintenance-utilities'
              , 'package_template'
              , 'papers'
              , 'policy'
@@ -80,10 +81,13 @@ def output_metadata(project, subdir, metadata):
     try:
         items = project.repository_tree(path=subdir, recursive=False)
     except gitlab.exceptions.GitlabGetError as err:
+        if "403 Forbidden" in str(err): 
+            print("%s/%s has restricted access to dir %s - will not parse" % (gpath, name, subdir), file=sys.stderr)
+            return None
         print("%s/%s does not seem to have dir %s (%s)" % (gpath, name, subdir, str(err)), file=sys.stderr)
         return None
     except:
-        print("%s/%s does not seem to have dir %s" % (gpath, name, subdir), file=sys.stderr)
+        print("%s/%s does not seem to have dir %s. PLEASE VERIFY THIS SINCE IT IS SUSPICIOUS! " % (gpath, name, subdir), file=sys.stderr)
         return None
 #        print("To be sure try again ... after sleeping %i" % SLEEPTIMEPROJECTS)
 #        time.sleep(SLEEPTIMEPROJECTS)
@@ -97,10 +101,6 @@ def output_metadata(project, subdir, metadata):
 #            return None
     for item in items:
         if item['name'] in metadata:
-            try:
-                file_info = project.repository_blob(item['id'])
-            except gitlab.exceptions.GitlabGetError:
-                wait_for_salsa()
             file_info = project.repository_blob(item['id'])
             content = base64.b64decode(file_info['content'])
             if item['name'] == 'metadata':
@@ -181,13 +181,20 @@ for blend, gpath in BLENDSGROUPS.items():
             out.write("%s%s\n" % (LAST_ACTIVITY, project.last_activity_at))
         out.close()
         # print(name)
-	# be friendly to Salsa and add some delay timr before real reading of data
+	# be friendly to Salsa and add some delay time before real reading of data
         # see thread around https://lists.debian.org/debian-devel/2018/07/msg00125.html
         time.sleep(SLEEPTIMEFILES)
         # for lazy=True read: https://python-gitlab.readthedocs.io/en/stable/api-usage.html#lazy-objects
         pr = gl.projects.get(project.id, lazy=True)
         if output_metadata(pr, 'debian', debianmetadata):
             output_metadata(pr, 'debian/upstream', upstreammetadata)
+        try:
+            with open(os.path.join(namedir,name+'.changelog'), 'r') as chlog:
+                chlogname = chlog.readline().strip().split(' ')[0]
+            if chlogname != name:
+                print("Warning: package name of project found at %s/%s is different than dir: %s" % (gpath, name, chlogname))
+        except FileNotFoundError:
+            pass # If no changelog file was created this project is irrelevant anyway
     print("In %s %i repositories changed, %i remained unchanged, %i were new and %i were ignored." % (blend, changed, unchanged, new, ignored))
     time.sleep(SLEEPTIMEPROJECTS)