misc/machine_readable/fetch-machine-readable_salsa.py

@@ -2,8 +2,8 @@
 
 import gitlab
 import os
-
-PER_PAGE=99
+import shutil
+import base64
 
 BLENDSGROUPS=[ 'Debian 3D Printing Team'
              , 'Debian Accessibility Team'
@@ -22,6 +22,27 @@ BLENDSGROUPS=[ 'Debian 3D Printing Team'
              , 'Debichem'
              ]
 
+
+# just start with one team
+BLENDSGROUPS=[ 'Debian Med' ]
+
+debianmetadata = [ 'changelog',
+                   'control',
+                   'copyright',
+                   'README.Debian'
+                 ]
+upstreammetadata = [ 'edam',
+                     'metadata'
+                   ]
+
+TDNAME='machine-readable'
+MACHINEREADABLEARCHIVE='/srv/blends.debian.org/www/_'+TDNAME+'/'+TDNAME+'.tar.xz'
+READMEDEBIANARCHIVE='/srv/blends.debian.org/www/_'+TDNAME+'/README.Debian.tar.xz'
+TARGETDIR=os.path.join(os.environ['HOME'],TDNAME)
+# cleanup TARGETDIR first
+shutil.rmtree(TARGETDIR)
+os.makedirs(TARGETDIR)
+
 def get_blends_groups():
     blends_groups=[]
     groups = gl.groups.list(all=True, order_by='name', sort='asc')
@@ -31,6 +52,16 @@ def get_blends_groups():
             blends_groups.append(group)
     return blends_groups
 
+def output_metadata(subdir, metadata):
+    items = project.repository_tree(path=subdir)
+    for item in items:
+        if item['name'] in metadata:
+            file_info = project.repository_blob(item['id'])
+            content = base64.b64decode(file_info['content'])
+            with open(os.path.join(namedir,name+'.'+item['name']), 'wb') as out:
+                out.write(content)
+            out.close()
+
 # SALSA_TOKEN=os.environ['SALSA_TOKEN']
 
 gl = gitlab.Gitlab("https://salsa.debian.org") # , private_token=SALSA_TOKEN) # anonymous access is fine
@@ -38,11 +69,21 @@ gl = gitlab.Gitlab("https://salsa.debian.org") # , private_token=SALSA_TOKEN) #
 blends_groups = get_blends_groups()
 for group in blends_groups:
     print(group.attributes['name'], group.attributes['id'], group.attributes['path']) # , group.attributes['description'], group.attributes['full_name'])
-    projects = group.projects.list(all=True, order_by='name', sort='asc')
-    for project in projects:
-        print("   ", project.attributes['name'])
-
-#for project in git.getall(git.getprojects):
-#    print(project['name'], project['path_with_namespace'], project['id'])
-    # getrawfile(project['id'], 'master', 'debian/changelog')
-    # getfile(project['id'], 'debian/changelog', 'master')
\ No newline at end of file
+    gpath = group.attributes['path']
+    #projects = group.projects.list(all=True, order_by='name', sort='asc')
+    #projects = group.projects.all(order_by='name', sort='asc')
+    # DEBUG : only few projects to be faster
+    projects = group.projects.list(page=1, per_page=10, order_by='name', sort='asc')
+    for pr in projects:
+        project = gl.projects.get(pr.attributes['id']) # without this extra get repository_tree() fails
+        name = project.attributes['name']
+        namedir = os.path.join(TARGETDIR, name[0])
+        if not os.path.exists(namedir):
+            os.makedirs(namedir)
+        with open(os.path.join(namedir,name+'.vcs'), 'w') as out:
+            out.write("Vcs-Browser: git@salsa.debian.org:%s/%s\n" % (gpath, name))
+            out.write("Vcs-Git: git@salsa.debian.org:%s/%s.git\n" % (gpath, name))
+        out.close()
+        # print(project)
+        output_metadata('debian', debianmetadata)
+        output_metadata('debian/upstream', upstreammetadata)