misc/machine_readable/fetch-machine-readable_salsa

-#!/bin/sh
-# This script should be called on Alioth to gather all machine readable files
-# of VCSes mentioned in SVNDIRS and GITDIRS
-#
-# TODO: store file <pkg>.vcs with the Vcs URL where the files were found
-
-. ./salsarc
-
-TDNAME=machine-readable
-MACHINEREADABLEARCHIVE=/srv/blends.debian.org/www/_${TDNAME}/${TDNAME}.tar.bz2
-READMEDEBIANARCHIVE=/srv/blends.debian.org/www/_${TDNAME}/README.Debian.tar.bz2
-TARGETDIR=${HOME}/$TDNAME
-mkdir -p $TARGETDIR
-
-ERRLOG=${HOME}/${TDNAME}.log
-rm -f $ERRLOG
-
-GITTEAMS="debichem-team
-          med-team
-          r-pkg-team
-          science-team"
-
-ANONSCM_DICT="debichem-team:debichem/packages\n
-              med-team:debian-med\n
-              r-pkg-team:debian-science/packages\n
-              science-team:debian-science/packages"
-
-get_one_page () {
-    curl --silent -D $HEADERS_FILE "$SALSA_URL/groups/$SALSA_GROUP_ID/projects?private_token=$SALSA_TOKEN&simple=true&per_page=$PER_PAGE&page=$PAGENO" | jq --raw-output ".[] | .path"
-}
-
-# Alternatively use
-#   gitlab-api-v4 --all group_projects science-team | jq -r '.[].name'
-
-list_salsa_projects () {
-  SALSA_GROUP="$1"
-  PER_PAGE=100
-  HEADERS_FILE=$(mktemp)
-  trap "rm -f $HEADERS_FILE" EXIT
-
-  SALSA_GROUP_ID=$(curl --silent -f -XGET --header "PRIVATE-TOKEN: $SALSA_TOKEN" "$SALSA_URL/groups?all_available=false" | jq ".[] | select(.path == \"$SALSA_GROUP\") | .id")
-
-  PAGENO=1
-
-  get_one_page
-
-  TOTAL_PAGES=$(grep ^X-Total-Pages: $HEADERS_FILE)
-  TOTAL_PAGES=${TOTAL_PAGES#*: }
-
-  if [ "$TOTAL_PAGES" != "" ] ; then
-    while [ $PAGENO -lt $TOTAL_PAGES ]
-    do
-      PAGENO=$((PAGENO+1))
-      get_one_page
-    done
-  fi
-}
-
-mkdir -p $TARGETDIR
-rm -rf $TARGETDIR/*
-
-for team in $GITTEAMS ; do
-  echo $team >> $ERRLOG
-  for gitrepo in `list_salsa_projects $team` ; do
-    firstletter=`echo $gitrepo | sed "s/^\(.\).*/\1/"`
-    mkdir -p $TARGETDIR/$firstletter
-    echo "Vcs-Git: git@salsa.debian.org:$team/$gitrepo.git" > $TARGETDIR/$firstletter/${gitrepo}.vcs
-    # Hint: Debian Med and Debichem R packages do not match this but who cares ;-)
-    #       Also some packages that never were on Alioth Git do not have a valid Vcs-Browser via that method
-    anonscmdir=`echo $ANONSCM_DICT | grep $team | head -n 1 | sed "s/^ *${team}://"`
-    echo "Vcs-Browser: https://anonscm.debian.org/cgit/$anonscmdir/$gitrepo.git" >> $TARGETDIR/$firstletter/${gitrepo}.vcs
-    wget -q https://salsa.debian.org/$team/$gitrepo/raw/master/debian/changelog -O $TARGETDIR/$firstletter/${gitrepo}.changelog
-    wget -q https://salsa.debian.org/$team/$gitrepo/raw/master/debian/control -O $TARGETDIR/$firstletter/${gitrepo}.control
-    wget -q https://salsa.debian.org/$team/$gitrepo/raw/master/debian/copyright -O $TARGETDIR/$firstletter/${gitrepo}.copyright
-    wget -q https://salsa.debian.org/$team/$gitrepo/raw/master/debian/upstream/metadata -O $TARGETDIR/$firstletter/${gitrepo}.upstream
-    wget -q https://salsa.debian.org/$team/$gitrepo/raw/master/debian/upstream/edam -O $TARGETDIR/$firstletter/${gitrepo}.edam
-  done
-done
-
-FOUNDDIRS=`ls $TDNAME | wc -l`
-if [ $FOUNDDIRS -lt 10 ] ; then
-  echo "Found way to less packages in $TDNAME" >>  $ERRLOG
-  exit
-fi
-rm -f $MACHINEREADABLEARCHIVE
-cd $TARGETDIR
-cd ..
-find $TDNAME -maxdepth 1 -mindepth 1 -name ".*" -delete
-# Remove empty files
-find $TDNAME -type f -empty -delete
-tar --exclude=*README.Debian -cjf $MACHINEREADABLEARCHIVE $TDNAME
-tar --exclude=*.control --exclude=*.changelog --exclude=*.upstream --exclude=*.edam --exclude=*.vcs --exclude=*.copyright -cjf $READMEDEBIANARCHIVE $TDNAME
-
-# Check for remaining tempfiles
-if ls -l /tmp/tmp.* 2>/dev/null | grep `whoami` > /dev/null ; then
-    echo "There are tempfiles remaining!\n" | mailx -s "$0 has left some remaining files in /tmp" tille@debian.org
-fi

misc/machine_readable/fetch-machine-readable_salsa.py

@@ -17,6 +17,7 @@ BLENDSGROUPS=[ 'Debian 3D Printing Team'
              , 'Debian Hamradio Maintainers'
              , 'Debian IoT'
              , 'Debian Java Maintainers'
+             , 'Debian JavaScript Maintainers'
              , 'Debian Med'
              , 'Debian Multimedia Team'
              , 'Debian Octave Group'
@@ -24,9 +25,16 @@ BLENDSGROUPS=[ 'Debian 3D Printing Team'
              , 'Debian PhotoTools Team'
              , 'Debian R Packages Maintainers'
              , 'Debian Science Team'
+             , 'Debian Tryton Maintainers'
+             , 'Debian VoIP Packaging Team'
              , 'Debichem'
              ]
 
+# missing on Salsa yet:
+#            pkg-exppsy
+#            Python
+# FIXME: Debian Perl Group is listed with only 2 projects which is definitely wrong
+
 # just start with one team
 # BLENDSGROUPS=[ 'Debian Med' ]
 
@@ -80,9 +88,9 @@ gl = gitlab.Gitlab("https://salsa.debian.org") # , private_token=SALSA_TOKEN) #
 
 blends_groups = get_blends_groups()
 for group in blends_groups:
-    print(group.attributes['name'], group.attributes['id'], group.attributes['path']) # , group.attributes['description'], group.attributes['full_name'])
     gpath = group.attributes['path']
     projects = group.projects.list(all=True, order_by='name', sort='asc')
+    print(group.attributes['name'], len(projects)) #  group.attributes['id'], group.attributes['path']) # , group.attributes['description'], group.attributes['full_name'])
     # DEBUG : only few projects to be faster
     # projects = group.projects.list(page=1, per_page=10, order_by='name', sort='asc')
     for pr in projects: