Commit fb1357e0 authored by Andreas Tille's avatar Andreas Tille

Read from Salsa only if changes are to be expected

parent fe001079
......@@ -37,7 +37,7 @@ BLENDSGROUPS={ 'Debian 3D Printing Team' : '3dprinter'
# FIXME: Debian Perl Group is listed with only 2 projects which is definitely wrong
# just start with one team
# BLENDSGROUPS={ 'Debian Med' : 'debian-med'}
BLENDSGROUPS={ 'Debian Med' : 'debian-med'}
debianmetadata = [ 'changelog',
'control',
......@@ -52,12 +52,15 @@ TDNAME='machine-readable'
MACHINEREADABLEARCHIVE='/srv/blends.debian.org/www/_'+TDNAME+'/'+TDNAME+'.tar.xz'
READMEDEBIANARCHIVE='/srv/blends.debian.org/www/_'+TDNAME+'/README.Debian.tar.xz'
TARGETDIR=os.path.join(os.environ['HOME'],TDNAME)
# cleanup TARGETDIR first
SLEEPTIME=1
debug=0
# do not clean dirs. FIXME: Find a method to detect removed repositories and clean this up here
try:
shutil.rmtree(TARGETDIR)
os.makedirs(TARGETDIR)
except:
pass
os.makedirs(TARGETDIR)
def get_blends_groups():
blends_groups=[]
......@@ -95,37 +98,53 @@ def output_metadata(subdir, metadata):
gl = gitlab.Gitlab("https://salsa.debian.org") # , private_token=SALSA_TOKEN) # anonymous access is fine
LAST_ACTIVITY='last_activity_at: '
LAST_ACTIVITY_LENGTH=len(LAST_ACTIVITY)
blends_groups = get_blends_groups()
for group in blends_groups:
gpath = group.attributes['path']
blend = BLENDSGROUPS[group.attributes['name']]
projects = group.projects.list(all=True, order_by='name', sort='asc')
print("%s has %i projects (Blend name %s)" % (group.attributes['name'], len(projects), blend)) # group.attributes['id'], group.attributes['path']) # , group.attributes['description'], group.attributes['full_name'])
# DEBUG : only few projects to be faster
# projects = group.projects.list(page=1, per_page=10, order_by='name', sort='asc')
print("%s has %i projects (Blend name %s)" % (group.attributes['name'], len(projects), blend)) # group.attributes['id'], group.attributes['path']) # , group.attributes['description'], group.attributes['full_name'])
for pr in projects:
# it seems that 1 second sleeping does not help anyway - there seems to be a block for about 1 hour
# FIXME: try adding sleep(60) statements in the main loop or so
# For the moment wait for responses to https://lists.debian.org/debian-devel/2018/07/msg00119.html
# before implementing anything based on wild guessing
try:
project = gl.projects.get(pr.attributes['id']) # without this extra get repository_tree() fails
except gitlab.exceptions.GitlabGetError:
print("Warning: gitlab.exceptions.GitlabGetError while trying to fetch attributes of %s. Wait one second and try again." % pr.attributes['id'])
time.sleep(1)
project = gl.projects.get(pr.attributes['id']) # without this extra get repository_tree() fails
except gitlab.exceptions.GitlabHttpError:
print("Warning: GitlabHttpError while trying to fetch attributes of %s. Wait one second and try again." % pr.attributes['id'])
time.sleep(1)
project = gl.projects.get(pr.attributes['id']) # without this extra get repository_tree() fails
time.sleep(SLEEPTIME)
project = gl.projects.get(pr.attributes['id']) # without this extra get repository_tree() fails
name = project.attributes['name']
namedir = os.path.join(TARGETDIR, name[0])
if not os.path.exists(namedir):
os.makedirs(namedir)
dosomething=True
try:
# try reading file - if not exists its just written
invcs = open(os.path.join(namedir,name+'.vcs'), 'r')
for line in invcs.readlines():
if line.startswith(LAST_ACTIVITY):
last_activity_at = line[LAST_ACTIVITY_LENGTH:].strip()
if last_activity_at == project.attributes['last_activity_at']:
dosomething=False
if debug > 0:
print("Do nothing in %s since found last_activity_at %s == repository %s" % (name, last_activity_at, project.attributes['last_activity_at']))
else:
if debug > 0:
print("Continue with %s since found last_activity_at %s != repository %s\n" % (name, last_activity_at, project.attributes['last_activity_at']))
invcs.close()
except FileNotFoundError:
pass
if not dosomething:
continue
print("Opening %s failed thus writing." % os.path.join(namedir,name+'.vcs'))
with open(os.path.join(namedir,name+'.vcs'), 'w') as out:
out.write("Vcs-Browser: https://salsa.debian.org/%s/%s\n" % (gpath, name))
out.write("Vcs-Git: https://salsa.debian.org/%s/%s.git\n" % (gpath, name))
out.write("Blend: %s\n" % blend)
out.write("%s%s\n" % (LAST_ACTIVITY, project.attributes['last_activity_at']))
out.close()
# print(project)
if output_metadata('debian', debianmetadata):
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment