Skip to content
GitLab
Explore
Sign in
Register
Primary navigation
Search or go to…
Project
W
website
Manage
Activity
Members
Labels
Plan
Wiki
Code
Merge requests
0
Repository
Branches
Commits
Tags
Repository graph
Compare revisions
Snippets
Build
Pipelines
Jobs
Pipeline schedules
Artifacts
Deploy
Releases
Container Registry
Model registry
Operate
Environments
Analyze
Value stream analytics
Contributor analytics
CI/CD analytics
Repository analytics
Model experiments
Help
Help
Support
GitLab documentation
Compare GitLab plans
Community forum
Contribute to GitLab
Provide feedback
Keyboard shortcuts
?
Snippets
Groups
Projects
Show more breadcrumbs
Debian Blends Team
website
Compare revisions
0733d0aa5e4e40743493131ccac2b6f513d6312c to e26aa67b67b3df7a361310bcd112f92d8986f7fc
Compare revisions
Changes are shown as if the
source
revision was being merged into the
target
revision.
Learn more about comparing revisions.
Source
blends-team/website
Select target project
No results found
e26aa67b67b3df7a361310bcd112f92d8986f7fc
Select Git revision
Swap
Target
blends-team/website
Select target project
gber/website
parodper/website
sunilmohan/website
pabs/blends-website
blends-team/website
moeller/website
jcristau/blends-website
7 results
0733d0aa5e4e40743493131ccac2b6f513d6312c
Select Git revision
Show changes
Only incoming changes from source
Include changes to target since source was created
Compare
Commits on Source (2)
Adjust timings
· a193b182
Andreas Tille
authored
6 years ago
a193b182
with_projects=False to reduce amount of submitted data (Thanks for the hint to Raphaël Herzog)
· e26aa67b
Andreas Tille
authored
6 years ago
e26aa67b
Hide whitespace changes
Inline
Side-by-side
Showing
1 changed file
misc/machine_readable/fetch-machine-readable_salsa.py
+12
-9
12 additions, 9 deletions
misc/machine_readable/fetch-machine-readable_salsa.py
with
12 additions
and
9 deletions
misc/machine_readable/fetch-machine-readable_salsa.py
View file @
e26aa67b
...
@@ -63,9 +63,9 @@ MACHINEREADABLEARCHIVE='/srv/blends.debian.org/www/_'+TDNAME+'/'+TDNAME+'.tar.xz
...
@@ -63,9 +63,9 @@ MACHINEREADABLEARCHIVE='/srv/blends.debian.org/www/_'+TDNAME+'/'+TDNAME+'.tar.xz
READMEDEBIANARCHIVE
=
'
/srv/blends.debian.org/www/_
'
+
TDNAME
+
'
/README.Debian.tar.xz
'
READMEDEBIANARCHIVE
=
'
/srv/blends.debian.org/www/_
'
+
TDNAME
+
'
/README.Debian.tar.xz
'
TARGETDIR
=
os
.
path
.
join
(
os
.
environ
[
'
HOME
'
],
TDNAME
)
TARGETDIR
=
os
.
path
.
join
(
os
.
environ
[
'
HOME
'
],
TDNAME
)
SLEEPTIMEFILES
=
5
SLEEPTIMEFILES
=
5
# Wait 5 seconds until next file is fetched
SLEEPTIMEPROJECTS
=
5
SLEEPTIMEPROJECTS
=
60
# Wait 60 seconds until next group is queried for all projects
SLEEPUNTILRETRY
=
3
600
SLEEPUNTILRETRY
=
600
# In case of overloaded Salsa wait for 10min
debug
=
1
debug
=
1
# do not clean dirs. FIXME: Find a method to detect removed repositories and clean this up here
# do not clean dirs. FIXME: Find a method to detect removed repositories and clean this up here
...
@@ -74,10 +74,6 @@ try:
...
@@ -74,10 +74,6 @@ try:
except
:
except
:
pass
pass
def
wait_for_salsa
():
print
(
"
Received GitlabGetError: 429 - Retry later ... waiting for %i seconds
"
%
SLEEPUNTILRETRY
)
time
.
sleep
(
SLEEPUNTILRETRY
)
def
output_metadata
(
project
,
subdir
,
metadata
):
def
output_metadata
(
project
,
subdir
,
metadata
):
try
:
try
:
items
=
project
.
repository_tree
(
path
=
subdir
)
items
=
project
.
repository_tree
(
path
=
subdir
)
...
@@ -116,9 +112,15 @@ LAST_ACTIVITY_LENGTH=len(LAST_ACTIVITY)
...
@@ -116,9 +112,15 @@ LAST_ACTIVITY_LENGTH=len(LAST_ACTIVITY)
#blends_groups = get_blends_groups()
#blends_groups = get_blends_groups()
for
blend
in
BLENDSGROUPS
:
for
blend
in
BLENDSGROUPS
:
unchanged
,
changed
,
new
,
ignored
=
(
0
,
0
,
0
,
0
)
unchanged
,
changed
,
new
,
ignored
=
(
0
,
0
,
0
,
0
)
group
=
gl
.
groups
.
get
(
BLENDSGROUPS
[
blend
])
group
=
gl
.
groups
.
get
(
BLENDSGROUPS
[
blend
]
,
with_projects
=
False
,
simple
=
True
)
gpath
=
group
.
path
gpath
=
group
.
path
projects
=
group
.
projects
.
list
(
all
=
True
,
order_by
=
'
name
'
,
sort
=
'
asc
'
,
simple
=
True
)
try
:
projects
=
group
.
projects
.
list
(
all
=
True
,
order_by
=
'
name
'
,
sort
=
'
asc
'
,
simple
=
True
)
except
gitlab
.
exceptions
.
GitlabGetError
as
err
:
print
(
"
GitlabGetError: %s - wait %s seconds
"
,
(
str
(
err
),
SLEEPUNTILRETRY
))
time
.
sleep
(
SLEEPUNTILRETRY
)
projects
=
group
.
projects
.
list
(
all
=
True
,
order_by
=
'
name
'
,
sort
=
'
asc
'
,
simple
=
True
)
# If Salsa is too busy do not try again
# DEBUG : only few projects to be faster
# DEBUG : only few projects to be faster
# projects = group.projects.list(page=1, per_page=10, order_by='name', sort='asc')
# projects = group.projects.list(page=1, per_page=10, order_by='name', sort='asc')
print
(
"
%s has %i projects (Blend name %s)
"
%
(
group
.
name
,
len
(
projects
),
blend
))
# group.attributes['id'], group.attributes['path']) # , group.attributes['description'], group.attributes['full_name'])
print
(
"
%s has %i projects (Blend name %s)
"
%
(
group
.
name
,
len
(
projects
),
blend
))
# group.attributes['id'], group.attributes['path']) # , group.attributes['description'], group.attributes['full_name'])
...
@@ -176,6 +178,7 @@ for blend in BLENDSGROUPS:
...
@@ -176,6 +178,7 @@ for blend in BLENDSGROUPS:
if
output_metadata
(
pr
,
'
debian
'
,
debianmetadata
):
if
output_metadata
(
pr
,
'
debian
'
,
debianmetadata
):
output_metadata
(
pr
,
'
debian/upstream
'
,
upstreammetadata
)
output_metadata
(
pr
,
'
debian/upstream
'
,
upstreammetadata
)
print
(
"
In %s %i repositories changed, %i remained unchanged, %i were new and %i were ignored.
"
%
(
blend
,
changed
,
unchanged
,
new
,
ignored
))
print
(
"
In %s %i repositories changed, %i remained unchanged, %i were new and %i were ignored.
"
%
(
blend
,
changed
,
unchanged
,
new
,
ignored
))
time
.
sleep
(
SLEEPTIMEPROJECTS
)
# os.system("tar --exclude=*README.Debian -caf %s %s" % (MACHINEREADABLEARCHIVE, TARGETDIR))
# os.system("tar --exclude=*README.Debian -caf %s %s" % (MACHINEREADABLEARCHIVE, TARGETDIR))
p
=
subprocess
.
Popen
([
'
tar
'
,
'
--exclude=*README.Debian
'
,
'
-caf
'
,
MACHINEREADABLEARCHIVE
,
TDNAME
],
cwd
=
os
.
environ
[
'
HOME
'
])
p
=
subprocess
.
Popen
([
'
tar
'
,
'
--exclude=*README.Debian
'
,
'
-caf
'
,
MACHINEREADABLEARCHIVE
,
TDNAME
],
cwd
=
os
.
environ
[
'
HOME
'
])
...
...
This diff is collapsed.
Click to expand it.