Skip to content
GitLab
Explore
Sign in
Register
Primary navigation
Search or go to…
Project
W
website
Manage
Activity
Members
Labels
Plan
Wiki
Code
Merge requests
0
Repository
Branches
Commits
Tags
Repository graph
Compare revisions
Snippets
Build
Pipelines
Jobs
Pipeline schedules
Artifacts
Deploy
Releases
Container Registry
Model registry
Operate
Environments
Analyze
Value stream analytics
Contributor analytics
CI/CD analytics
Repository analytics
Model experiments
Help
Help
Support
GitLab documentation
Compare GitLab plans
Community forum
Contribute to GitLab
Provide feedback
Keyboard shortcuts
?
Snippets
Groups
Projects
Show more breadcrumbs
Debian Blends Team
website
Compare revisions
9288beff415db2ce5d8346075299d58390090d19 to 24f5e093cef3665786d1eb74d9fc6878870da493
Compare revisions
Changes are shown as if the
source
revision was being merged into the
target
revision.
Learn more about comparing revisions.
Source
blends-team/website
Select target project
No results found
24f5e093cef3665786d1eb74d9fc6878870da493
Select Git revision
Swap
Target
blends-team/website
Select target project
gber/website
parodper/website
sunilmohan/website
pabs/blends-website
blends-team/website
moeller/website
jcristau/blends-website
7 results
9288beff415db2ce5d8346075299d58390090d19
Select Git revision
Show changes
Only incoming changes from source
Include changes to target since source was created
Compare
Commits on Source (5)
No extra newline
· 19a9b142
Andreas Tille
authored
6 years ago
19a9b142
More fine grained debugging
· 3ad6fb42
Andreas Tille
authored
6 years ago
3ad6fb42
Use blacklist for project names that are probably no packages and thus can be ignored
· c83a81bb
Andreas Tille
authored
6 years ago
c83a81bb
Inject break to not stress test Salsa to much
· 86a17244
Andreas Tille
authored
6 years ago
86a17244
Fix syntax
· 24f5e093
Andreas Tille
authored
6 years ago
24f5e093
Hide whitespace changes
Inline
Side-by-side
Showing
1 changed file
misc/machine_readable/fetch-machine-readable_salsa.py
+26
-8
26 additions, 8 deletions
misc/machine_readable/fetch-machine-readable_salsa.py
with
26 additions
and
8 deletions
misc/machine_readable/fetch-machine-readable_salsa.py
View file @
24f5e093
...
@@ -31,6 +31,17 @@ BLENDSGROUPS={ 'Debian 3D Printing Team' : '3dprinter'
...
@@ -31,6 +31,17 @@ BLENDSGROUPS={ 'Debian 3D Printing Team' : '3dprinter'
,
'
Debichem
'
:
'
debichem
'
,
'
Debichem
'
:
'
debichem
'
}
}
# Do not parse these projects which are no real packages
BLACKLIST
=
[
'
bio-linux
'
,
'
communication
'
,
'
helper-scripts
'
,
'
package_template
'
,
'
papers
'
,
'
policy
'
,
'
scripts
'
,
'
website
'
]
# missing on Salsa yet:
# missing on Salsa yet:
# pkg-exppsy
# pkg-exppsy
# Python
# Python
...
@@ -53,7 +64,8 @@ MACHINEREADABLEARCHIVE='/srv/blends.debian.org/www/_'+TDNAME+'/'+TDNAME+'.tar.xz
...
@@ -53,7 +64,8 @@ MACHINEREADABLEARCHIVE='/srv/blends.debian.org/www/_'+TDNAME+'/'+TDNAME+'.tar.xz
READMEDEBIANARCHIVE
=
'
/srv/blends.debian.org/www/_
'
+
TDNAME
+
'
/README.Debian.tar.xz
'
READMEDEBIANARCHIVE
=
'
/srv/blends.debian.org/www/_
'
+
TDNAME
+
'
/README.Debian.tar.xz
'
TARGETDIR
=
os
.
path
.
join
(
os
.
environ
[
'
HOME
'
],
TDNAME
)
TARGETDIR
=
os
.
path
.
join
(
os
.
environ
[
'
HOME
'
],
TDNAME
)
SLEEPTIME
=
1
SLEEPTIMEFILES
=
3
SLEEPTIMEPROJECTS
=
0.1
SLEEPUNTILRETRY
=
2400
SLEEPUNTILRETRY
=
2400
debug
=
1
debug
=
1
...
@@ -112,7 +124,7 @@ LAST_ACTIVITY_LENGTH=len(LAST_ACTIVITY)
...
@@ -112,7 +124,7 @@ LAST_ACTIVITY_LENGTH=len(LAST_ACTIVITY)
blends_groups
=
get_blends_groups
()
blends_groups
=
get_blends_groups
()
for
group
in
blends_groups
:
for
group
in
blends_groups
:
unchanged
,
changed
,
new
=
(
0
,
0
,
0
)
unchanged
,
changed
,
new
,
ignored
=
(
0
,
0
,
0
,
0
)
gpath
=
group
.
attributes
[
'
path
'
]
gpath
=
group
.
attributes
[
'
path
'
]
blend
=
BLENDSGROUPS
[
group
.
attributes
[
'
name
'
]]
blend
=
BLENDSGROUPS
[
group
.
attributes
[
'
name
'
]]
projects
=
group
.
projects
.
list
(
all
=
True
,
order_by
=
'
name
'
,
sort
=
'
asc
'
)
projects
=
group
.
projects
.
list
(
all
=
True
,
order_by
=
'
name
'
,
sort
=
'
asc
'
)
...
@@ -126,6 +138,12 @@ for group in blends_groups:
...
@@ -126,6 +138,12 @@ for group in blends_groups:
wait_for_salsa
()
wait_for_salsa
()
project
=
gl
.
projects
.
get
(
pr
.
attributes
[
'
id
'
])
# without this extra get repository_tree() fails
project
=
gl
.
projects
.
get
(
pr
.
attributes
[
'
id
'
])
# without this extra get repository_tree() fails
name
=
project
.
attributes
[
'
name
'
]
name
=
project
.
attributes
[
'
name
'
]
time
.
sleep
(
SLEEPTIMEPROJECTS
)
if
name
in
BLACKLIST
or
name
.
endswith
(
'
.pages.debian.net
'
):
ignored
+=
1
if
debug
>
0
:
print
(
"
Ignore project %s of blend %s.
"
%
(
name
,
blend
))
continue
namedir
=
os
.
path
.
join
(
TARGETDIR
,
name
[
0
])
namedir
=
os
.
path
.
join
(
TARGETDIR
,
name
[
0
])
if
not
os
.
path
.
exists
(
namedir
):
if
not
os
.
path
.
exists
(
namedir
):
os
.
makedirs
(
namedir
)
os
.
makedirs
(
namedir
)
...
@@ -139,11 +157,11 @@ for group in blends_groups:
...
@@ -139,11 +157,11 @@ for group in blends_groups:
if
last_activity_at
==
project
.
attributes
[
'
last_activity_at
'
]:
if
last_activity_at
==
project
.
attributes
[
'
last_activity_at
'
]:
dosomething
=
False
dosomething
=
False
unchanged
+=
1
unchanged
+=
1
if
debug
>
0
:
if
debug
>
1
:
print
(
"
Do nothing in %s since found last_activity_at %s == repository %s
"
%
(
name
,
last_activity_at
,
project
.
attributes
[
'
last_activity_at
'
]))
print
(
"
Do nothing in %s since found last_activity_at %s == repository %s
"
%
(
name
,
last_activity_at
,
project
.
attributes
[
'
last_activity_at
'
]))
else
:
else
:
if
debug
>
0
:
if
debug
>
0
:
print
(
"
Continue with %s since found last_activity_at %s != repository %s
\n
"
%
(
name
,
last_activity_at
,
project
.
attributes
[
'
last_activity_at
'
]))
print
(
"
Continue with %s since found last_activity_at %s != repository %s
"
%
(
name
,
last_activity_at
,
project
.
attributes
[
'
last_activity_at
'
]))
changed
+=
1
changed
+=
1
invcs
.
close
()
invcs
.
close
()
except
FileNotFoundError
:
except
FileNotFoundError
:
...
@@ -151,8 +169,8 @@ for group in blends_groups:
...
@@ -151,8 +169,8 @@ for group in blends_groups:
pass
pass
if
not
dosomething
:
if
not
dosomething
:
continue
continue
if
debug
>
0
:
if
debug
>
1
:
print
(
"
Opening %s failed thus w
riting.
"
%
os
.
path
.
join
(
namedir
,
name
+
'
.vcs
'
))
print
(
"
W
riting
%s
.
"
%
os
.
path
.
join
(
namedir
,
name
+
'
.vcs
'
))
with
open
(
os
.
path
.
join
(
namedir
,
name
+
'
.vcs
'
),
'
w
'
)
as
out
:
with
open
(
os
.
path
.
join
(
namedir
,
name
+
'
.vcs
'
),
'
w
'
)
as
out
:
out
.
write
(
"
Vcs-Browser: https://salsa.debian.org/%s/%s
\n
"
%
(
gpath
,
name
))
out
.
write
(
"
Vcs-Browser: https://salsa.debian.org/%s/%s
\n
"
%
(
gpath
,
name
))
out
.
write
(
"
Vcs-Git: https://salsa.debian.org/%s/%s.git
\n
"
%
(
gpath
,
name
))
out
.
write
(
"
Vcs-Git: https://salsa.debian.org/%s/%s.git
\n
"
%
(
gpath
,
name
))
...
@@ -162,10 +180,10 @@ for group in blends_groups:
...
@@ -162,10 +180,10 @@ for group in blends_groups:
# print(project)
# print(project)
# be friendly to Salsa and add some delay timr before real reading of data
# be friendly to Salsa and add some delay timr before real reading of data
# see thread around https://lists.debian.org/debian-devel/2018/07/msg00125.html
# see thread around https://lists.debian.org/debian-devel/2018/07/msg00125.html
time
.
sleep
(
SLEEPTIME
)
time
.
sleep
(
SLEEPTIME
FILES
)
if
output_metadata
(
'
debian
'
,
debianmetadata
):
if
output_metadata
(
'
debian
'
,
debianmetadata
):
output_metadata
(
'
debian/upstream
'
,
upstreammetadata
)
output_metadata
(
'
debian/upstream
'
,
upstreammetadata
)
print
(
"
In %s %i repositories changed, %i remained unchanged and %i were
new
.
"
%
(
blend
,
changed
,
unchanged
,
new
))
print
(
"
In %s %i repositories changed, %i remained unchanged
, %i were new
and %i were
ignored
.
"
%
(
blend
,
changed
,
unchanged
,
new
,
ignored
))
# os.system("tar --exclude=*README.Debian -caf %s %s" % (MACHINEREADABLEARCHIVE, TARGETDIR))
# os.system("tar --exclude=*README.Debian -caf %s %s" % (MACHINEREADABLEARCHIVE, TARGETDIR))
p
=
subprocess
.
Popen
([
'
tar
'
,
'
--exclude=*README.Debian
'
,
'
-caf
'
,
MACHINEREADABLEARCHIVE
,
TDNAME
],
cwd
=
os
.
environ
[
'
HOME
'
])
p
=
subprocess
.
Popen
([
'
tar
'
,
'
--exclude=*README.Debian
'
,
'
-caf
'
,
MACHINEREADABLEARCHIVE
,
TDNAME
],
cwd
=
os
.
environ
[
'
HOME
'
])
...
...
This diff is collapsed.
Click to expand it.