Commit 831f08df authored by Antonio Terceiro's avatar Antonio Terceiro

compress artifacts (except log) in a single tarball

parent 9b093e4b
......@@ -77,6 +77,7 @@ generate_package_run() {
duration_human=$(seconds_to_human_time "$duration")
extract_latest_autopkgtest
generate_package_run_log_file > "$log_file"
report_status "$pkg" "$status" "($duration_human)"
......@@ -95,6 +96,16 @@ generate_package_run() {
fi
ln -s "../../../../..${adtresult#$debci_data_basedir}" \
"${status_dir}/latest-autopkgtest"
compress_artifacts "$adtresult"
}
extract_latest_autopkgtest() {
mkdir -p ${tmp_dir}/latest-autopkgtest
local latest_artifacts="${status_dir}/latest-autopkgtest/artifacts.tar.gz"
if [ -f "$latest_artifacts" ]; then
tar -xaf "$latest_artifacts" -C "${tmp_dir}/latest-autopkgtest"
fi
}
# write package test log to stdout (gets redirected to $log_file)
......@@ -171,9 +182,9 @@ generate_package_run_testbed_packages() {
return 0
fi
if [ -f "${status_dir}/latest-autopkgtest/testbed-packages" ]; then
if [ -f "${tmp_dir}/latest-autopkgtest/testbed-packages" ]; then
if ! diff -u \
--label previous-run/testbed-packages "${status_dir}/latest-autopkgtest/testbed-packages" \
--label previous-run/testbed-packages "${tmp_dir}/latest-autopkgtest/testbed-packages" \
--label current-run/testbed-packages "${adtresult}/testbed-packages" \
> "${tmp_dir}/base.diff"; then
banner "Change in the base system since last run"
......@@ -193,8 +204,8 @@ generate_package_run_dependencies() {
if [ -n "$deps" ]; then
banner "Test dependencies"
echo "$deps"
if [ -d ${status_dir}/latest-autopkgtest ]; then
cat ${status_dir}/latest-autopkgtest/*t-*-packages | sort -u > $tmp_dir/last_test_packages
if [ -d ${tmp_dir}/latest-autopkgtest ]; then
cat ${tmp_dir}/latest-autopkgtest/*t-*-packages 2>/dev/null | sort -u > $tmp_dir/last_test_packages
if [ -s "$tmp_dir/last_test_packages" ]; then
if ! echo "$deps" | diff -u --label last-run/test-packages "$tmp_dir/last_test_packages" --label current-run/test-packages - > "$tmp_dir/test-packages.diff"; then
banner "Change in test packages for $pkg since last test run"
......@@ -333,6 +344,15 @@ EOF
ln -sf "${date}.json" "${debci_status_dir}/status.json"
}
compress_artifacts() {
local resultdir="$1"
(
cd $resultdir
tar -caf artifacts.tar.gz --exclude=log.gz *
tar -taf artifacts.tar.gz | xargs rm -f
)
}
generate_history() {
debci-generate-history "${debci_status_dir}" "${debci_status_dir}/history.json"
}
......
#!/bin/sh
set -eu
datadir="$1"
if [ -d "$datadir/autopkgtest" ]; then
echo "Compressing artifacts; this might take a while ..."
compressed=0
find "$datadir/autopkgtest" -name exitcode | while read exitcode; do
test_run_dir=$(dirname $exitcode)
test_run_id=$(basename $test_run_dir)
(
cd $test_run_dir
tar -caf artifacts.tar.gz --exclude='log.gz' *
tar -taf artifacts.tar.gz | xargs rm -f
)
compressed=$(($compressed + 1))
if [ $(($compressed % 100)) -eq 0 ]; then
echo "$compressed test runs compressed"
fi
done
fi
......@@ -7,6 +7,9 @@ debci (1.3) UNRELEASED; urgency=medium
- use one tmp_dir per package/suite/architecture
* bin/debci-generate-index
- use one tmp_dir per package/suite/architecture
- compress artifacts (except log) in a single tarball so that instead of N
files per test run, we only keep 2 files per test run: log.gz and
artifacts.tar.gz. This should significantly reduce the usage of inodes.
-- Antonio Terceiro <terceiro@debian.org> Fri, 03 Jun 2016 14:45:08 -0300
......
......@@ -53,7 +53,7 @@
<td><a href="<%= @artifacts_url_base %>/<%= @package_dir %>/<%= test.run_id %>/artifacts.tar.gz">artifacts</a></td>
<% else %>
<td><a href="/<%= @packages_dir %>/<%= @package_dir %>/<%= test.run_id %>.autopkgtest.log.gz">test log</a></td>
<td><a href="/<%= @autopkgtest_dir %>/<%= @package_dir %>/<%= test.run_id %>">artifacts</a></td>
<td><a href="/<%= @autopkgtest_dir %>/<%= @package_dir %>/<%= test.run_id %>/artifacts.tar.gz">artifacts</a></td>
<% end %>
</tr>
<% end %>
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment