Treat autopkgtest exit status 8 as a neutral result

Closes: #901846
parent 3f477eb9
......@@ -74,11 +74,13 @@ if ENV["DEBCI_FAKE_COMMAND"]
end
else
log.puts "Not really running anything .."
log.puts "This has 80% chance of passing, 10% of failing, and 10% of tmpfailing"
log.puts "This has 70% chance of passing, 10% of all skipped, 10% of failing, and 10% of tmpfailing"
r = nil
case ENV['DEBCI_FAKE_RESULT']
when 'pass'
r = 0
when 'neutral'
r = 7
when 'fail'
r = 8
when 'tmpfail'
......@@ -88,9 +90,12 @@ else
end
case r
when 0..7
when 0..6
log.puts "Passed :-)"
rc = 0
when 7
log.puts "Neutral :-|"
rc = 8
when 8
log.puts "Failed :-("
rc = 4
......
......@@ -183,8 +183,8 @@ exitcode_to_statusmsg()
message='Tests failed, and at least one test skipped'
;;
8)
status=fail
message='No tests in this package'
status=neutral
message='No tests in this package or all skipped'
;;
12)
status=fail
......@@ -340,9 +340,10 @@ EOF
generate_status_entry() {
pass=$(grep -l '"status":\s*"pass",' ${debci_packages_dir}/*/*/latest.json | wc -l)
neutral=$(grep -l '"status":\s*"neutral",' ${debci_packages_dir}/*/*/latest.json | wc -l)
fail=$(grep -l '"status":\s*"fail",' ${debci_packages_dir}/*/*/latest.json | wc -l)
tmpfail=$(grep -l '"status":\s*"tmpfail",' ${debci_packages_dir}/*/*/latest.json | wc -l)
total=$(($pass + $fail + $tmpfail))
total=$(($pass + $neutral + $fail + $tmpfail))
date="$(date +%Y-%m-%dT%H:%M:%S)"
path="$(echo "$date" | tr 'T-' '/')"
mkdir -p "$(dirname "${debci_status_dir}/${path}")"
......@@ -350,6 +351,7 @@ generate_status_entry() {
{
"date": "$date",
"pass": $pass,
"neutral": $neutral,
"fail": $fail,
"tmpfail": $tmpfail,
"total": $total
......
......@@ -97,9 +97,12 @@ do_request() {
0|2)
debci_suite="$suite" report_status $pkg pass
;;
4|6|8|12)
4|6|12)
debci_suite="$suite" report_status $pkg fail
;;
8)
debci_suite="$suite" report_status $pkg neutral
;;
*)
debci_suite="$suite" report_status $pkg tmpfail
;;
......
......@@ -142,6 +142,7 @@ module Debci
ICONS = {
pass: 'thumbs-up',
neutral: 'minus-circle',
fail: 'thumbs-down',
fail_passed_never: ['thumbs-down', 'ban'],
fail_passed_current: ['thumbs-down', 'bolt'],
......
......@@ -29,9 +29,9 @@
<% packages.each do |package| %>
<tr>
<td><a href="/packages/<%= @prefix %>/<%= package.name %>"><%= package.name %></a></td>
<% if package.failures.any? %>
<% if package.fail_or_neutral.any? %>
<td>
<% package.failures.each do |s| %>
<% package.fail_or_neutral.each do |s| %>
<a href='<%= "/packages/#{@prefix}/#{package.name}/#{s.suite}/#{s.architecture}"%>'>
<%= icon(s.extended_status) %>
<%= s.version %> <%= s.status.upcase %> <%= s.suite %>/<%= s.architecture %> <%= s.failmsg && '(%s)' % s.failmsg %>
......
......@@ -16,6 +16,7 @@
<tr>
<th>Date</th>
<th>Pass</th>
<th>Neutral</th>
<th>Fail</th>
<th>Temporary failure</th>
<th>Pass percentage</th>
......@@ -24,6 +25,7 @@
<tr>
<td><%= entry['date'] %></td>
<td><%= entry['pass'] %></td>
<td><%= entry['neutral'] %></td>
<td><%= entry['fail'] %></td>
<td><%= entry['tmpfail'] %></td>
<td><%= 100 * entry['pass'].to_f / entry['total'].to_f %>%</td>
......
......@@ -41,11 +41,16 @@ module Debci
repository.news_for(self)
end
# Returns an Array of statuses where this package is temporarily failing.
# Returns an Array of statuses where this package is failing.
def failures
status.flatten.select { |p| p.status == :fail }
end
# Returns an Array of statuses where this package is failing or neutral.
def fail_or_neutral
status.flatten.select { |p| p.status == :fail or p.status == :neutral }
end
# Returns an Array of statuses where this package is temporarily failing. If
def tmpfail
status.flatten.select { |p| p.status == :tmpfail }
......
......@@ -15,6 +15,10 @@ module Debci
[
[:fail, :pass],
[:pass, :fail],
[:fail, :neutral],
[:pass, :neutral],
[:neutral, :pass],
[:neutral, :fail],
].include?([status, previous_status])
end
......@@ -22,6 +26,7 @@ module Debci
{
:pass => "Pass",
:fail => "Fail",
:neutral => "No tests or all skipped",
:tmpfail => "Temporary failure",
:no_test_data => "No test data",
}.fetch(status, "Unknown")
......@@ -78,7 +83,7 @@ module Debci
# Returns a headline for this status object, to be used as a short
# description of the event it represents
def headline
msg = "#{package} #{version} #{status.upcase}ED on #{suite}/#{architecture}"
msg = "#{package} #{version} #{status.upcase} on #{suite}/#{architecture}"
if status == :fail && failmsg
msg += " (#{failmsg})"
end
......@@ -88,7 +93,7 @@ module Debci
# A longer version of the headline
# for a new failure, include whether this version previously passed
def description
msg = "The tests for #{package}, version #{version}, #{status.upcase}ED on #{suite}/#{architecture} but have previously #{previous_status.upcase}ED"
msg = "The tests for #{package} (version #{version}) resulted in #{status.upcase} on #{suite}/#{architecture}. Previously it was #{previous_status.upcase}"
msg += case extended_status
when :fail_passed_current
" for the current version."
......
......@@ -103,6 +103,9 @@ report_status() {
pass)
color=2
;;
neutral)
color=8
;;
fail)
color=1
;;
......
......@@ -23,6 +23,7 @@ jQuery(function($) {
console.log('skipping ' + platform + '...')
} else {
var pass = [];
var neutral = [];
var fail = [];
var tmpfail = [];
var pass_percentage = [];
......@@ -31,6 +32,7 @@ jQuery(function($) {
$.each(data, function(index, entry) {
var date = Date.parse(entry.date);
pass.push([date, entry.pass]);
neutral.push([date, entry.neutral]);
fail.push([date, entry.fail]);
tmpfail.push([date, entry.tmpfail || 0]);
pass_percentage.push([date, entry.pass / entry.total]);
......@@ -45,6 +47,10 @@ jQuery(function($) {
label: "Pass",
data: pass
},
{
label: "Neutral",
data: neutral
},
{
label: "Fail",
data: fail
......@@ -64,7 +70,7 @@ jQuery(function($) {
steps: false,
}
},
colors: [ '#8ae234', '#ef2929', '#ffd862' ],
colors: [ '#8ae234', '#8080ff', '#ef2929', '#ffd862' ],
legend: {
show: true,
backgroundOpacity: 0.2,
......
......@@ -48,6 +48,10 @@
color: #8ae234;
}
.neutral {
color: #8080ff;
}
.fail {
color: #ef2929;
}
......
......@@ -49,7 +49,12 @@ describe Debci::Package do
fail_status.architecture = 'i386'
fail_status.status = :fail
status << tmpfail_status << pass_status << fail_status
neutral_status = Debci::Status.new
neutral_status.suite = 'unstable'
neutral_status.architecture = 'i386'
neutral_status.status = :neutral
status << tmpfail_status << pass_status << fail_status << neutral_status
allow(repository).to receive(:status_for).with(package).and_return(status)
expect(package.status).to eq(status)
......@@ -57,6 +62,8 @@ describe Debci::Package do
expect(package.tmpfail).to eq([tmpfail_status])
expect(package.failures).to eq([fail_status])
expect(package.fail_or_neutral).to eq([fail_status, neutral_status])
end
it 'converts to string' do
......
......@@ -77,6 +77,26 @@ describe Debci::Status do
expect(status).to be_newsworthy
end
it 'is newsworthy when going from pass to neutral' do
status = status_with(status: :neutral, previous_status: :pass)
expect(status).to be_newsworthy
end
it 'is newsworthy when going from neutral to pass' do
status = status_with(status: :pass, previous_status: :neutral)
expect(status).to be_newsworthy
end
it 'is newsworthy when going from neutral to fail' do
status = status_with(status: :fail, previous_status: :neutral)
expect(status).to be_newsworthy
end
it 'is newsworthy when going from fail to neutral' do
status = status_with(status: :neutral, previous_status: :fail)
expect(status).to be_newsworthy
end
it 'is not newsworthy when keeps passing' do
status = status_with(status: :pass, previous_status: :pass)
expect(status).to_not be_newsworthy
......@@ -86,6 +106,11 @@ describe Debci::Status do
status = status_with(status: :fail, previous_status: :fail)
expect(status).to_not be_newsworthy
end
it 'is not newsworthiness when remains neutral' do
status = status_with(status: :neutral, previous_status: :neutral)
expect(status).to_not be_newsworthy
end
end
context 'headline' do
......@@ -102,7 +127,7 @@ describe Debci::Status do
end
context 'title' do
[:pass, :fail, :tmpfail, :no_test_data, :INVALID].each do |s|
[:pass, :fail, :neutral, :tmpfail, :no_test_data, :INVALID].each do |s|
it "should have a title for #{s}" do
expect(status_with(status: s).title).to be_a(String)
end
......
......@@ -8,13 +8,13 @@ test_no_new_is_good_news() {
result_pass debci test --quiet foobar
debci update --quiet
news_count=$(grep -c "<title>foobar.*ED" "$debci_data_basedir/feeds/f/foobar.xml")
news_count=$(grep -c "<title>foobar .* on " "$debci_data_basedir/feeds/f/foobar.xml")
assertEquals 0 "$news_count"
}
# XXX the tests below are inherently racy; every pair of tests will most
# probably get the same timestamp as our resolution is 1 second; but since the
# behavior with regards to status items being being newsworthy is properly unit
# behavior with regards to status items being newsworthy is properly unit
# tested in the Ruby code, either FAIL/PASS or PASS/FAIL, in any order, is good
# enough here.
......@@ -23,7 +23,7 @@ test_notify_on_fail() {
result_fail debci test --quiet foobar
debci update --quiet
news_count=$(grep -c '<title>foobar.*ED' "$debci_data_basedir/feeds/f/foobar.xml")
news_count=$(grep -c '<title>foobar .* on ' "$debci_data_basedir/feeds/f/foobar.xml")
assertEquals 1 "$news_count"
}
......@@ -34,10 +34,10 @@ test_system_wide_feed() {
result_pass debci test --quiet bazqux
debci update --quiet
foobar_news=$(grep -c '<title>foobar.*ED' "$debci_data_basedir/feeds/all-packages.xml")
foobar_news=$(grep -c '<title>foobar .* on ' "$debci_data_basedir/feeds/all-packages.xml")
assertEquals 1 "$foobar_news"
bazqux_news=$(grep -c '<title>bazqux.*ED' "$debci_data_basedir/feeds/all-packages.xml")
bazqux_news=$(grep -c '<title>bazqux .* on ' "$debci_data_basedir/feeds/all-packages.xml")
assertEquals 1 "$bazqux_news"
}
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment