...
 
Commits (5)
......@@ -2302,7 +2302,7 @@ class TeamMembership(models.Model):
:type package_name: :class:`PackageName` or :class:`str`
"""
if not isinstance(package_name, PackageName):
package_name = PackageName.objects.get(package_name)
package_name = get_or_none(PackageName, name=package_name)
if self.muted:
return True
try:
......@@ -2397,7 +2397,7 @@ class TeamMembership(models.Model):
:class:`Keyword` instances.
"""
if not isinstance(package_name, PackageName):
package_name = PackageName.objects.get(package_name)
package_name = get_or_none(PackageName, name=package_name)
try:
membership_package_specifics = \
......
......@@ -780,7 +780,11 @@ class PackageUtilsTests(SimpleTestCase):
def test_html_package_list(self):
"""Tests the output of html_package_list function"""
list_of_packages = ['dummy-package', 'other-dummy-package']
list_of_packages = [
'dummy-package',
'other-dummy-package',
'last-dummy-package/amd64',
]
output = html_package_list(list_of_packages)
......@@ -794,11 +798,17 @@ class PackageUtilsTests(SimpleTestCase):
'other-dummy-package',
)
third_url = '<a href="%s">%s</a>/amd64' % (
package_url('last-dummy-package'),
'last-dummy-package',
)
self.assertEqual(
output,
"%s, %s" % (
"%s, %s, %s" % (
first_url,
second_url,
third_url,
),
)
......
......@@ -706,8 +706,13 @@ class AptCache(object):
def html_package_list(packages):
packages_html = []
for package in packages:
html = '<a href="{}">{}</a>'.format(
package_url(package), package)
if "/" in package:
(source_package_name, remain) = package.split("/", 1)
remain = "/%s" % (remain,)
else:
(source_package_name, remain) = (package, "")
html = '<a href="{}">{}</a>{}'.format(
package_url(source_package_name), source_package_name, remain)
packages_html.append(html)
return ', '.join(packages_html)
......@@ -47,7 +47,7 @@ class SkipMessage(Exception):
The mail is then silently dropped."""
def _get_logdata(msg, package, keyword):
def _get_logdata(msg, package, keyword, team):
return {
'from': extract_email_address_from_header(msg.get('From', '')),
'msgid': msg.get('Message-ID', 'no-msgid-present@localhost'),
......@@ -56,6 +56,16 @@ def _get_logdata(msg, package, keyword):
}
def _must_discard(msg, logdata):
# Check loop
dispatch_email = 'dispatch@{}'.format(DISTRO_TRACKER_FQDN)
if dispatch_email in msg.get_all('X-Loop', ()):
# Bad X-Loop, discard the message
logger.info('dispatch :: discarded %(msgid)s due to X-Loop', logdata)
return True
return False
def process(msg, package=None, keyword=None):
"""
Dispatches received messages by identifying where they should
......@@ -68,7 +78,7 @@ def process(msg, package=None, keyword=None):
:param str keyword: The keyword under which the message must be dispatched.
"""
logdata = _get_logdata(msg, package, keyword)
logdata = _get_logdata(msg, package, keyword, None)
logger.info("dispatch :: received from %(from)s :: %(msgid)s",
logdata)
try:
......@@ -82,6 +92,9 @@ def process(msg, package=None, keyword=None):
logdata)
return
if _must_discard(msg, logdata):
return
if isinstance(package, (list, set)):
for pkg in package:
forward(msg, pkg, keyword)
......@@ -101,16 +114,10 @@ def forward(msg, package, keyword):
:param str keyword: The keyword under which the message must be forwarded.
"""
logdata = _get_logdata(msg, package, keyword)
logdata = _get_logdata(msg, package, keyword, None)
logger.info("dispatch :: forward to %(package)s %(keyword)s :: %(msgid)s",
logdata)
# Check loop
dispatch_email = 'dispatch@{}'.format(DISTRO_TRACKER_FQDN)
if dispatch_email in msg.get_all('X-Loop', ()):
# Bad X-Loop, discard the message
logger.info('dispatch :: discarded %(msgid)s due to X-Loop', logdata)
return
# Default keywords require special approvement
if keyword == 'default' and not approved_default(msg):
......@@ -119,11 +126,44 @@ def forward(msg, package, keyword):
return
# Now send the message to subscribers
add_new_headers(msg, package, keyword)
add_new_headers(msg, package_name=package, keyword=keyword)
send_to_subscribers(msg, package, keyword)
send_to_teams(msg, package, keyword)
def process_for_team(msg, team_slug):
logdata = _get_logdata(msg, None, None, team_slug)
logger.info("dispatch :: received for team %(team)s "
"from %(from)s :: %(msgid)s", logdata)
if _must_discard(msg, logdata):
return
try:
team = Team.objects.get(slug=team_slug)
except Team.DoesNotExist:
logger.info("dispatch :: discarded %(msgid)s for team %(team)s "
"since team doesn't exist", logdata)
return
package, keyword = classify_message(msg)
if package:
logger.info("dispatch :: discarded %(msgid)s for team %(team)s "
"as an automatic mail", logdata)
return
forward_to_team(msg, team)
def forward_to_team(msg, team):
logdata = _get_logdata(msg, None, None, team.slug)
logger.info("dispatch :: forward to team %(team)s :: %(msgid)s",
logdata)
add_new_headers(msg, keyword="contact", team=team.slug)
send_to_team(msg, team, keyword="contact")
def classify_message(msg, package=None, keyword=None):
"""
Analyzes a message to identify what package it is about and
......@@ -171,7 +211,8 @@ def approved_default(msg):
return False
def add_new_headers(received_message, package_name, keyword):
def add_new_headers(received_message, package_name=None, keyword=None,
team=None):
"""
The function adds new distro-tracker specific headers to the received
message. This is used before forwarding the message to subscribers.
......@@ -193,17 +234,24 @@ def add_new_headers(received_message, package_name, keyword):
"""
new_headers = [
('X-Loop', 'dispatch@{}'.format(DISTRO_TRACKER_FQDN)),
('X-Distro-Tracker-Package', package_name),
('X-Distro-Tracker-Keyword', keyword),
('List-Id', '<{}.{}>'.format(package_name, DISTRO_TRACKER_FQDN)),
]
if keyword:
new_headers.append(('X-Distro-Tracker-Keyword', keyword))
if package_name:
new_headers.extend([
('X-Distro-Tracker-Package', package_name),
('List-Id', '<{}.{}>'.format(package_name, DISTRO_TRACKER_FQDN)),
])
if team:
new_headers.append(('X-Distro-Tracker-Team', team))
extra_vendor_headers, implemented = vendor.call(
'add_new_headers', received_message, package_name, keyword)
'add_new_headers', received_message, package_name, keyword, team)
if implemented:
new_headers.extend(extra_vendor_headers)
add_headers(received_message, new_headers)
for header_name, header_value in new_headers:
received_message[header_name] = header_value
def add_direct_subscription_headers(received_message, package_name, keyword):
......@@ -218,27 +266,16 @@ def add_direct_subscription_headers(received_message, package_name, keyword):
control_email=DISTRO_TRACKER_CONTROL_EMAIL,
package=package_name)),
]
add_headers(received_message, new_headers)
for header_name, header_value in new_headers:
received_message[header_name] = header_value
def add_team_membership_headers(received_message, package_name, keyword, team):
def add_team_membership_headers(received_message, keyword, team):
"""
The function adds headers to the received message which are specific for
messages to be sent to users that are members of a team.
"""
new_headers = [
('X-Distro-Tracker-Team', team.slug),
]
add_headers(received_message, new_headers)
def add_headers(message, new_headers):
"""
Adds the given headers to the given message. This used to
contain more code for Python 2 compat.
"""
for header_name, header_value in new_headers:
message[header_name] = header_value
received_message['X-Distro-Tracker-Team'] = team.slug
def send_to_teams(received_message, package_name, keyword):
......@@ -270,25 +307,30 @@ def send_to_teams(received_message, package_name, keyword):
teams = Team.objects.filter(packages=package)
teams = teams.prefetch_related('team_membership_set')
for team in teams:
send_to_team(received_message, team, keyword, package.name)
def send_to_team(received_message, team, keyword, package_name=None):
keyword = get_or_none(Keyword, name=keyword)
package = get_or_none(PackageName, name=package_name)
date = timezone.now().date()
messages_to_send = []
for team in teams:
logger.info('dispatch :: sending to team %s', team.slug)
team_message = deepcopy(received_message)
add_team_membership_headers(
team_message, package_name, keyword.name, team)
# Send the message to each member of the team
for membership in team.team_membership_set.all():
# Do not send messages to muted memberships
if membership.is_muted(package):
continue
# Do not send the message if the user has disabled the keyword
if keyword not in membership.get_keywords(package):
continue
messages_to_send.append(prepare_message(
team_message, membership.user_email.email, date))
logger.info('dispatch :: sending to team %s', team.slug)
team_message = deepcopy(received_message)
add_team_membership_headers(team_message, keyword.name, team)
# Send the message to each member of the team
for membership in team.team_membership_set.all():
# Do not send messages to muted memberships
if membership.is_muted(package):
continue
# Do not send the message if the user has disabled the keyword
if keyword not in membership.get_keywords(package):
continue
messages_to_send.append(prepare_message(
team_message, membership.user_email.email, date))
send_messages(messages_to_send, date)
......
......@@ -171,7 +171,7 @@ class MailProcessor(object):
keyword=keyword)
def handle_team(self, team):
pass
distro_tracker.mail.dispatch.process_for_team(self.message, team)
def run_mail_processor(mail_path, log_failure=False):
......
......@@ -105,6 +105,12 @@ class DispatchTestHelperMixin(object):
keyword=keyword,
)
def run_dispatch_to_team(self, team=None):
"""
Starts the dispatch process for a team
"""
dispatch.process_for_team(self.message, team)
def run_forward(self, package=None, keyword=None):
"""
Starts the forward process.
......@@ -349,7 +355,7 @@ class DispatchBaseTest(TestCase, DispatchTestHelperMixin):
self.set_header('X-Loop', 'dispatch@' + DISTRO_TRACKER_FQDN)
self.subscribe_user_to_package('user@domain.com', self.package_name)
self.run_forward()
self.run_dispatch()
self.assertEqual(len(mail.outbox), 0)
......@@ -821,3 +827,68 @@ class DispatchToTeamsTests(DispatchTestHelperMixin, TestCase):
self.run_forward()
self.assertEqual(0, len(mail.outbox))
def test_dispatch_to_team_manual_mail(self):
"""
Tests that the message is forwarded to team members.
"""
self.run_dispatch_to_team(self.team.slug)
self.assert_message_forwarded_to(self.user.main_email)
def test_dispatch_to_team_has_correct_headers(self):
"""
Tests that the message forwarded to team members has the required
headers.
"""
self.run_dispatch_to_team(self.team.slug)
headers = [
('X-Loop', 'dispatch@{}'.format(DISTRO_TRACKER_FQDN)),
('X-Distro-Tracker-Team', self.team.slug),
('X-Distro-Tracker-Keyword', 'contact'),
]
self.assert_all_headers_found(headers)
def test_dispatch_to_non_existing_team(self):
"""
Tests that nothing is forwarded when the team doesn't exist.
"""
self.run_dispatch_to_team('does-not-exist')
self.assertEqual(0, len(mail.outbox))
def test_dispatch_to_team_discards_automatic_emails(self):
"""
Tests that nothing is forwarded when the email is an automatic
message that we should have received by another channel already.
"""
self.add_header('X-Distro-Tracker-Package', 'foobar')
self.run_dispatch_to_team(self.team.slug)
self.assertEqual(0, len(mail.outbox))
def test_dispatch_to_team_respects_the_contact_keyword_setting(self):
"""
Tests that a team message is not forwarded to a team member that does
not have the "contact" keyword set.
"""
email = self.user.main_email
membership = self.team.team_membership_set.get(user_email__email=email)
membership.set_membership_keywords(
[k.name for k in Keyword.objects.exclude(name='contact')])
self.run_dispatch_to_team(self.team.slug)
self.assertEqual(0, len(mail.outbox))
def test_dispatch_to_team_xloop_already_set(self):
"""
Tests that the message is dropped when the X-Loop header is already
set.
"""
self.set_header('X-Loop', 'somevalue')
self.set_header('X-Loop', 'dispatch@' + DISTRO_TRACKER_FQDN)
self.run_dispatch_to_team(self.team.slug)
self.assertEqual(len(mail.outbox), 0)
......@@ -148,7 +148,7 @@ def classify_message(msg, package, keyword):
return (package, keyword)
def add_new_headers(received_message, package_name, keyword):
def add_new_headers(received_message, package_name, keyword, team):
"""
Debian adds the following new headers:
- X-Debian-Package
......@@ -165,11 +165,15 @@ def add_new_headers(received_message, package_name, keyword):
:type keyword: string
"""
new_headers = [
('X-Debian-Package', package_name),
('X-Debian', 'tracker.debian.org'),
('X-PTS-Package', package_name), # for compat with old PTS
('X-PTS-Keyword', keyword), # for compat with old PTS
]
if package_name:
new_headers.append(('X-Debian-Package', package_name))
new_headers.append(
('X-PTS-Package', package_name)) # for compat with old PTS
if keyword:
new_headers.append(
('X-PTS-Keyword', keyword)) # for compat with old PTS
return new_headers
......
......@@ -852,6 +852,30 @@ class UpdateExcusesTask(BaseTask):
return (source['item-name'], {'age': age, 'limit': limit})
@staticmethod
def _make_excuses_check_dependencies(source):
"""Checks the dependencies of the package (blocked-by and
migrate-after) and returns a list to display."""
addendum = []
if 'dependencies' in source:
blocked_by = source['dependencies'].get('blocked-by', [])
after = source['dependencies'].get('migrate-after', [])
after = [
element
for element in after
if element not in blocked_by
]
addendum.append("Blocked by: %s" % (
html_package_list(blocked_by),
))
addendum.append("Migrates after: %s" % (
html_package_list(after),
))
return addendum
@staticmethod
def _make_excuses_check_verdict(source):
"""Checks the migration policy verdict of the package and builds an
excuses message depending on the result."""
......@@ -861,20 +885,10 @@ class UpdateExcusesTask(BaseTask):
if 'migration-policy-verdict' in source:
verdict = source['migration-policy-verdict']
if verdict == 'REJECTED_BLOCKED_BY_ANOTHER_ITEM':
addendum.append((
"Migration status: Blocked. Can't migrate due to a "
"non-migrable dependency. Check status below."
))
if 'dependencies' in source:
blocked_by = source['dependencies'].get('blocked-by', [])
after = source['dependencies'].get('migrate-after', [])
deps = list({
element
for element in blocked_by + after
})
addendum.append("Blocked by: %s" % (
html_package_list(deps),
))
addendum.append("Migration status: Blocked. Can't migrate "
"due to a non-migrable dependency. Check "
"status below."
)
return addendum
......@@ -916,6 +930,7 @@ class UpdateExcusesTask(BaseTask):
addendum = []
addendum.extend(self._make_excuses_check_verdict(source))
addendum.extend(self._make_excuses_check_dependencies(source))
addendum.extend(self._make_excuses_check_age(source))
excuses = addendum + excuses
......
......@@ -27,7 +27,7 @@ def classify_message(msg, package=None, keyword=None):
return (package, keyword)
def add_new_headers(received_message, package_name, keyword):
def add_new_headers(received_message, package_name, keyword, team):
"""
The function should return a list of two-tuples (header_name, header_value)
which are extra headers that should be added to package messages before
......@@ -40,10 +40,14 @@ def add_new_headers(received_message, package_name, keyword):
:param package_name: The name of the package for which the message was
intended
:type package_name: string
:type package_name: str
:param keyword: The keyword with which the message is tagged.
:type keyword: string
:type keyword: str
:param team: The team slug for a message sent to the team or received
through the team.
:type team: str
"""
pass
......