Add more FIXMEs and TODO items

Some were thought up during Outreachy Round 16 and
others were suggested earlier by Unit193.
parent e16a66e7
......@@ -109,6 +109,7 @@ sources.stamp: sources.list apt FORCE | error-logs
sources.links sources.new sources.log 2>&1 | tee $|/$@.error.log
touch $@
# FIXME: integrate sources.log.warnings with the error log reporting
sources.log.warnings: sources.log sources.stamp
cp --force $< $@.tmp
sed -i '/^WARNING: Retrying /s/WARNING/INFO/' $@.tmp
......@@ -173,6 +174,7 @@ distrowatch: wiki.txt
sources.patches.dd-list: sources.patches debian.sources.list debian.apt | error-logs
sources-patches-to-dd-list $< ../../etc/debian.apt.deriv.conf $@ 2>&1 | tee $|/$@.error.log
# FIXME: integrate check-* with the error log reporting
check: check-package-list check-sources-list
check-package-list: sources.list apt | error-logs
......
......@@ -53,6 +53,7 @@ for deriv in derivatives:
for sha1 in set(files)-set(new_files):
files[sha1]['obsolete'] = True
# FIXME: write out data files atomically
f = open(sources_files,'w')
yaml.dump(files, f, Dumper=yaml.CSafeDumper)
f.close()
......@@ -60,6 +60,7 @@ for deriv in derivatives:
if hash not in entry_hashes:
entry_hashes.append(hash)
# FIXME: write out data files atomically
with open(sources_new, 'w') as f:
yaml.dump(all_entries, f, Dumper=yaml.CSafeDumper)
......@@ -82,8 +83,10 @@ for name, versions in old_data.items():
obsolete.setdefault(name, {}).setdefault(version, {})
obsolete[name][version] = missing_hashes
# FIXME: write out data files atomically
with open(sources_new + '.obsolete', 'w') as f:
yaml.dump(obsolete, f, Dumper=yaml.CSafeDumper)
# FIXME: write out data files atomically
with open(sources_new + '.obsolete-no-hashes', 'w') as f:
yaml.dump(obsolete_no_hashes, f, Dumper=yaml.CSafeDumper)
......@@ -67,11 +67,13 @@ patches = old_patches + new_patches
# Need to exclude Ubuntu since the PTS has other links to its patches
debian_names = set([patch['debian_name'] for patch in patches if not patch.get('obsolete', False) and patch['deriv'] != 'Ubuntu'])
patches_pts = dict((name, '/'.join((url,shortslug(name),name,''))) for name in debian_names)
# FIXME: write out data files atomically
f = open(sources_patches_pts,'w')
yaml.dump(patches_pts, f, Dumper=yaml.CSafeDumper)
f.close()
# The general patch metadata
# FIXME: write out data files atomically
f = open(sources_patches,'w')
yaml.dump(patches, f, Dumper=yaml.CSafeDumper)
f.close()
......@@ -10,6 +10,8 @@
# check-package-list <sources.list file> <apt dir> <log file>
# FIXME: check for missing secure hashes in PGP sigs, Release, Packages, Sources
# FIXME: update apt repository checks against the format documentation
# https://wiki.debian.org/DebianRepository/Format
import re
import os
......@@ -302,6 +304,8 @@ packages_binary_packages.sort(key=lambda pkg: (pkg[0], pkg[1]))
# Check if the package name lists match
if sources_source_packages != packages_source_packages:
# FIXME: clarify the mismatched packages message
# https://lists.debian.org/msgid-search/871s71d6a6.fsf@boum.org
logging.warning('mismatched source packages in Sources/Packages')
dump_list('sources_source_packages', sources_source_packages)
dump_list('packages_source_packages', packages_source_packages)
......@@ -315,6 +319,8 @@ else:
remove('packages_source_packages')
remove('diff_source_packages')
if sources_binary_packages != packages_binary_packages:
# FIXME: clarify the mismatched packages message
# https://lists.debian.org/msgid-search/871s71d6a6.fsf@boum.org
logging.warning('mismatched binary packages in Sources/Packages')
dump_list('sources_binary_packages', sources_binary_packages)
dump_list('packages_binary_packages', packages_binary_packages)
......
......@@ -9,6 +9,10 @@
# Usage:
# check-sources-list <sources.list file> <reprepro.config file> <log file>
# FIXME: Split this into checking sources.list itself and against reprepro and Release files
# FIXME: update apt repository checks against the format documentation
# https://wiki.debian.org/DebianRepository/Format
import os
import sys
import glob
......@@ -101,6 +105,7 @@ for entry in sources_entries:
if entry.type == 'deb-src' and 'arch' in entry.options:
logging.warning('arch option in deb-src entry: %s', entry.text)
if entry.type == 'deb' and 'arch' in entry.options and entry.options['arch'] and [arch for arch in entry.options['arch'] if arch not in archs]:
# FIXME: highlight which architectures are new
logging.info('new architecture(s): %s', entry.text)
if entry.uri.scheme not in ('ftp', 'http', 'https'):
logging.warning('unknown or inappropriate uri scheme: %s', entry.text)
......@@ -136,8 +141,10 @@ for uri, dists in repos.items():
elif 'deb' not in types:
logging.warning('deb-src without matching deb entry: %s %s %s', uri, dist, ','.join(types['deb-src']))
elif 'deb-src' not in types:
# FIXME: print the component name instead of the architecture list
logging.warning('deb without matching deb-src entry: %s %s %s', uri, dist, ','.join(types['deb']))
elif set.union(*types['deb'].values()) != set.union(*types['deb-src'].values()):
# FIXME: ignore d-i subcomponents, their source is in the main component
logging.warning('mismatched deb/deb-src entries: %s %s %s != %s', uri, dist,
','.join(set.union(*types['deb'].values())), ','.join(set.union(*types['deb-src'].values())))
......@@ -162,6 +169,7 @@ def check_arches(found_url, components, dist, arch_type, arch):
missing_elements[arch_type][arch][dist] = missing_comps
def check_type(found_url, source_for_check, link):
# FIXME: determine archive types from input data
archive_type = ['deb']
if 'source' in source_for_check['Architectures']:
archive_type.append('deb-src')
......@@ -218,6 +226,7 @@ try:
names, pretty_name = preparation_for_checking(derivative)
conf_url = os.path.join(derivative['RepoUrl'], 'conf/distributions')
if not any(name in repos[derivative['RepoUrl']] for name in names):
# FIXME: add a mechanism to ignore Ubuntu related suites
logging.warning('Distribution %s is missing from the sources.list but is present in the reprepro config %s',
pretty_name, conf_url)
continue
......@@ -268,8 +277,11 @@ for item in acquire.items:
continue
rel = Release(f)
try:
# FIXME: check sources.list is using Suite rather than Codename
# FIXME: check for mismatch between Architectures and binary-*/ files
names, pretty_name = preparation_for_checking(rel)
miss_types = check_type(found_url, rel, links[found_url])
# FIXME: change the filename to a URL
print_warning(miss_types, pretty_name, fn, 'release file')
except:
pass
......@@ -55,6 +55,10 @@
# FIXME: add an in-memory cache of hashes so that hashes in multiple releases hit the disk once
# FIXME: deal with rate-limited websites like alioth that do not like many requests
# FIXME: demote this to info for new files: snapshot database issue, no Debian files found
# FIXME: delete debian patch when it is identical to the general patch
# FIXME: replace lsdiff with PatchSet? (check speed)
# FIXME: catch psycopg2 errors and avoid sending them to subscribers
# FIXME: write a sources.missing file for broken 404 packages
import re
import os
......@@ -139,6 +143,8 @@ logging.basicConfig(format='%(levelname)s: %(message)s', level=logging.DEBUG, fi
logging.captureWarnings(True)
# Don't do anything on hosts without the snapshot.d.o farm
# FIXME: add support for the snapshot API:
# https://salsa.debian.org/snapshot-team/snapshot/blob/master/API
if not os.path.isdir(snapshot_cache_dir):
logging.error('Snapshot cache dir not found and snapshot API not yet supported: %s', snapshot_cache_dir)
sys.exit()
......@@ -1207,6 +1213,7 @@ if data:
output_data[sha1][hash_type] = hash
elif hash_type != 'sha1' and hash != output_data[sha1][hash_type]:
logging.warning('hashes mismatched: %s: %s %s != %s', sha1, hash_type, hash, output_data[sha1][hash_type])
# FIXME: write out data files atomically
with open(os.path.abspath(filename), 'w') as output:
yaml.dump(output_data, output, Dumper=yaml.CSafeDumper)
......@@ -1234,6 +1241,7 @@ if data:
item['patches'] = patches
item['parts'] = parts_sha1
output_data.append(item)
# FIXME: write out data files atomically
with open (os.path.abspath(filename), 'w') as output:
yaml.dump(output_data, output, Dumper=yaml.CSafeDumper)
else:
......@@ -1255,6 +1263,7 @@ if data:
item['version'] = version
item['dsc'] = dsc_url
output_data[debian_name][debian_version].append(item)
# FIXME: write out data files atomically
with open (os.path.abspath(filename), 'w') as output:
yaml.dump(output_data, output, Dumper=yaml.CSafeDumper)
else:
......@@ -1274,6 +1283,7 @@ if data:
item = output_data[name][version]
item.setdefault('dsc', []).append(str(dsc_url))
item.update({'hashes' : hashes})
# FIXME: write out data files atomically
with open(os.path.abspath(filename), 'w') as output:
yaml.dump(output_data, output, Dumper=yaml.CSafeDumper)
else:
......
......@@ -9,6 +9,8 @@
# Usage:
# derivative-to-planet-config <shortname> <long name file> <rss feed files x2> <logo head file> <planet config>
# FIXME: make the planet-config output file deterministically
import os
import sys
try: from PIL import Image
......
......@@ -7,6 +7,8 @@
# trusted key. We need this because we do not nessecarily have trust paths to
# all the people creating Debian derivatives.
#
# FIXME: replace this with proper signature verification
#
# Usage:
# Only run by apt, no need to run it manually
......
......@@ -111,8 +111,10 @@ for pkg in new_pkg_info:
files.update(pkg_hashes)
# Save the new data
# FIXME: write out data files atomically
with open(sources_new, 'w') as f:
yaml.dump(new, f, Dumper=yaml.CSafeDumper)
# FIXME: write out data files atomically
with open(sources_files, 'w') as f:
yaml.dump(files, f, Dumper=yaml.CSafeDumper)
......@@ -86,6 +86,7 @@ for root, dirs, files in os.walk(patches_dir):
# Calculate the output
# The general patch metadata
# FIXME: write out data files atomically
f = open(sources_patches,'w')
yaml.dump(patches, f, Dumper=yaml.CSafeDumper)
f.close()
......@@ -11,6 +11,7 @@ git fetch --quiet
git status --short
git --no-pager log --branches --not --remotes --simplify-by-decoration --decorate --oneline
git diff HEAD
# FIXME: the script doesn't seem to get this far
crontab etc/general.crontab
) 666>var/lock
......@@ -7,6 +7,8 @@
# trusted key. We need this because we do not nessecarily have trust paths to
# all the people creating Debian derivatives.
#
# FIXME: replace this with proper signature verification
#
# Usage:
# Only run by apt, no need to run it manually
......
......@@ -9,6 +9,8 @@
# Usage:
# remove-expired-patches <sources patches list> <sources files list> <log file>
# FIXME: add more logging, especially for time consuming loading/saving operations
from datetime import datetime
from datetime import timedelta
import logging
......@@ -293,9 +295,11 @@ for patch in patches:
patches = [patch for patch in patches if patch not in expired_patches]
# FIXME: write out data files atomically
with open(sources_patches, 'w') as f:
yaml.dump(patches, f, Dumper=yaml.CSafeDumper)
# FIXME: write out data files atomically
with open(sources_files, 'w') as f:
yaml.dump(files, f, Dumper=yaml.CSafeDumper)
......
......@@ -8,6 +8,8 @@
# Usage:
# remove-merged-packages <sources files list> <log file>
# FIXME: add more logging, especially for time consuming loading/saving operations
import logging
import os
import sys
......
......@@ -8,6 +8,8 @@
# Usage:
# remove-obsolete-new-packages <obsolete packages> <source files list> <log file>
# FIXME: add more logging, especially for time consuming loading/saving operations
from datetime import datetime
import logging
import os
......@@ -185,5 +187,6 @@ for name, versions in data.items():
logging.shutdown()
# FIXME: write out data files atomically
with open(sources_files, 'w') as f:
yaml.dump(files, f, Dumper=yaml.CSafeDumper)
......@@ -19,6 +19,7 @@ git fetch --quiet
git status --short
git --no-pager log --branches --not --remotes --simplify-by-decoration --decorate --oneline
git diff HEAD
# FIXME: the script doesn't seem to get this far
crontab etc/snapshot.crontab
) 666>var/lock
......@@ -8,6 +8,8 @@
# Usage
# wiki-text-to-subscribers-list <wiki text file> <subscribers emails file>
# FIXME: switch from Subscribers to a longer string, update wiki pages
set -e
sed -r -n -e 's/^ *\* *Subscribers: (.*)/\1/p' "$1" | tr --complement --delete '0-9a-z _@.-' > "$2"
find "$2" -empty -delete
......@@ -21,3 +21,26 @@ Add a layer for merging with external configs, reporting diffs
compare script: rsync down files when unable to connect to postgres
Fix all the static analysis errors
Detect when inactive distros become active again using Homepages
Check that OpenPGP keys are secure and not expired
Add Facebook to RSS translation using tweeper or rss-bridge
Add way to have archive.org save all URLs monthly
Add monitoring for each of the lists for new and newly inactive derivatives
https://wiki.debian.org/Derivatives#Lists
Compare census status field with distrowatch status & wikipedia discontinued fields
Detect inactive websites using DUCK:
http://duck.debian.net/
Check for issues in the reprepro config (duplicate/obsolete suites etc)
Compress patches on-disk and configure Apache to send compressed files when possible and decompress when not possible
......@@ -2,5 +2,8 @@ Acquire::http::User-Agent "Debian Derivatives Census QA bot";
Dir "apt";
Dir::State::status "./apt/var/lib/dpkg/status";
Dir::Etc::sourcelist "./sources.list";
// FIXME: replace this with proper signature verification
Dir::Bin::apt-key "../../bin/fakegpgv";
Dir::Bin::gpg "fakegpgv";
// FIXME: investigate performance impact of compressed apt indicies:
//Acquire::GzipIndexes "yes";
......@@ -38,3 +38,5 @@ AddDescription "Config file for Planet Debian Derivatives"
AddDescription "Git repository of Planet Debian configuration" planet-git
AddDescription "Tarball of Planet heads for the review script" planet-heads.tar
AddDescription "Small logos for Planet Debian Derivatives" planet-heads
# FIXME: add icons and descriptions for each derivative
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment