...
 
Commits (19)
......@@ -18,3 +18,4 @@
/man
/tmp
/.env
/.vagrant
\ No newline at end of file
This diff is collapsed.
......@@ -5,6 +5,11 @@
VAGRANTFILE_API_VERSION = '2'.freeze
Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
config.vm.box = ENV['BOX'] || 'debian/stretch64'
# contrib has the vboxsf kernel module, which is needed for guest addition
config.vm.box = ENV['BOX'] || 'debian/contrib-stretch64'
config.vm.network 'forwarded_port', guest: 8080, host: 8080
config.vm.synced_folder '.', '/vagrant'
config.vm.provision 'shell', path: 'tools/vagrant_root.sh'
config.vm.provision 'shell', path: 'tools/vagrant_user.sh', privileged: false
end
......@@ -14,37 +14,36 @@ PROBABILITY = 0.2
#
#
if ENV["DEBCI_FAKE_DEPS"]
ENV["DEBCI_FAKE_DEPS"].split('|').each do |line|
if ENV['DEBCI_FAKE_DEPS']
ENV['DEBCI_FAKE_DEPS'].split('|').each do |line|
puts line
end
exit
end
bin = File.dirname(__FILE__);
bin = File.dirname(__FILE__)
pkg = ARGV.first
unless pkg
puts "usage: list-packages PACKAGE"
puts 'usage: list-packages PACKAGE'
exit 1
end
pkgdir = pkg.gsub(/((lib)?.).*/, '\1/\&')
suite = ENV['debci_suite'] || 'unstable'
arch = ENV['debci_arch'] || 'amd64';
dependencies_file = "#{bin}/../../data/packages/#{suite}/#{arch}/#{pkgdir}/dependencies.txt";
arch = ENV['debci_arch'] || 'amd64'
dependencies_dir = "#{bin}/../../data/packages/#{suite}/#{arch}/#{pkgdir}"
dependencies_file = File.join(dependencies_dir, 'dependencies.txt')
if (File.exists? dependencies_file)
if File.exist? dependencies_file
File.readlines(dependencies_file).each do |line|
line.strip!
if (rand() <= PROBABILITY)
line += '+1';
end
line += '+1' if rand <= PROBABILITY
puts line
end
else
%w/foo bar baz qux/.each do |fake_pkg|
%w[foo bar baz qux].each do |fake_pkg|
printf("%s %d.%d-%d\n", fake_pkg, rand(4), rand(4), rand(4))
end
end
......@@ -5,24 +5,22 @@ require 'thor'
module Debci
class APICLI < Thor
desc 'setkey USERNAME [KEYNAME]', "Sets a API key for the given user"
def setkey(username, keyname='default')
desc 'setkey USERNAME [KEYNAME]', 'Sets a API key for the given user'
def setkey(username, _keyname = 'default')
key = Debci::Key.create!(user: username)
puts key.key
end
desc 'auth KEY', "Checks authentication for the given KEY"
desc 'auth KEY', 'Checks authentication for the given KEY'
def auth(key)
user = Debci::Key.authenticate(key)
if user
puts "I: Valid key for user `#{user}`"
else
$stderr.puts "E: invalid key"
warn 'E: invalid key'
exit(1)
end
end
end
end
......
......@@ -9,15 +9,15 @@ require 'debci/job'
trigger = nil
pin_packages = []
run_id = nil
requestor = ENV["USER"] || 'nobody'
requestor = ENV['USER'] || 'nobody'
arch = Debci.config.arch
suite = Debci.config.suite
priority = 0
# parse command line
optparse = OptionParser.new do |opts|
opts.banner = "Usage: debci enqueue [OPTIONS] PKG [PKG ...]"
opts.separator "Options:"
opts.banner = 'Usage: debci enqueue [OPTIONS] PKG [PKG ...]'
opts.separator 'Options:'
opts.on('-s', '--suite SUITE', 'sets the suite to test') do |s|
suite = s
......@@ -46,8 +46,8 @@ optparse = OptionParser.new do |opts|
opts.on('-P', '--priority N', 'sets priority for the test (0-10)') do |p|
priority = Integer(p)
if ! (0..10).include?(priority)
$stderr.puts "E: priority must be a number between 0 and 10"
unless (0..10).cover?(priority)
warn 'E: priority must be a number between 0 and 10'
exit 1
end
end
......@@ -55,7 +55,6 @@ optparse = OptionParser.new do |opts|
opts.on('-i', '--run-id RUNID') do |id|
run_id = id
end
end
optparse.parse!
......@@ -68,7 +67,7 @@ ARGV.each do |pkg|
status: nil,
trigger: trigger,
pin_packages: pin_packages,
run_id: run_id,
run_id: run_id
)
job.save!
job.enqueue(priority)
......
#!/usr/bin/ruby
require 'fileutils'
require 'optparse'
require 'debci'
require 'debci/html'
require 'debci/graph'
$all_packages = false
optparse = OptionParser.new do |opts|
opts.banner = 'Usage: debci status [OPTIONS] [PACKAGE]'
opts.separator 'Options:'
opts.on('-a', '--all', 'Regenerate HTML for all packages') do
$all_packages = true
end
end
optparse.parse!
if $all_packages && !ARGV.empty?
warn "E: -a/--all is incompatible with passing a list of packages"
exit(1)
end
$DEBCI_HTML_FAST = (ENV['DEBCI_HTML_FAST'] != nil)
Debci.log('debci generate-html started')
......@@ -24,11 +41,13 @@ Debci.log('status pages html updated')
repository = Debci::Repository.new
packages = ARGV
if packages.empty?
exit
if $all_packages
packages = repository.packages
else
exit
end
end
packages += Debci.blacklist.packages.keys
prefixes = Set.new
......
......@@ -7,17 +7,16 @@ require 'thor'
module Debci
class JobCLI < Thor
desc 'import STATUS_FILE [STATUS_FILE ...]', 'Import data from status file'
method_option :remove, type: :boolean, default: false
method_option :verbose, type: :boolean, default: false
def import(*status_files)
status_files.each do |status_file|
Debci::Job.import(status_file, suite, arch)
puts('I: imported %s' % status_file) if options{:verbose}
puts(format('I: imported %<status_file>s', status_file: status_file)) if options { :verbose }
if options[:remove]
File.unlink(status_file)
puts('I: removed %s' % status_file) if options{:verbose}
puts(format('I: removed %<status_file>s', status_file: status_file)) if options { :verbose }
end
end
end
......@@ -27,13 +26,12 @@ module Debci
history = Debci::Job.where(
package: pkg,
suite: suite,
arch: arch,
arch: arch
).where.not(status: nil).order('run_id')
data = JSON.parse(history.to_json)
puts JSON.pretty_generate(data)
end
desc 'declare-queue', 'Initializes job queue'
def declare_queue
Debci::Job.get_queue(Debci.config.arch)
......@@ -43,11 +41,11 @@ module Debci
def arch
Debci.config.arch
end
def suite
Debci.config.suite
end
end
end
end
......
......@@ -23,72 +23,75 @@ name="${debci_suite}-${debci_arch}"
if [ ! -e "$root/$name" ]; then
log "I: Creating new chdist $root/$name"
call_chdist create >/dev/null
fi
# figure out default mirror from debootstrap scripts
DEF_MIRROR="${debci_mirror}"
SUITE=$debci_suite
TARGET="$root/$name"
ARCH=$debci_arch
set +u
export DEBOOTSTRAP_DIR=/usr/share/debootstrap
. /usr/share/debootstrap/functions
exec 4>&1
# this updates $DEF_MIRROR (Ubuntu, ports, ..)
. /usr/share/debootstrap/scripts/$debci_suite
set -u
# enable all components
if [ "${DEF_MIRROR%ubuntu*}" = "$DEF_MIRROR" ]; then
COMPONENTS="main contrib non-free" # Debian
else
COMPONENTS="main restricted universe multiverse" # Ubuntu
fi
# figure out default mirror from debootstrap scripts
DEF_MIRROR="${debci_mirror}"
SUITE=$debci_suite
TARGET="$root/$name"
ARCH=$debci_arch
set +u
export DEBOOTSTRAP_DIR=/usr/share/debootstrap
. /usr/share/debootstrap/functions
exec 4>&1
# this updates $DEF_MIRROR (Ubuntu, ports, ..)
. /usr/share/debootstrap/scripts/$debci_suite
set -u
# enable all components
if [ "${DEF_MIRROR%ubuntu*}" = "$DEF_MIRROR" ]; then
COMPONENTS="main contrib non-free" # Debian
else
COMPONENTS="main restricted universe multiverse" # Ubuntu
fi
mirror=${debci_mirror:-$DEF_MIRROR}
mirror=${debci_mirror:-$DEF_MIRROR}
# write apt sources.list
mkdir -p $TARGET/etc/apt/
echo "deb [arch=${debci_arch}] $mirror $SUITE $COMPONENTS
# write apt sources.list
mkdir -p $TARGET/etc/apt/
echo "deb [arch=${debci_arch}] $mirror $SUITE $COMPONENTS
deb-src $mirror $SUITE $COMPONENTS" > "$TARGET/etc/apt/sources.list"
# add buildd sources
# FIXME duplicates logic in backends/(schroot|lxc)/create-testbed
if grep -q debian "$TARGET/etc/apt/sources.list"; then
if [ "$SUITE" = unstable ]; then
buildd_suite=buildd-$SUITE
else
buildd_suite=buildd-$SUITE-proposed-updates
fi
cat >> "$TARGET/etc/apt/sources.list" <<EOF
# add buildd sources
# FIXME duplicates logic in backends/(schroot|lxc)/create-testbed
if grep -q debian "$TARGET/etc/apt/sources.list"; then
if [ "$SUITE" = unstable ]; then
buildd_suite=buildd-$SUITE
elif [ "$SUITE" = stable ]; then
# workaround https://bugs.debian.org/880105
buildd_suite=buildd-$(distro-info --stable)-proposed-updates
else
buildd_suite=buildd-$SUITE-proposed-updates
fi
cat >> "$TARGET/etc/apt/sources.list" <<EOF
deb [arch=${debci_arch}] http://incoming.debian.org/debian-buildd $buildd_suite $COMPONENTS
deb-src http://incoming.debian.org/debian-buildd $buildd_suite $COMPONENTS
EOF
fi
fi
# use a local proxy if available
http_proxy="${http_proxy:-}"
if [ -z "$http_proxy" ]; then
# detect a local apt-cacher-ng cache running. 10.0.2.2 = default IP
# assigned to host system as seen from a kvm/virtualbox virtual machine
for ip in 127.0.0.1 10.0.2.2; do
if nc -z -w 1 $ip 3142; then
export http_proxy=http://$ip:3142
fi
done
fi
if [ -n "$http_proxy" ]; then
echo "Acquire::http::Proxy \"$http_proxy\";" > "$TARGET/etc/apt/apt.conf.d/01proxy"
fi
# use a local proxy if available
http_proxy="${http_proxy:-}"
if [ -z "$http_proxy" ]; then
# detect a local apt-cacher-ng cache running. 10.0.2.2 = default IP
# assigned to host system as seen from a kvm/virtualbox virtual machine
for ip in 127.0.0.1 10.0.2.2; do
if nc -z -w 1 $ip 3142; then
export http_proxy=http://$ip:3142
fi
done
fi
if [ -n "$http_proxy" ]; then
echo "Acquire::http::Proxy \"$http_proxy\";" > "$TARGET/etc/apt/apt.conf.d/01proxy"
fi
# disable multi-arch
echo "Apt::Architectures {\"$ARCH\";};" > "$TARGET/etc/apt/apt.conf.d/97_no_multiarch"
# disable multi-arch
echo "Apt::Architectures {\"$ARCH\";};" > "$TARGET/etc/apt/apt.conf.d/97_no_multiarch"
# disable unnecessary srcpkgcache
echo 'Dir::Cache::srcpkgcache "";' > "$TARGET/etc/apt/apt.conf.d/98disable_cache"
# disable unnecessary srcpkgcache
echo 'Dir::Cache::srcpkgcache "";' > "$TARGET/etc/apt/apt.conf.d/98disable_cache"
# do not download translations
echo 'Acquire::Languages "none";' > "$TARGET/etc/apt/apt.conf.d/99translations"
fi
# do not download translations
echo 'Acquire::Languages "none";' > "$TARGET/etc/apt/apt.conf.d/99translations"
update_chdist() {
call_chdist apt-get update
......
......@@ -11,6 +11,7 @@ Build-Depends: debhelper (>= 9.20160709),
dctrl-tools,
debian-archive-keyring,
devscripts,
distro-info,
fonts-font-awesome (>= 4.0.3~),
inotify-tools <!pkg.debci.runtime>,
jq,
......
......@@ -2,6 +2,44 @@
## Setting up a development environment
There are two ways to setup the development environment:
* Manual Setup
* Vagrant
## Vagrant
### Prerequisites
* Vagrant: 2.2.4 https://www.vagrantup.com/
* VirtualBox: 6.0 https://www.virtualbox.org/
### Install the virtual machine
Run this at the same path where `VagrantFile` is
```
$ vagrant up
```
### Start debci
SSH into vagrant environment
```
$ vagrant ssh
```
Once inside vagrant, you can start run debci with the following commands
```
vagrant@stretch $ cd /vagrant
vagrant@stretch $ foreman start
```
Note: The other commands are the same as the ones mentioned in manual setup.
## Manual Setup
### Grab the dependencies and required software
Install the dependencies and build dependencies:
......@@ -182,3 +220,16 @@ following:
If your code passed the appropriate tests, you will see that there
are no failures reported by rspec.
## Contribution guidelines
* If you are new to Free Software/Open Source, read [How to Contribute to Open Source](https://opensource.guide/how-to-contribute/) first.
* Some of the advice in there is specific to GitHub, but most of it is general enough to be useful.
* Separate commits by logical change
* Write meaningful commit messages. See:
* [A Note About Git Commit Messages](https://tbaggery.com/2008/04/19/a-note-about-git-commit-messages.html)
* [How to Write a Git Commit Message](https://chris.beams.io/posts/git-commit/)
* [Useful Tips for writing better Git commit messages](https://code.likeagirl.io/useful-tips-for-writing-better-git-commit-messages-808770609503)
* _Read_ your commits before sending them out, i.e. put yourself at the position of others:
* Was I the one receiving these patches, without knowing what I know after writing them, do they make sense. Are they self-explanatory?
* Does the coding style (indentation, variable naming, etc) match the existing code?
......@@ -3,9 +3,7 @@ require 'debci/config'
require 'debci/blacklist'
module Debci
class << self
def config
@config ||= Debci::Config.new
end
......@@ -15,17 +13,15 @@ module Debci
end
def config!(data)
data.each do |k,v|
data.each do |k, v|
ENV["debci_#{k}"] = v
end
@config = nil
@blacklist = nil
end
def log(*s)
puts(*s) unless config.quiet
def log(*str)
puts(*str) unless config.quiet
end
end
end
module Debci
class Blacklist
def initialize(config_dir)
@config_dir = config_dir
end
def include?(package)
packages.keys.include?(String(package))
packages.key?(String(package))
end
def packages
......@@ -20,7 +19,9 @@ module Debci
if line =~ /^\s*$/
true # skip blank lines
elsif line =~ /^\s*#/
reason << line.sub(/^\s*#\s*/, '').gsub(/(https?:\/\/\S*)/, '<a href="\1">\1</a>')
old_str = %r{(https?://\S*)}
new_str = '<a href="\1">\1</a>'
reason << line.sub(/^\s*#\s*/, '').gsub(old_str, new_str)
else
pkg = line.strip
packages[pkg] = reason
......
......@@ -3,34 +3,30 @@ require 'active_record'
module Debci
module DB
def self.config
@config ||= ENV['DATABASE_URL'] || Debci.config.database_url
end
def self.establish_connection
ActiveRecord::Base.establish_connection(self.config)
ActiveRecord::Base.establish_connection(config)
end
def self.migrate
migrations_path = File.join(File.dirname(__FILE__), 'db', 'migrations')
ActiveRecord::Migration.verbose = !Debci.config.quiet
version = nil
if ActiveRecord.version.release() < Gem::Version.new('5.2.0')
if ActiveRecord.version.release < Gem::Version.new('5.2.0')
ActiveRecord::Migrator.migrate(migrations_path, nil)
else
ActiveRecord::MigrationContext.new(migrations_path).migrate
end
end
if ActiveRecord.version.release() < Gem::Version.new('5.1.0')
LegacyMigration = ActiveRecord::Migration
else
LegacyMigration = ActiveRecord::Migration[4.2]
end
version_isnewer = ActiveRecord.version.release < Gem::Version.new('5.1.0')
LEGACY_MIGRATION = if version_isnewer
ActiveRecord::Migration
else
ActiveRecord::Migration[4.2]
end
end
end
Debci::DB.establish_connection
class CreateJobs < Debci::DB::LegacyMigration
class CreateJobs < Debci::DB::LEGACY_MIGRATION
def up
create_table(:jobs, primary_key: 'run_id') do |t|
t.timestamps(null: false)
t.string :suite, :limit => 100
t.string :arch, :limit => 100
t.string :package, :limit => 100
t.string :version, :limit => 100
t.string :suite, limit: 100
t.string :arch, limit: 100
t.string :package, limit: 100
t.string :version, limit: 100
t.string :trigger
t.string :status, :limit => 25
t.string :requestor, :limit => 256, index: true
t.string :status, limit: 25
t.string :requestor, limit: 256, index: true
t.text :pin_packages
t.string :worker
end
......@@ -20,4 +20,3 @@ class CreateJobs < Debci::DB::LegacyMigration
drop_table :jobs
end
end
class CreateKeys < Debci::DB::LegacyMigration
class CreateKeys < Debci::DB::LEGACY_MIGRATION
def up
create_table(:keys) do |t|
t.timestamps(null: false)
......
class CompleteJobFields < Debci::DB::LegacyMigration
class CompleteJobFields < Debci::DB::LEGACY_MIGRATION
def up
add_column :jobs, :date, :datetime
......
......@@ -4,6 +4,6 @@
title='On the Package Tracking System'>Tracker</a>
<a class='btn btn-default' href='https://qa.debian.org/excuses.php?package=<%= @package.name %>'
title='On the Excuses Page'>Excuses</a>
<a class='btn btn-default' href='https://bugs.debian.org/<%= @package.name %>'
<a class='btn btn-default' href='https://bugs.debian.org/src:<%= @package.name %>'
title='Bug Reports'>Bug Reports</a>
</div>
\ No newline at end of file
module Debci
# This class represents a single package. See Debci::Repository for how to
# obtain one of these.
class Package < Struct.new(:name, :repository)
Package = Struct.new(:name, :repository) do
# Returns the architectures in which this package is available
def architectures
repository.architectures_for(self)
......@@ -48,7 +46,7 @@ module Debci
# Returns an Array of statuses where this package is failing or neutral.
def fail_or_neutral
status.flatten.select { |p| p.status == :fail or p.status == :neutral }
status.flatten.select { |p| (p.status == :fail) || (p.status == :neutral) }
end
# Returns an Array of statuses where this package is temporarily failing. If
......@@ -68,13 +66,11 @@ module Debci
def prefix
name =~ /^((lib)?.)/
$1
Regexp.last_match(1)
end
def blacklisted?
Debci.blacklist.include?(self)
end
end
end
......@@ -164,7 +164,10 @@ module Debci
status.requestor = data['requestor']
status.date =
begin
Time.parse((data['date'] || 'unknown') + ' UTC')
d = data['date']
d ||= data['created_at']
d ||= 'unknown'
Time.parse(d + ' UTC')
rescue ArgumentError
nil
end
......
......@@ -21,4 +21,4 @@ esac
echo "I: Web UI at http://localhost:$port/"
echo "I: Hit Control+C to stop"
echo ""
exec rerun --no-notify --background --dir lib -p '**/*.rb' -- rackup --include lib --port="$port"
exec rerun --no-notify --background --dir lib -p '**/*.rb' -- rackup --include lib --port="$port" --host=0.0.0.0
#!/bin/bash
echo "Running as root"
export DEBIAN_FRONTEND=noninteractive
echo "deb http://deb.debian.org/debian stretch-backports main" >> /etc/apt/sources.list
apt-get -y update
apt-get -qqyt stretch-backports install autopkgtest
apt-get -qqy install make ruby git debootstrap
cd /vagrant
apt-get -qqy build-dep .
#!/bin/bash
echo "Running as user vagrant"
cd /vagrant
./tools/init-dev.sh
make
./bin/debci migrate
./bin/debci setup-chdist
echo "Development virtual machine is installed!"
\ No newline at end of file