playbook.yml

@@ -10,11 +10,25 @@
   become_method: sudo
   vars_files:
   - external_vars.yml
+  tags:
+  - builders
   roles:
   - builders
   # The gpgkey generation role had to be disabled because it was taking a lot of time. I'm not sure we are supposed to do this.
   # - { role: juju4.gpgkey_generate, gpg_user: "{{ build_gpg_user }}", gpg_realname: "{{ build_gpg_realname }}", gpg_useremail: "{{ build_gpg_email }}" , gpg_generator_user: "root" }
 
+- name: Setup Scheduler
+  hosts: schedulers
+  become: yes
+  become_user: root
+  become_method: sudo
+  vars_files:
+  - external_vars.yml
+  tags:
+  - schedulers
+  roles:
+  - schedulers
+
 - name: Setup Visualizers
   hosts: visualizers
   become: yes
@@ -22,6 +36,8 @@
   become_method: sudo
   vars_files:
   - external_vars.yml
+  tags:
+  - visualizers
   pre_tasks:
   - name: Install dirmngr for nginx
     apt:

roles/schedulers/tasks/main.yml

+- name: Install all dependencies
+  apt:
+    name:
+    - redis-server
+    - python3-redis
+    - python3-requests
+
+- name: Copy scheduler-worker
+  copy:
+    src: ../../../scheduler/srebuild-worker
+    dest: /usr/bin/srebuild-worker
+
+- name: Copy scheduler-monitor
+  copy:
+    src: ../../../scheduler/srebuild-monitor
+    dest: /usr/bin/srebuild-monitor
+
+- name: Set permissions
+  file:
+    path: "{{ item }}"
+    mode: 0755
+  with_items:
+  - /usr/bin/srebuild-worker
+  - /usr/bin/srebuild-monitor

scheduler/srebuild-monitor

+#!/usr/bin/env python3
+import requests
+import urllib.parse
+import redis
+import time
+import sys
+
+
+def main(server):
+    most_recent = 0
+    db = redis.StrictRedis(host='localhost', port=6379, db=0)
+
+    # try to load most_recent, default to 0
+    most_recent = db.get('rebuild-most-recent') or '0'
+    most_recent = int(most_recent)
+
+    while True:
+        print('[*] Requesting new buildinfo files')
+
+        url = urllib.parse.urljoin(server, '/api/buildinfo/since/%d' % most_recent)
+        r = requests.get(url)
+        r.raise_for_status()
+        buildinfos = r.json()['buildinfos']
+
+        for buildinfo in buildinfos:
+            raw_uri = buildinfo['raw-uri']
+            print('[+] Adding %r' % raw_uri)
+
+            # request buildinfo file
+            r = requests.get(raw_uri)
+            r.raise_for_status()
+
+            # add buildinfo to queue
+            db.rpush('rebuild-q', r.text)
+
+            # update most_recent
+            most_recent = buildinfo['created']
+
+        # store most_recent
+        db.set('rebuild-most-recent', most_recent)
+
+        if not buildinfos:
+            print('[*] Sleeping zZz')
+            time.sleep(10)
+
+
+if __name__ == '__main__':
+    if len(sys.argv) < 2:
+        print('Usage: %s http://buildinfo.nyu.wtf/' % sys.argv[0])
+    else:
+        main(sys.argv[1])

scheduler/srebuild-worker

+#!/usr/bin/env python3
+import redis
+import tempfile
+import subprocess
+
+
+def rebuild(buildinfo):
+    with tempfile.NamedTemporaryFile() as f:
+        f.write(buildinfo)
+        cmd = ['srebuild-runner', f.name]
+        print('[+] invoking %r' % cmd)
+        rc = subprocess.call(cmd)
+        if rc != 0:
+            # TODO: we should handle this properly
+            print('[!] srebuild returned an error')
+
+
+def main():
+    r = redis.StrictRedis(host='localhost', port=6379, db=0)
+    print('[*] monitoring queue')
+    while True:
+        q, item = r.blpop('rebuild-q')
+        print('[*] rebuilding %r' % item)
+        rebuild(item)
+
+
+if __name__ == '__main__':
+    main()