.travis.yml

+language: python
+sudo: false
+python:
+- '2.7'
+- '3.4'
+- '3.5'
+- '3.6'
+branches:
+  except:
+  - "/^feature.*$/"
+install:
+- pip install flake8
+- pip install -r requirements.txt
+- pip install coverage
+- pip install python-coveralls
+- python setup.py -q install
+script:
+- python setup.py test
+- flake8 --ignore E501,E123,W504 biomaj_daemon
+deploy:
+  provider: pypi
+  skip_existing: true
+  user: osallou
+  password:
+    secure: YQSe/QsKszbPDQSQSS+u1WGvDB6akPFukU+msOqEX7lupyfLHQPnfvRt4AbiwMvdslXjAn8Y25vp25Gb5cnut4E9RBS+hUihcCBMuwtC/E+Hyy+kAG8zue5oH5sv5kuKLxfNmHSyi4t0VfusuabHGw1Wq5QIqBhpbOlefxnUpPrOeKcDwVE17fLcdhPsz5E4InsoTfFKGFtsgTzRMSqfdCWuzWdtO7oglSLuDk0Ow2E8I37jAiOvBLB9Cj4dynmRVUMRrXfZT76DWX9fEzI5OoT9O62nt6l5p6dEewRrSqU5ODSOlpUXiehuS1wCKKHm8ORdERpxATN274xg8lYeFFs8CVCdXFRoEy8YBql8YOw+qmzOpK8tDQAneDFbCOWTg3kR/HUGZwuvo8lEka/SGsZsOJXdBq3C1WVUxb//vTf0owu7p82Z0INMaZ0UB/MLwYq1O8lTzsrY7f+Za2enayOUmFs1li+Ib5JLs8yoVGjzieadxSrmpYbCWjz6bGPuXUtkd00X/xiQYehQmoiOMD/lTDOiC+CcySG4mwjtj+fO7KLS8LH2SRcjoIxIdKNGifXoL1Mu1/iQzrpEoY8KuPXgFuqWyg9w5KWQ1LV8EAoUUJTbQDb/XxtIn1RUGxS7CEmVEheIiu+hI/VmWdBuqPYoI/pzRA9g3YpaylvJqxM=
+  distributions: sdist bdist_wheel
+  on:
+    tags: true

CHANGES.txt

+3.0.19:
+  Fix traefix prefix
+3.0.18:
+  Fix some bank status info checks
+  Add new endpoint /api/daemon/bank/<bank>/owns
+  Add queued bank names in whatsup
+  Update pika dependency release
+  Add tags for traefik support
 3.0.17:
   Allow use of env variables to override micro.x and rabbitmq variables in global.properties
     like for config.yml.

biomaj_daemon/daemon/biomaj_daemon_web.py

@@ -84,7 +84,22 @@ biomaj_time_metric = Gauge("biomaj_daemon_time", "Bank update execution time in
 def consul_declare(config):
     if config['consul']['host']:
         consul_agent = consul.Consul(host=config['consul']['host'])
-        consul_agent.agent.service.register('biomaj-daemon', service_id=config['consul']['id'], address=config['web']['hostname'], port=config['web']['port'], tags=['biomaj'])
+        consul_agent.agent.service.register(
+            'biomaj-daemon',
+            service_id=config['consul']['id'],
+            address=config['web']['hostname'],
+            port=config['web']['port'],
+            tags=[
+                'biomaj',
+                'api',
+                'traefik.backend=biomaj-daemon',
+                'traefik.frontend.rule=PathPrefix:/api/daemon',
+                'traefik.enable=true',
+                'traefik-int.backend=biomaj-daemon',
+                'traefik-int.frontend.rule=PathPrefix:/api/daemon',
+                'traefik-int.enable=true'
+                ]
+        )
         check = consul.Check.http(url='http://' + config['web']['hostname'] + ':' + str(config['web']['port']) + '/api/daemon', interval=20)
         consul_agent.agent.check.register(config['consul']['id'] + '_check', check=check, service_id=config['consul']['id'])
 
@@ -230,7 +245,7 @@ def biomaj_status_info():
             biomaj_mongo.server_info()
             mongo_ok = True
         except Exception as e:
-            logging.error('Failed to connect to mongo')
+            logging.exception('Failed to connect to mongo:' + str(e))
         if not mongo_ok:
             status['status'].append({'service': 'biomaj-mongo', 'status': -1, 'count': 0})
         else:
@@ -278,7 +293,7 @@ def biomaj_status_info():
     r = requests.get('http://' + config['consul']['host'] + ':8500/v1/agent/services')
     if not r.status_code == 200:
         status['status'].append({'service': 'biomaj-consul', 'status': -1, 'count': 0})
-        last_status = status
+        # last_status = status
         return status
 
     status['status'].append({'service': 'biomaj-consul', 'status': 1, 'count': 1})
@@ -299,7 +314,7 @@ def biomaj_status_info():
         check_r = requests.get('http://' + config['consul']['host'] + ':8500/v1/health/service/' + consul_services[consul_service]['Service'])
         if not check_r.status_code == 200:
             status['status'].append({'service': 'biomaj-consul', 'status': -1, 'count': 0})
-            last_status = status
+            # last_status = status
             return status
         checks = check_r.json()
         nb_service = 0
@@ -333,14 +348,16 @@ def biomaj_status_info():
         if biomaj_service not in services:
             status['status'].append({'service': biomaj_service, 'count': 0, 'status': -1})
 
-    last_status = status
+    # last_status = status
 
     return jsonify(status)
 
+
 @app.route('/api/daemon/bank/<bank>/log', methods=['GET'])
 def biomaj_bank_log(bank):
     return biomaj_bank_log_tail(bank, 0)
 
+
 @app.route('/api/daemon/bank/<bank>/log/<tail>', methods=['GET'])
 def biomaj_bank_log_tail(bank, tail=100):
     apikey = request.headers.get('Authorization')
@@ -389,7 +406,6 @@ def biomaj_bank_log_tail(bank, tail=100):
     return Response(generate(), mimetype='text/plain')
 
 
-
 def daemon_api_auth(request):
     apikey = request.headers.get('Authorization')
     token = None
@@ -426,6 +442,7 @@ def daemon_api_auth(request):
 
     return (200, options_object, None)
 
+
 @app.route('/api/daemon/aboutme', methods=['POST'])
 def biomaj_daemon_aboutme():
     (http_code, options, error) = daemon_api_auth(request)
@@ -446,6 +463,7 @@ def biomaj_daemon_aboutme():
     except Exception as e:
         abort(500, str(e))
 
+
 @app.route('/api/daemon/maintenance', methods=['POST'])
 def biomaj_daemon_maintenance_on():
     (http_code, options, error) = daemon_api_auth(request)
@@ -464,6 +482,7 @@ def biomaj_daemon_maintenance_on():
     except Exception as e:
         abort(500, str(e))
 
+
 @app.route('/api/daemon/maintenance', methods=['DELETE'])
 def biomaj_daemon_maintenance_off():
     (http_code, options, error) = daemon_api_auth(request)
@@ -482,6 +501,7 @@ def biomaj_daemon_maintenance_off():
     except Exception as e:
         abort(500, str(e))
 
+
 @app.route('/api/daemon/maintenance', methods=['GET'])
 def biomaj_daemon_maintenance_status():
     (http_code, options, error) = daemon_api_auth(request)
@@ -498,6 +518,7 @@ def biomaj_daemon_maintenance_status():
     except Exception as e:
         abort(500, str(e))
 
+
 @app.route('/api/daemon/whatsup', methods=['GET'])
 def biomaj_daemon_maintenance_whatsup():
     (http_code, options, error) = daemon_api_auth(request)
@@ -514,6 +535,7 @@ def biomaj_daemon_maintenance_whatsup():
     except Exception as e:
         abort(500, str(e))
 
+
 @app.route('/api/daemon/stats', methods=['GET'])
 def biomaj_daemon_stats():
     (http_code, options, error) = daemon_api_auth(request)
@@ -530,6 +552,7 @@ def biomaj_daemon_stats():
     except Exception as e:
         abort(500, str(e))
 
+
 @app.route('/api/daemon/version', methods=['GET'])
 def biomaj_daemon_version():
     (http_code, options, error) = daemon_api_auth(request)
@@ -582,6 +605,7 @@ def biomaj_daemon_banks_status():
     except Exception as e:
         abort(500, str(e))
 
+
 @app.route('/api/daemon/bank/<bank>', methods=['GET'])
 def biomaj_daemon_bank_status(bank):
     (http_code, options, error) = daemon_api_auth(request)
@@ -599,6 +623,22 @@ def biomaj_daemon_bank_status(bank):
     except Exception as e:
         abort(500, str(e))
 
+
+@app.route('/api/daemon/bank/<bank>/owns', methods=['GET'])
+def biomaj_daemon_bank_isowner(bank):
+    '''
+    Checks that logged user is admin or owner of the bank
+    '''
+    (http_code, options, error) = daemon_api_auth(request)
+    if error:
+        abort(http_code, error)
+    bank_log = Bank(bank, options=options, no_log=True)
+    if bank_log.is_owner():
+        return jsonify({'is_admin_or_owner': True})
+    else:
+        return jsonify({'is_admin_or_owner': False})
+
+
 @app.route('/api/daemon/bank/<bank>/check', methods=['GET'])
 def biomaj_daemon_bank_check(bank):
     (http_code, options, error) = daemon_api_auth(request)
@@ -616,6 +656,7 @@ def biomaj_daemon_bank_check(bank):
     except Exception as e:
         abort(500, str(e))
 
+
 @app.route('/api/daemon/bank/<bank>/status', methods=['GET'])
 def biomaj_daemon_bank_update_status(bank):
     (http_code, options, error) = daemon_api_auth(request)
@@ -633,6 +674,7 @@ def biomaj_daemon_bank_update_status(bank):
     except Exception as e:
         abort(500, str(e))
 
+
 @app.route('/api/daemon/status/ko', methods=['GET'])
 def biomaj_daemon_bank_status_ko(bank):
     (http_code, options, error) = daemon_api_auth(request)
@@ -649,6 +691,7 @@ def biomaj_daemon_bank_status_ko(bank):
     except Exception as e:
         abort(500, str(e))
 
+
 @app.route('/api/daemon/bank/<bank>/cancel', methods=['PUT'])
 def biomaj_daemon_bank_cancel(bank):
     (http_code, options, error) = daemon_api_auth(request)
@@ -668,6 +711,7 @@ def biomaj_daemon_bank_cancel(bank):
     except Exception as e:
         abort(500, str(e))
 
+
 @app.route('/api/daemon/bank/<bank>/owner/<owner>', methods=['PUT'])
 def biomaj_daemon_bank_upate_owner(bank, owner):
     (http_code, options, error) = daemon_api_auth(request)
@@ -687,6 +731,7 @@ def biomaj_daemon_bank_upate_owner(bank, owner):
     except Exception as e:
         abort(500, str(e))
 
+
 @app.route('/api/daemon/bank/<bank>/visibility/<visibility>', methods=['PUT'])
 def biomaj_daemon_bank_upate_visibility(bank, visibility):
     (http_code, options, error) = daemon_api_auth(request)
@@ -706,6 +751,7 @@ def biomaj_daemon_bank_upate_visibility(bank, visibility):
     except Exception as e:
         abort(500, str(e))
 
+
 @app.route('/api/daemon/bank/<bank>/name/<name>', methods=['PUT'])
 def biomaj_daemon_bank_update_name(bank, name):
     (http_code, options, error) = daemon_api_auth(request)
@@ -725,6 +771,7 @@ def biomaj_daemon_bank_update_name(bank, name):
     except Exception as e:
         abort(500, str(e))
 
+
 @app.route('/api/daemon/bank/<bank>/move', methods=['PUT'])
 def biomaj_daemon_bank_update_directory(bank):
     (http_code, options, error) = daemon_api_auth(request)
@@ -745,6 +792,7 @@ def biomaj_daemon_bank_update_directory(bank):
     except Exception as e:
         abort(500, str(e))
 
+
 @app.route('/api/daemon/bank/<bank>', methods=['POST'])
 def biomaj_daemon_bank_update(bank):
     (http_code, options, error) = daemon_api_auth(request)
@@ -781,6 +829,7 @@ def biomaj_daemon_bank_update(bank):
     except Exception as e:
         abort(500, str(e))
 
+
 @app.route('/api/daemon/bank/<bank>/publish', methods=['PUT'])
 def biomaj_daemon_bank_publish(bank):
     (http_code, options, error) = daemon_api_auth(request)
@@ -805,6 +854,7 @@ def biomaj_daemon_bank_publish(bank):
     except Exception as e:
         abort(500, str(e))
 
+
 @app.route('/api/daemon/bank/<bank>/publish', methods=['DELETE'])
 def biomaj_daemon_bank_unpublish(bank):
     (http_code, options, error) = daemon_api_auth(request)
@@ -824,6 +874,7 @@ def biomaj_daemon_bank_unpublish(bank):
     except Exception as e:
         abort(500, str(e))
 
+
 @app.route('/api/daemon/bank/<bank>', methods=['DELETE'])
 def biomaj_daemon_bank_remove(bank):
     (http_code, options, error) = daemon_api_auth(request)
@@ -858,6 +909,7 @@ def biomaj_daemon_bank_remove(bank):
     except Exception as e:
         abort(500, str(e))
 
+
 @app.route('/api/daemon/bank/<bank>/freeze/<release>', methods=['PUT'])
 def biomaj_daemon_bank_freeze(bank, release):
     (http_code, options, error) = daemon_api_auth(request)
@@ -878,6 +930,7 @@ def biomaj_daemon_bank_freeze(bank, release):
     except Exception as e:
         abort(500, str(e))
 
+
 @app.route('/api/daemon/bank/<bank>/freeze/<release>', methods=['DELETE'])
 def biomaj_daemon_bank_unfreeze(bank, release):
     (http_code, options, error) = daemon_api_auth(request)
@@ -898,6 +951,7 @@ def biomaj_daemon_bank_unfreeze(bank, release):
     except Exception as e:
         abort(500, str(e))
 
+
 @app.route('/api/daemon/search', methods=['GET'])
 def biomaj_daemon_bank_search(bank):
     (http_code, options, error) = daemon_api_auth(request)
@@ -923,6 +977,7 @@ def biomaj_daemon_bank_search(bank):
     except Exception as e:
         abort(500, str(e))
 
+
 @app.route('/api/daemon/bank/<bank>/show', methods=['GET'])
 def biomaj_daemon_bank_show(bank):
     (http_code, options, error) = daemon_api_auth(request)
@@ -947,6 +1002,7 @@ def biomaj_daemon_bank_show(bank):
     except Exception as e:
         abort(500, str(e))
 
+
 @app.route('/api/daemon', methods=['POST'])
 def biomaj_daemon():
     '''
@@ -980,7 +1036,6 @@ def biomaj_daemon():
             if user:
                 options_object.user = user['id']
 
-
         if options_object.maintenance in ['on', 'off']:
             if not options_object.user or 'admin' not in config['biomaj'] or options_object.user not in config['biomaj']['admin']:
                 abort(401, {'message': 'This action requires authentication with api key'})

biomaj_daemon/daemon/utils.py

@@ -31,7 +31,7 @@ def biomaj_version(options, config):
     Get biomaj version
     '''
     version = pkg_resources.require('biomaj')[0].version
-    tools = []
+    # tools = []
     biomaj_modules = [
         'biomaj',
         'biomaj-core',
@@ -234,15 +234,23 @@ def biomaj_search(options, config):
                 if prod['session'] == bank['current']:
                     iscurrent = "yes"
                 if options.json:
-                    results.append([b if b else '',
+                    results.append(
+                        [
+                            b if b else '',
                             prod['release'],
                             prod['formats'],
-                        prod['types'], iscurrent])
+                            prod['types'], iscurrent
+                        ]
+                    )
                 else:
-                    results.append([b if b else '',
+                    results.append(
+                        [
+                            b if b else '',
                             prod['release'],
                             ','.join(prod['formats']),
-                        ','.join(prod['types']), iscurrent])
+                            ','.join(prod['types']), iscurrent
+                        ]
+                    )
         if options.json:
             return (True, {'matches': results, 'headers': headers})
 
@@ -433,6 +441,12 @@ def biomaj_whatsup(options, config):
             pending_len = 0
 
         whatsup.append(['queue', str(pending_len), 'waiting'])
+        for index in range(pending_len):
+            pending_action = redis_client.lindex(config['redis']['prefix'] + ':queue', index)
+            if pending_action:
+                pending_bank = json.loads(pending_action)
+                if pending_bank.get('bank', None):
+                    whatsup.append(['queued', pending_bank['bank'], 'waiting'])
 
     if not options.proxy:
         data_dir = BiomajConfig.global_config.get('GENERAL', 'data.dir')
@@ -466,7 +480,7 @@ def biomaj_whatsup(options, config):
 
 def biomaj_send_message(options, config):
     '''
-    Send message to rabbitmq listener
+    Send message to listener
     '''
     if not options.proxy:
         return (False, 'option not allowed without --proxy option')
@@ -751,6 +765,7 @@ def biomaj_update_status(options, config):
                 else:
                     msg.append([step, 'waiting'])
         if step in ['postprocess', 'preprocess', 'removeprocess']:
+            if status_info.get(step, None) and status_info[step].get('progress', None):
                 progress = status_info[step]['progress']
                 for proc in list(progress.keys()):
                     msg.append([proc, str(progress[proc])])
@@ -789,6 +804,7 @@ def biomaj_user_info(options, config):
         msg += 'Api key: ' + str(user['apikey']) + '\n'
     return (True, msg)
 
+
 def biomaj_stats(options, config):
     disk_stats = Bank.get_banks_disk_usage()
     results = []
@@ -810,6 +826,7 @@ def biomaj_stats(options, config):
         msg += tabulate(results, headers="firstrow", tablefmt="grid")
     return (True, msg)
 
+
 def biomaj_history(options, config):
     history = Bank.get_history(options.historyLimit)
     results = []
@@ -818,13 +835,16 @@ def biomaj_history(options, config):
         results.append(headers)
     msg = 'BioMAJ history\n'
     for h in history:
-        results.append([h['bank'],
+        results.append(
+            [
+                h['bank'],
                 h['action'],
                 datetime.datetime.utcfromtimestamp(h['start']),
                 datetime.datetime.utcfromtimestamp(h['end']),
                 str(h['updated']),
                 str(h['error'])
-        ])
+            ]
+        )
     if options.json:
         msg = {'headers': headers, 'history': results}
     else:

debian/changelog

+biomaj3-daemon (3.0.19-1) unstable; urgency=medium
+
+  * New upstream release 
+
+ -- Olivier Sallou <osallou@debian.org>  Sat, 09 Mar 2019 10:23:28 +0000
+
 biomaj3-daemon (3.0.17-1) unstable; urgency=medium
 
   [ Jelmer Vernooij ]

requirements.txt

-biomaj
 biomaj-user
 biomaj-zipkin
 redis
@@ -7,5 +6,5 @@ flask
 python-consul
 prometheus_client>=0.0.18
 requests
-biomaj-core>=3.0.10
-biomaj>=3.1.6
+biomaj-core>=3.0.16
+biomaj>=3.1.7

setup.py

@@ -21,7 +21,7 @@ config = {
     'url': 'http://biomaj.genouest.org',
     'download_url': 'http://biomaj.genouest.org',
     'author_email': 'olivier.sallou@irisa.fr',
-    'version': '3.0.17',
+    'version': '3.0.19',
      'classifiers': [
         # How mature is this project? Common values are
         #   3 - Alpha