Skip to content

Commits on Source 3

Show whitespace changes
Inline Side-by-side
CHANGES.txt
View file @ 83899ece
3.0.10
3.0.11:
Catch exception for microservices if remote server is not reachable
Rename package for protobuf to avoid collisions
3.0.10:
README doc fixes
Add dependency banks additional volumes for Docker
3.0.9:
......
biomaj_process/biomaj_process_web.py
View file @ 83899ece
......@@ -17,7 +17,7 @@ from prometheus_client import CollectorRegistry
import consul
import redis
from biomaj_process.message import message_pb2
from biomaj_process.message import procmessage_pb2
from biomaj_process.process_service import ProcessService
from biomaj_core.utils import Utils
......@@ -97,7 +97,7 @@ def create_session(bank):
@app.route('/api/process/session/<bank>/<session>', methods=['DELETE'])
def clean_session(bank, session):
dserv = ProcessService(config_file, rabbitmq=False)
biomaj_file_info = message_pb2.Process()
biomaj_file_info = procmessage_pb2.Process()
biomaj_file_info.bank = bank
biomaj_file_info.session = session
dserv.clean(biomaj_file_info)
......
biomaj_process/message/message_pb2.py → biomaj_process/message/procmessage_pb2.py
View file @ 83899ece
# Generated by the protocol buffer compiler. DO NOT EDIT!
# source: message.proto
# source: procmessage.proto
import sys
_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1'))
......@@ -16,9 +16,9 @@ _sym_db = _symbol_database.Default()
DESCRIPTOR = _descriptor.FileDescriptor(
name='message.proto',
package='biomaj',
serialized_pb=_b('\n\rmessage.proto\x12\x06\x62iomaj\"\xec\x02\n\x07Process\x12\x0c\n\x04\x62\x61nk\x18\x01 \x02(\t\x12\x0f\n\x07session\x18\x02 \x02(\t\x12\x0f\n\x07log_dir\x18\x03 \x02(\t\x12\x0b\n\x03\x65xe\x18\x04 \x02(\t\x12\x0c\n\x04\x61rgs\x18\x05 \x03(\t\x12(\n\x08\x65nv_vars\x18\x06 \x03(\x0b\x32\x16.biomaj.Process.EnvVar\x12\x1b\n\x0cshell_expand\x18\x07 \x01(\x08:\x05\x66\x61lse\x12\x0c\n\x04name\x18\x08 \x01(\t\x12\x13\n\x0b\x64\x65scription\x18\t \x01(\t\x12\x11\n\tproc_type\x18\n \x01(\t\x12\x18\n\tis_docker\x18\x0b \x01(\x08:\x05\x66\x61lse\x12&\n\x06\x64ocker\x18\x0c \x01(\x0b\x32\x16.biomaj.Process.Docker\x1a%\n\x06\x45nvVar\x12\x0c\n\x04name\x18\x01 \x02(\t\x12\r\n\x05value\x18\x02 \x02(\t\x1a\x30\n\x06\x44ocker\x12\r\n\x05image\x18\x01 \x02(\t\x12\x17\n\x08use_sudo\x18\x02 \x01(\x08:\x05\x66\x61lse\"\xcf\x01\n\tOperation\x12 \n\x07process\x18\x01 \x01(\x0b\x32\x0f.biomaj.Process\x12\x32\n\x04type\x18\x02 \x02(\x0e\x32\x1b.biomaj.Operation.OPERATION:\x07\x45XECUTE\x12&\n\x05trace\x18\x03 \x01(\x0b\x32\x17.biomaj.Operation.Trace\x1a*\n\x05Trace\x12\x10\n\x08trace_id\x18\x01 \x02(\t\x12\x0f\n\x07span_id\x18\x02 \x02(\t\"\x18\n\tOPERATION\x12\x0b\n\x07\x45XECUTE\x10\x01')
name='procmessage.proto',
package='biomaj.process',
serialized_pb=_b('\n\x11procmessage.proto\x12\x0e\x62iomaj.process\"\xfc\x02\n\x07Process\x12\x0c\n\x04\x62\x61nk\x18\x01 \x02(\t\x12\x0f\n\x07session\x18\x02 \x02(\t\x12\x0f\n\x07log_dir\x18\x03 \x02(\t\x12\x0b\n\x03\x65xe\x18\x04 \x02(\t\x12\x0c\n\x04\x61rgs\x18\x05 \x03(\t\x12\x30\n\x08\x65nv_vars\x18\x06 \x03(\x0b\x32\x1e.biomaj.process.Process.EnvVar\x12\x1b\n\x0cshell_expand\x18\x07 \x01(\x08:\x05\x66\x61lse\x12\x0c\n\x04name\x18\x08 \x01(\t\x12\x13\n\x0b\x64\x65scription\x18\t \x01(\t\x12\x11\n\tproc_type\x18\n \x01(\t\x12\x18\n\tis_docker\x18\x0b \x01(\x08:\x05\x66\x61lse\x12.\n\x06\x64ocker\x18\x0c \x01(\x0b\x32\x1e.biomaj.process.Process.Docker\x1a%\n\x06\x45nvVar\x12\x0c\n\x04name\x18\x01 \x02(\t\x12\r\n\x05value\x18\x02 \x02(\t\x1a\x30\n\x06\x44ocker\x12\r\n\x05image\x18\x01 \x02(\t\x12\x17\n\x08use_sudo\x18\x02 \x01(\x08:\x05\x66\x61lse\"\xe7\x01\n\tOperation\x12(\n\x07process\x18\x01 \x01(\x0b\x32\x17.biomaj.process.Process\x12:\n\x04type\x18\x02 \x02(\x0e\x32#.biomaj.process.Operation.OPERATION:\x07\x45XECUTE\x12.\n\x05trace\x18\x03 \x01(\x0b\x32\x1f.biomaj.process.Operation.Trace\x1a*\n\x05Trace\x12\x10\n\x08trace_id\x18\x01 \x02(\t\x12\x0f\n\x07span_id\x18\x02 \x02(\t\"\x18\n\tOPERATION\x12\x0b\n\x07\x45XECUTE\x10\x01')
)
_sym_db.RegisterFileDescriptor(DESCRIPTOR)
......@@ -26,7 +26,7 @@ _sym_db.RegisterFileDescriptor(DESCRIPTOR)
_OPERATION_OPERATION = _descriptor.EnumDescriptor(
name='OPERATION',
full_name='biomaj.Operation.OPERATION',
full_name='biomaj.process.Operation.OPERATION',
filename=None,
file=DESCRIPTOR,
values=[
......@@ -37,28 +37,28 @@ _OPERATION_OPERATION = _descriptor.EnumDescriptor(
],
containing_type=None,
options=None,
serialized_start=576,
serialized_end=600,
serialized_start=628,
serialized_end=652,
)
_sym_db.RegisterEnumDescriptor(_OPERATION_OPERATION)
_PROCESS_ENVVAR = _descriptor.Descriptor(
name='EnvVar',
full_name='biomaj.Process.EnvVar',
full_name='biomaj.process.Process.EnvVar',
filename=None,
file=DESCRIPTOR,
containing_type=None,
fields=[
_descriptor.FieldDescriptor(
name='name', full_name='biomaj.Process.EnvVar.name', index=0,
name='name', full_name='biomaj.process.Process.EnvVar.name', index=0,
number=1, type=9, cpp_type=9, label=2,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None),
_descriptor.FieldDescriptor(
name='value', full_name='biomaj.Process.EnvVar.value', index=1,
name='value', full_name='biomaj.process.Process.EnvVar.value', index=1,
number=2, type=9, cpp_type=9, label=2,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
......@@ -75,26 +75,26 @@ _PROCESS_ENVVAR = _descriptor.Descriptor(
extension_ranges=[],
oneofs=[
],
serialized_start=303,
serialized_end=340,
serialized_start=331,
serialized_end=368,
)
_PROCESS_DOCKER = _descriptor.Descriptor(
name='Docker',
full_name='biomaj.Process.Docker',
full_name='biomaj.process.Process.Docker',
filename=None,
file=DESCRIPTOR,
containing_type=None,
fields=[
_descriptor.FieldDescriptor(
name='image', full_name='biomaj.Process.Docker.image', index=0,
name='image', full_name='biomaj.process.Process.Docker.image', index=0,
number=1, type=9, cpp_type=9, label=2,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None),
_descriptor.FieldDescriptor(
name='use_sudo', full_name='biomaj.Process.Docker.use_sudo', index=1,
name='use_sudo', full_name='biomaj.process.Process.Docker.use_sudo', index=1,
number=2, type=8, cpp_type=7, label=1,
has_default_value=True, default_value=False,
message_type=None, enum_type=None, containing_type=None,
......@@ -111,96 +111,96 @@ _PROCESS_DOCKER = _descriptor.Descriptor(
extension_ranges=[],
oneofs=[
],
serialized_start=342,
serialized_end=390,
serialized_start=370,
serialized_end=418,
)
_PROCESS = _descriptor.Descriptor(
name='Process',
full_name='biomaj.Process',
full_name='biomaj.process.Process',
filename=None,
file=DESCRIPTOR,
containing_type=None,
fields=[
_descriptor.FieldDescriptor(
name='bank', full_name='biomaj.Process.bank', index=0,
name='bank', full_name='biomaj.process.Process.bank', index=0,
number=1, type=9, cpp_type=9, label=2,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None),
_descriptor.FieldDescriptor(
name='session', full_name='biomaj.Process.session', index=1,
name='session', full_name='biomaj.process.Process.session', index=1,
number=2, type=9, cpp_type=9, label=2,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None),
_descriptor.FieldDescriptor(
name='log_dir', full_name='biomaj.Process.log_dir', index=2,
name='log_dir', full_name='biomaj.process.Process.log_dir', index=2,
number=3, type=9, cpp_type=9, label=2,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None),
_descriptor.FieldDescriptor(
name='exe', full_name='biomaj.Process.exe', index=3,
name='exe', full_name='biomaj.process.Process.exe', index=3,
number=4, type=9, cpp_type=9, label=2,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None),
_descriptor.FieldDescriptor(
name='args', full_name='biomaj.Process.args', index=4,
name='args', full_name='biomaj.process.Process.args', index=4,
number=5, type=9, cpp_type=9, label=3,
has_default_value=False, default_value=[],
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None),
_descriptor.FieldDescriptor(
name='env_vars', full_name='biomaj.Process.env_vars', index=5,
name='env_vars', full_name='biomaj.process.Process.env_vars', index=5,
number=6, type=11, cpp_type=10, label=3,
has_default_value=False, default_value=[],
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None),
_descriptor.FieldDescriptor(
name='shell_expand', full_name='biomaj.Process.shell_expand', index=6,
name='shell_expand', full_name='biomaj.process.Process.shell_expand', index=6,
number=7, type=8, cpp_type=7, label=1,
has_default_value=True, default_value=False,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None),
_descriptor.FieldDescriptor(
name='name', full_name='biomaj.Process.name', index=7,
name='name', full_name='biomaj.process.Process.name', index=7,
number=8, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None),
_descriptor.FieldDescriptor(
name='description', full_name='biomaj.Process.description', index=8,
name='description', full_name='biomaj.process.Process.description', index=8,
number=9, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None),
_descriptor.FieldDescriptor(
name='proc_type', full_name='biomaj.Process.proc_type', index=9,
name='proc_type', full_name='biomaj.process.Process.proc_type', index=9,
number=10, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None),
_descriptor.FieldDescriptor(
name='is_docker', full_name='biomaj.Process.is_docker', index=10,
name='is_docker', full_name='biomaj.process.Process.is_docker', index=10,
number=11, type=8, cpp_type=7, label=1,
has_default_value=True, default_value=False,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None),
_descriptor.FieldDescriptor(
name='docker', full_name='biomaj.Process.docker', index=11,
name='docker', full_name='biomaj.process.Process.docker', index=11,
number=12, type=11, cpp_type=10, label=1,
has_default_value=False, default_value=None,
message_type=None, enum_type=None, containing_type=None,
......@@ -217,27 +217,27 @@ _PROCESS = _descriptor.Descriptor(
extension_ranges=[],
oneofs=[
],
serialized_start=26,
serialized_end=390,
serialized_start=38,
serialized_end=418,
)
_OPERATION_TRACE = _descriptor.Descriptor(
name='Trace',
full_name='biomaj.Operation.Trace',
full_name='biomaj.process.Operation.Trace',
filename=None,
file=DESCRIPTOR,
containing_type=None,
fields=[
_descriptor.FieldDescriptor(
name='trace_id', full_name='biomaj.Operation.Trace.trace_id', index=0,
name='trace_id', full_name='biomaj.process.Operation.Trace.trace_id', index=0,
number=1, type=9, cpp_type=9, label=2,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None),
_descriptor.FieldDescriptor(
name='span_id', full_name='biomaj.Operation.Trace.span_id', index=1,
name='span_id', full_name='biomaj.process.Operation.Trace.span_id', index=1,
number=2, type=9, cpp_type=9, label=2,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
......@@ -254,33 +254,33 @@ _OPERATION_TRACE = _descriptor.Descriptor(
extension_ranges=[],
oneofs=[
],
serialized_start=532,
serialized_end=574,
serialized_start=584,
serialized_end=626,
)
_OPERATION = _descriptor.Descriptor(
name='Operation',
full_name='biomaj.Operation',
full_name='biomaj.process.Operation',
filename=None,
file=DESCRIPTOR,
containing_type=None,
fields=[
_descriptor.FieldDescriptor(
name='process', full_name='biomaj.Operation.process', index=0,
name='process', full_name='biomaj.process.Operation.process', index=0,
number=1, type=11, cpp_type=10, label=1,
has_default_value=False, default_value=None,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None),
_descriptor.FieldDescriptor(
name='type', full_name='biomaj.Operation.type', index=1,
name='type', full_name='biomaj.process.Operation.type', index=1,
number=2, type=14, cpp_type=8, label=2,
has_default_value=True, default_value=1,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None),
_descriptor.FieldDescriptor(
name='trace', full_name='biomaj.Operation.trace', index=2,
name='trace', full_name='biomaj.process.Operation.trace', index=2,
number=3, type=11, cpp_type=10, label=1,
has_default_value=False, default_value=None,
message_type=None, enum_type=None, containing_type=None,
......@@ -298,8 +298,8 @@ _OPERATION = _descriptor.Descriptor(
extension_ranges=[],
oneofs=[
],
serialized_start=393,
serialized_end=600,
serialized_start=421,
serialized_end=652,
)
_PROCESS_ENVVAR.containing_type = _PROCESS
......@@ -318,20 +318,20 @@ Process = _reflection.GeneratedProtocolMessageType('Process', (_message.Message,
EnvVar = _reflection.GeneratedProtocolMessageType('EnvVar', (_message.Message,), dict(
DESCRIPTOR = _PROCESS_ENVVAR,
__module__ = 'message_pb2'
# @@protoc_insertion_point(class_scope:biomaj.Process.EnvVar)
__module__ = 'procmessage_pb2'
# @@protoc_insertion_point(class_scope:biomaj.process.Process.EnvVar)
))
,
Docker = _reflection.GeneratedProtocolMessageType('Docker', (_message.Message,), dict(
DESCRIPTOR = _PROCESS_DOCKER,
__module__ = 'message_pb2'
# @@protoc_insertion_point(class_scope:biomaj.Process.Docker)
__module__ = 'procmessage_pb2'
# @@protoc_insertion_point(class_scope:biomaj.process.Process.Docker)
))
,
DESCRIPTOR = _PROCESS,
__module__ = 'message_pb2'
# @@protoc_insertion_point(class_scope:biomaj.Process)
__module__ = 'procmessage_pb2'
# @@protoc_insertion_point(class_scope:biomaj.process.Process)
))
_sym_db.RegisterMessage(Process)
_sym_db.RegisterMessage(Process.EnvVar)
......@@ -341,13 +341,13 @@ Operation = _reflection.GeneratedProtocolMessageType('Operation', (_message.Mess
Trace = _reflection.GeneratedProtocolMessageType('Trace', (_message.Message,), dict(
DESCRIPTOR = _OPERATION_TRACE,
__module__ = 'message_pb2'
# @@protoc_insertion_point(class_scope:biomaj.Operation.Trace)
__module__ = 'procmessage_pb2'
# @@protoc_insertion_point(class_scope:biomaj.process.Operation.Trace)
))
,
DESCRIPTOR = _OPERATION,
__module__ = 'message_pb2'
# @@protoc_insertion_point(class_scope:biomaj.Operation)
__module__ = 'procmessage_pb2'
# @@protoc_insertion_point(class_scope:biomaj.process.Operation)
))
_sym_db.RegisterMessage(Operation)
_sym_db.RegisterMessage(Operation.Trace)
......
biomaj_process/process.py
View file @ 83899ece
......@@ -141,11 +141,15 @@ class RemoteProcess(Process):
def run(self, simulate=False):
psc = ProcessServiceClient(self.rabbit_mq, self.rabbit_mq_port, self.rabbit_mq_virtualhost, self.rabbit_mq_user, self.rabbit_mq_password)
try:
session = psc.create_session(self.bank, self.proxy)
from biomaj_process.message import message_pb2
biomaj_process = message_pb2.Operation()
except Exception as e:
logging.exception('Process:RemoteProcess:Session:Create:Error:' + str(e))
return False
from biomaj_process.message import procmessage_pb2
biomaj_process = procmessage_pb2.Operation()
biomaj_process.type = 1
process = message_pb2.Process()
process = procmessage_pb2.Process()
process.bank = self.bank
process.session = session
process.log_dir = self.log_dir
......@@ -163,18 +167,23 @@ class RemoteProcess(Process):
process.proc_type = str(self.proc_type)
if self.docker:
process.is_docker = True
docker_info = message_pb2.Process.Docker()
docker_info = procmessage_pb2.Process.Docker()
docker_info.image = self.docker
docker_info.use_sudo = self.docker_sudo
process.docker.MergeFrom(docker_info)
biomaj_process.process.MergeFrom(process)
if self.trace_id:
trace = message_pb2.Operation.Trace()
trace = procmessage_pb2.Operation.Trace()
trace.trace_id = self.trace_id
trace.span_id = self.parent_id
biomaj_process.trace.MergeFrom(trace)
psc.execute_process(biomaj_process)
exitcode = 0
try:
(exitcode, info) = psc.wait_for_process()
except Exception as e:
logging.exception('Error during process execution: ' + str(e))
exitcode = 1
psc.clean()
if exitcode > 0:
return False
......
biomaj_process/process_client.py
View file @ 83899ece
......@@ -104,4 +104,4 @@ class ProcessServiceClient(object):
return
except Exception:
logging.exception('Failed to send clean operation: %s' % (url))
raise Exception('Failed to connect to the process proxy')
logging.error('Process:RemoteProcess:Session:Clean:Error:' + self.bank + '/' + self.session)
biomaj_process/process_service.py
View file @ 83899ece
......@@ -11,7 +11,7 @@ from flask import Flask
from flask import jsonify
import consul
from biomaj_process.message import message_pb2
from biomaj_process.message import procmessage_pb2
from biomaj_process.process import Process
from biomaj_process.process import DockerProcess
from biomaj_core.utils import Utils
......@@ -225,7 +225,7 @@ class ProcessService(object):
Manage download and send ACK message
'''
try:
operation = message_pb2.Operation()
operation = procmessage_pb2.Operation()
operation.ParseFromString(body)
# self.logger.debug('Received message: %s' % (operation))
if operation.type == 1:
......
debian/changelog
View file @ 83899ece
biomaj3-process (3.0.10-2) UNRELEASED; urgency=medium
biomaj3-process (3.0.11-1) unstable; urgency=medium
[ Jelmer Vernooij ]
* Use secure copyright file specification URI.
* Trim trailing whitespace.
-- Jelmer Vernooij <jelmer@debian.org> Sat, 20 Oct 2018 13:15:56 +0000
[ Olivier Sallou ]
* New upstream release
-- Olivier Sallou <osallou@debian.org> Thu, 25 Oct 2018 09:08:51 +0000
biomaj3-process (3.0.10-1) unstable; urgency=medium
......
debian/control
View file @ 83899ece
......@@ -22,8 +22,8 @@ Build-Depends: debhelper (>= 9), dh-python,
python3-biomaj3-zipkin
Standards-Version: 4.1.3
Homepage: https://github.com/genouest/biomaj-process
Vcs-Browser: https://anonscm.debian.org/cgit/debian-med/biomaj3-process.git
Vcs-Git: https://anonscm.debian.org/git/debian-med/biomaj3-process.git
Vcs-Browser: https://salsa.debian.org/med-team/biomaj3-process
Vcs-Git: https://salsa.debian.org/med-team/biomaj3-process.git
Package: python3-biomaj3-process
Architecture: all
......
debian/rules
View file @ 83899ece
......@@ -7,7 +7,7 @@ export PYBUILD_NAME=biomaj-process
dh $@ --with python3 --buildsystem=pybuild
override_dh_auto_build:
cd biomaj_process/message && protoc --python_out=. message.proto
cd biomaj_process/message && protoc --python_out=. procmessage.proto
dh_auto_build
override_dh_install:
......
setup.py
View file @ 83899ece
......@@ -21,7 +21,7 @@ config = {
'url': 'http://biomaj.genouest.org',
'download_url': 'http://biomaj.genouest.org',
'author_email': 'olivier.sallou@irisa.fr',
'version': '3.0.10',
'version': '3.0.11',
'classifiers': [
# How mature is this project? Common values are
# 3 - Alpha
......
tests/biomaj_tests.py
View file @ 83899ece
......@@ -12,7 +12,7 @@ import time
from mock import patch
from biomaj_process.message import message_pb2
from biomaj_process.message import procmessage_pb2
from biomaj_process.process_service import ProcessService
import unittest
......@@ -34,7 +34,7 @@ class TestBiomajProcess(unittest.TestCase):
"""
Checks bank init
"""
msg = message_pb2.Process()
msg = procmessage_pb2.Process()
msg.bank = 'test'
msg.session = self.session
msg.log_dir = self.test_dir
......