Commit 0434aec8 authored by Olivier Sallou's avatar Olivier Sallou
Browse files

New upstream version 3.0.11

parent 1dcde4b2

CHANGES.txt

+4 −1
Original line number Diff line number Diff line 
3.0.10

3.0.11:

  Catch exception for microservices if remote server is not reachable

  Rename package for protobuf to avoid collisions

3.0.10:

  README doc fixes

  Add dependency banks additional volumes for Docker

3.0.9:

biomaj_process/biomaj_process_web.py

+2 −2
Original line number Diff line number Diff line
@@ -17,7 +17,7 @@ from prometheus_client import CollectorRegistry 
import consul

import redis



from biomaj_process.message import message_pb2

from biomaj_process.message import procmessage_pb2

from biomaj_process.process_service import ProcessService

from biomaj_core.utils import Utils
@@ -97,7 +97,7 @@ def create_session(bank): 
@app.route('/api/process/session/<bank>/<session>', methods=['DELETE'])

def clean_session(bank, session):

    dserv = ProcessService(config_file, rabbitmq=False)

    biomaj_file_info = message_pb2.Process()

    biomaj_file_info = procmessage_pb2.Process()

    biomaj_file_info.bank = bank

    biomaj_file_info.session = session

    dserv.clean(biomaj_file_info)

biomaj_process/message/message.proto→biomaj_process/message/procmessage.proto

+1 −1
Original line number Diff line number Diff line 
package biomaj;

package biomaj.process;



message Process {

biomaj_process/message/message_pb2.py→biomaj_process/message/procmessage_pb2.py

+53 −53
Original line number Diff line number Diff line 
# Generated by the protocol buffer compiler.  DO NOT EDIT!

# source: message.proto

# source: procmessage.proto



import sys

_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1'))
@@ -16,9 +16,9 @@ _sym_db = _symbol_database.Default() 




DESCRIPTOR = _descriptor.FileDescriptor(

  name='message.proto',

  package='biomaj',

  serialized_pb=_b('\n\rmessage.proto\x12\x06\x62iomaj\"\xec\x02\n\x07Process\x12\x0c\n\x04\x62\x61nk\x18\x01 \x02(\t\x12\x0f\n\x07session\x18\x02 \x02(\t\x12\x0f\n\x07log_dir\x18\x03 \x02(\t\x12\x0b\n\x03\x65xe\x18\x04 \x02(\t\x12\x0c\n\x04\x61rgs\x18\x05 \x03(\t\x12(\n\x08\x65nv_vars\x18\x06 \x03(\x0b\x32\x16.biomaj.Process.EnvVar\x12\x1b\n\x0cshell_expand\x18\x07 \x01(\x08:\x05\x66\x61lse\x12\x0c\n\x04name\x18\x08 \x01(\t\x12\x13\n\x0b\x64\x65scription\x18\t \x01(\t\x12\x11\n\tproc_type\x18\n \x01(\t\x12\x18\n\tis_docker\x18\x0b \x01(\x08:\x05\x66\x61lse\x12&\n\x06\x64ocker\x18\x0c \x01(\x0b\x32\x16.biomaj.Process.Docker\x1a%\n\x06\x45nvVar\x12\x0c\n\x04name\x18\x01 \x02(\t\x12\r\n\x05value\x18\x02 \x02(\t\x1a\x30\n\x06\x44ocker\x12\r\n\x05image\x18\x01 \x02(\t\x12\x17\n\x08use_sudo\x18\x02 \x01(\x08:\x05\x66\x61lse\"\xcf\x01\n\tOperation\x12 \n\x07process\x18\x01 \x01(\x0b\x32\x0f.biomaj.Process\x12\x32\n\x04type\x18\x02 \x02(\x0e\x32\x1b.biomaj.Operation.OPERATION:\x07\x45XECUTE\x12&\n\x05trace\x18\x03 \x01(\x0b\x32\x17.biomaj.Operation.Trace\x1a*\n\x05Trace\x12\x10\n\x08trace_id\x18\x01 \x02(\t\x12\x0f\n\x07span_id\x18\x02 \x02(\t\"\x18\n\tOPERATION\x12\x0b\n\x07\x45XECUTE\x10\x01')

  name='procmessage.proto',

  package='biomaj.process',

  serialized_pb=_b('\n\x11procmessage.proto\x12\x0e\x62iomaj.process\"\xfc\x02\n\x07Process\x12\x0c\n\x04\x62\x61nk\x18\x01 \x02(\t\x12\x0f\n\x07session\x18\x02 \x02(\t\x12\x0f\n\x07log_dir\x18\x03 \x02(\t\x12\x0b\n\x03\x65xe\x18\x04 \x02(\t\x12\x0c\n\x04\x61rgs\x18\x05 \x03(\t\x12\x30\n\x08\x65nv_vars\x18\x06 \x03(\x0b\x32\x1e.biomaj.process.Process.EnvVar\x12\x1b\n\x0cshell_expand\x18\x07 \x01(\x08:\x05\x66\x61lse\x12\x0c\n\x04name\x18\x08 \x01(\t\x12\x13\n\x0b\x64\x65scription\x18\t \x01(\t\x12\x11\n\tproc_type\x18\n \x01(\t\x12\x18\n\tis_docker\x18\x0b \x01(\x08:\x05\x66\x61lse\x12.\n\x06\x64ocker\x18\x0c \x01(\x0b\x32\x1e.biomaj.process.Process.Docker\x1a%\n\x06\x45nvVar\x12\x0c\n\x04name\x18\x01 \x02(\t\x12\r\n\x05value\x18\x02 \x02(\t\x1a\x30\n\x06\x44ocker\x12\r\n\x05image\x18\x01 \x02(\t\x12\x17\n\x08use_sudo\x18\x02 \x01(\x08:\x05\x66\x61lse\"\xe7\x01\n\tOperation\x12(\n\x07process\x18\x01 \x01(\x0b\x32\x17.biomaj.process.Process\x12:\n\x04type\x18\x02 \x02(\x0e\x32#.biomaj.process.Operation.OPERATION:\x07\x45XECUTE\x12.\n\x05trace\x18\x03 \x01(\x0b\x32\x1f.biomaj.process.Operation.Trace\x1a*\n\x05Trace\x12\x10\n\x08trace_id\x18\x01 \x02(\t\x12\x0f\n\x07span_id\x18\x02 \x02(\t\"\x18\n\tOPERATION\x12\x0b\n\x07\x45XECUTE\x10\x01')

)

_sym_db.RegisterFileDescriptor(DESCRIPTOR)
@@ -26,7 +26,7 @@ _sym_db.RegisterFileDescriptor(DESCRIPTOR) 


_OPERATION_OPERATION = _descriptor.EnumDescriptor(

  name='OPERATION',

  full_name='biomaj.Operation.OPERATION',

  full_name='biomaj.process.Operation.OPERATION',

  filename=None,

  file=DESCRIPTOR,

  values=[
@@ -37,28 +37,28 @@ _OPERATION_OPERATION = _descriptor.EnumDescriptor( 
  ],

  containing_type=None,

  options=None,

  serialized_start=576,

  serialized_end=600,

  serialized_start=628,

  serialized_end=652,

)

_sym_db.RegisterEnumDescriptor(_OPERATION_OPERATION)





_PROCESS_ENVVAR = _descriptor.Descriptor(

  name='EnvVar',

  full_name='biomaj.Process.EnvVar',

  full_name='biomaj.process.Process.EnvVar',

  filename=None,

  file=DESCRIPTOR,

  containing_type=None,

  fields=[

    _descriptor.FieldDescriptor(

      name='name', full_name='biomaj.Process.EnvVar.name', index=0,

      name='name', full_name='biomaj.process.Process.EnvVar.name', index=0,

      number=1, type=9, cpp_type=9, label=2,

      has_default_value=False, default_value=_b("").decode('utf-8'),

      message_type=None, enum_type=None, containing_type=None,

      is_extension=False, extension_scope=None,

      options=None),

    _descriptor.FieldDescriptor(

      name='value', full_name='biomaj.Process.EnvVar.value', index=1,

      name='value', full_name='biomaj.process.Process.EnvVar.value', index=1,

      number=2, type=9, cpp_type=9, label=2,

      has_default_value=False, default_value=_b("").decode('utf-8'),

      message_type=None, enum_type=None, containing_type=None,
@@ -75,26 +75,26 @@ _PROCESS_ENVVAR = _descriptor.Descriptor( 
  extension_ranges=[],

  oneofs=[

  ],

  serialized_start=303,

  serialized_end=340,

  serialized_start=331,

  serialized_end=368,

)



_PROCESS_DOCKER = _descriptor.Descriptor(

  name='Docker',

  full_name='biomaj.Process.Docker',

  full_name='biomaj.process.Process.Docker',

  filename=None,

  file=DESCRIPTOR,

  containing_type=None,

  fields=[

    _descriptor.FieldDescriptor(

      name='image', full_name='biomaj.Process.Docker.image', index=0,

      name='image', full_name='biomaj.process.Process.Docker.image', index=0,

      number=1, type=9, cpp_type=9, label=2,

      has_default_value=False, default_value=_b("").decode('utf-8'),

      message_type=None, enum_type=None, containing_type=None,

      is_extension=False, extension_scope=None,

      options=None),

    _descriptor.FieldDescriptor(

      name='use_sudo', full_name='biomaj.Process.Docker.use_sudo', index=1,

      name='use_sudo', full_name='biomaj.process.Process.Docker.use_sudo', index=1,

      number=2, type=8, cpp_type=7, label=1,

      has_default_value=True, default_value=False,

      message_type=None, enum_type=None, containing_type=None,
@@ -111,96 +111,96 @@ _PROCESS_DOCKER = _descriptor.Descriptor( 
  extension_ranges=[],

  oneofs=[

  ],

  serialized_start=342,

  serialized_end=390,

  serialized_start=370,

  serialized_end=418,

)



_PROCESS = _descriptor.Descriptor(

  name='Process',

  full_name='biomaj.Process',

  full_name='biomaj.process.Process',

  filename=None,

  file=DESCRIPTOR,

  containing_type=None,

  fields=[

    _descriptor.FieldDescriptor(

      name='bank', full_name='biomaj.Process.bank', index=0,

      name='bank', full_name='biomaj.process.Process.bank', index=0,

      number=1, type=9, cpp_type=9, label=2,

      has_default_value=False, default_value=_b("").decode('utf-8'),

      message_type=None, enum_type=None, containing_type=None,

      is_extension=False, extension_scope=None,

      options=None),

    _descriptor.FieldDescriptor(

      name='session', full_name='biomaj.Process.session', index=1,

      name='session', full_name='biomaj.process.Process.session', index=1,

      number=2, type=9, cpp_type=9, label=2,

      has_default_value=False, default_value=_b("").decode('utf-8'),

      message_type=None, enum_type=None, containing_type=None,

      is_extension=False, extension_scope=None,

      options=None),

    _descriptor.FieldDescriptor(

      name='log_dir', full_name='biomaj.Process.log_dir', index=2,

      name='log_dir', full_name='biomaj.process.Process.log_dir', index=2,

      number=3, type=9, cpp_type=9, label=2,

      has_default_value=False, default_value=_b("").decode('utf-8'),

      message_type=None, enum_type=None, containing_type=None,

      is_extension=False, extension_scope=None,

      options=None),

    _descriptor.FieldDescriptor(

      name='exe', full_name='biomaj.Process.exe', index=3,

      name='exe', full_name='biomaj.process.Process.exe', index=3,

      number=4, type=9, cpp_type=9, label=2,

      has_default_value=False, default_value=_b("").decode('utf-8'),

      message_type=None, enum_type=None, containing_type=None,

      is_extension=False, extension_scope=None,

      options=None),

    _descriptor.FieldDescriptor(

      name='args', full_name='biomaj.Process.args', index=4,

      name='args', full_name='biomaj.process.Process.args', index=4,

      number=5, type=9, cpp_type=9, label=3,

      has_default_value=False, default_value=[],

      message_type=None, enum_type=None, containing_type=None,

      is_extension=False, extension_scope=None,

      options=None),

    _descriptor.FieldDescriptor(

      name='env_vars', full_name='biomaj.Process.env_vars', index=5,

      name='env_vars', full_name='biomaj.process.Process.env_vars', index=5,

      number=6, type=11, cpp_type=10, label=3,

      has_default_value=False, default_value=[],

      message_type=None, enum_type=None, containing_type=None,

      is_extension=False, extension_scope=None,

      options=None),

    _descriptor.FieldDescriptor(

      name='shell_expand', full_name='biomaj.Process.shell_expand', index=6,

      name='shell_expand', full_name='biomaj.process.Process.shell_expand', index=6,

      number=7, type=8, cpp_type=7, label=1,

      has_default_value=True, default_value=False,

      message_type=None, enum_type=None, containing_type=None,

      is_extension=False, extension_scope=None,

      options=None),

    _descriptor.FieldDescriptor(

      name='name', full_name='biomaj.Process.name', index=7,

      name='name', full_name='biomaj.process.Process.name', index=7,

      number=8, type=9, cpp_type=9, label=1,

      has_default_value=False, default_value=_b("").decode('utf-8'),

      message_type=None, enum_type=None, containing_type=None,

      is_extension=False, extension_scope=None,

      options=None),

    _descriptor.FieldDescriptor(

      name='description', full_name='biomaj.Process.description', index=8,

      name='description', full_name='biomaj.process.Process.description', index=8,

      number=9, type=9, cpp_type=9, label=1,

      has_default_value=False, default_value=_b("").decode('utf-8'),

      message_type=None, enum_type=None, containing_type=None,

      is_extension=False, extension_scope=None,

      options=None),

    _descriptor.FieldDescriptor(

      name='proc_type', full_name='biomaj.Process.proc_type', index=9,

      name='proc_type', full_name='biomaj.process.Process.proc_type', index=9,

      number=10, type=9, cpp_type=9, label=1,

      has_default_value=False, default_value=_b("").decode('utf-8'),

      message_type=None, enum_type=None, containing_type=None,

      is_extension=False, extension_scope=None,

      options=None),

    _descriptor.FieldDescriptor(

      name='is_docker', full_name='biomaj.Process.is_docker', index=10,

      name='is_docker', full_name='biomaj.process.Process.is_docker', index=10,

      number=11, type=8, cpp_type=7, label=1,

      has_default_value=True, default_value=False,

      message_type=None, enum_type=None, containing_type=None,

      is_extension=False, extension_scope=None,

      options=None),

    _descriptor.FieldDescriptor(

      name='docker', full_name='biomaj.Process.docker', index=11,

      name='docker', full_name='biomaj.process.Process.docker', index=11,

      number=12, type=11, cpp_type=10, label=1,

      has_default_value=False, default_value=None,

      message_type=None, enum_type=None, containing_type=None,
@@ -217,27 +217,27 @@ _PROCESS = _descriptor.Descriptor( 
  extension_ranges=[],

  oneofs=[

  ],

  serialized_start=26,

  serialized_end=390,

  serialized_start=38,

  serialized_end=418,

)





_OPERATION_TRACE = _descriptor.Descriptor(

  name='Trace',

  full_name='biomaj.Operation.Trace',

  full_name='biomaj.process.Operation.Trace',

  filename=None,

  file=DESCRIPTOR,

  containing_type=None,

  fields=[

    _descriptor.FieldDescriptor(

      name='trace_id', full_name='biomaj.Operation.Trace.trace_id', index=0,

      name='trace_id', full_name='biomaj.process.Operation.Trace.trace_id', index=0,

      number=1, type=9, cpp_type=9, label=2,

      has_default_value=False, default_value=_b("").decode('utf-8'),

      message_type=None, enum_type=None, containing_type=None,

      is_extension=False, extension_scope=None,

      options=None),

    _descriptor.FieldDescriptor(

      name='span_id', full_name='biomaj.Operation.Trace.span_id', index=1,

      name='span_id', full_name='biomaj.process.Operation.Trace.span_id', index=1,

      number=2, type=9, cpp_type=9, label=2,

      has_default_value=False, default_value=_b("").decode('utf-8'),

      message_type=None, enum_type=None, containing_type=None,
@@ -254,33 +254,33 @@ _OPERATION_TRACE = _descriptor.Descriptor( 
  extension_ranges=[],

  oneofs=[

  ],

  serialized_start=532,

  serialized_end=574,

  serialized_start=584,

  serialized_end=626,

)



_OPERATION = _descriptor.Descriptor(

  name='Operation',

  full_name='biomaj.Operation',

  full_name='biomaj.process.Operation',

  filename=None,

  file=DESCRIPTOR,

  containing_type=None,

  fields=[

    _descriptor.FieldDescriptor(

      name='process', full_name='biomaj.Operation.process', index=0,

      name='process', full_name='biomaj.process.Operation.process', index=0,

      number=1, type=11, cpp_type=10, label=1,

      has_default_value=False, default_value=None,

      message_type=None, enum_type=None, containing_type=None,

      is_extension=False, extension_scope=None,

      options=None),

    _descriptor.FieldDescriptor(

      name='type', full_name='biomaj.Operation.type', index=1,

      name='type', full_name='biomaj.process.Operation.type', index=1,

      number=2, type=14, cpp_type=8, label=2,

      has_default_value=True, default_value=1,

      message_type=None, enum_type=None, containing_type=None,

      is_extension=False, extension_scope=None,

      options=None),

    _descriptor.FieldDescriptor(

      name='trace', full_name='biomaj.Operation.trace', index=2,

      name='trace', full_name='biomaj.process.Operation.trace', index=2,

      number=3, type=11, cpp_type=10, label=1,

      has_default_value=False, default_value=None,

      message_type=None, enum_type=None, containing_type=None,
@@ -298,8 +298,8 @@ _OPERATION = _descriptor.Descriptor( 
  extension_ranges=[],

  oneofs=[

  ],

  serialized_start=393,

  serialized_end=600,

  serialized_start=421,

  serialized_end=652,

)



_PROCESS_ENVVAR.containing_type = _PROCESS
@@ -318,20 +318,20 @@ Process = _reflection.GeneratedProtocolMessageType('Process', (_message.Message, 


  EnvVar = _reflection.GeneratedProtocolMessageType('EnvVar', (_message.Message,), dict(

    DESCRIPTOR = _PROCESS_ENVVAR,

    __module__ = 'message_pb2'

    # @@protoc_insertion_point(class_scope:biomaj.Process.EnvVar)

    __module__ = 'procmessage_pb2'

    # @@protoc_insertion_point(class_scope:biomaj.process.Process.EnvVar)

    ))

  ,



  Docker = _reflection.GeneratedProtocolMessageType('Docker', (_message.Message,), dict(

    DESCRIPTOR = _PROCESS_DOCKER,

    __module__ = 'message_pb2'

    # @@protoc_insertion_point(class_scope:biomaj.Process.Docker)

    __module__ = 'procmessage_pb2'

    # @@protoc_insertion_point(class_scope:biomaj.process.Process.Docker)

    ))

  ,

  DESCRIPTOR = _PROCESS,

  __module__ = 'message_pb2'

  # @@protoc_insertion_point(class_scope:biomaj.Process)

  __module__ = 'procmessage_pb2'

  # @@protoc_insertion_point(class_scope:biomaj.process.Process)

  ))

_sym_db.RegisterMessage(Process)

_sym_db.RegisterMessage(Process.EnvVar)
@@ -341,13 +341,13 @@ Operation = _reflection.GeneratedProtocolMessageType('Operation', (_message.Mess 


  Trace = _reflection.GeneratedProtocolMessageType('Trace', (_message.Message,), dict(

    DESCRIPTOR = _OPERATION_TRACE,

    __module__ = 'message_pb2'

    # @@protoc_insertion_point(class_scope:biomaj.Operation.Trace)

    __module__ = 'procmessage_pb2'

    # @@protoc_insertion_point(class_scope:biomaj.process.Operation.Trace)

    ))

  ,

  DESCRIPTOR = _OPERATION,

  __module__ = 'message_pb2'

  # @@protoc_insertion_point(class_scope:biomaj.Operation)

  __module__ = 'procmessage_pb2'

  # @@protoc_insertion_point(class_scope:biomaj.process.Operation)

  ))

_sym_db.RegisterMessage(Operation)

_sym_db.RegisterMessage(Operation.Trace)

biomaj_process/process.py

+16 −7
Original line number Diff line number Diff line
@@ -141,11 +141,15 @@ class RemoteProcess(Process): 


    def run(self, simulate=False):

        psc = ProcessServiceClient(self.rabbit_mq, self.rabbit_mq_port, self.rabbit_mq_virtualhost, self.rabbit_mq_user, self.rabbit_mq_password)

        try:

            session = psc.create_session(self.bank, self.proxy)

        from biomaj_process.message import message_pb2

        biomaj_process = message_pb2.Operation()

        except Exception as e:

            logging.exception('Process:RemoteProcess:Session:Create:Error:' + str(e))

            return False

        from biomaj_process.message import procmessage_pb2

        biomaj_process = procmessage_pb2.Operation()

        biomaj_process.type = 1

        process = message_pb2.Process()

        process = procmessage_pb2.Process()

        process.bank = self.bank

        process.session = session

        process.log_dir = self.log_dir
@@ -163,18 +167,23 @@ class RemoteProcess(Process): 
        process.proc_type = str(self.proc_type)

        if self.docker:

            process.is_docker = True

            docker_info = message_pb2.Process.Docker()

            docker_info = procmessage_pb2.Process.Docker()

            docker_info.image = self.docker

            docker_info.use_sudo = self.docker_sudo

            process.docker.MergeFrom(docker_info)

        biomaj_process.process.MergeFrom(process)

        if self.trace_id:

            trace = message_pb2.Operation.Trace()

            trace = procmessage_pb2.Operation.Trace()

            trace.trace_id = self.trace_id

            trace.span_id = self.parent_id

            biomaj_process.trace.MergeFrom(trace)

        psc.execute_process(biomaj_process)

        exitcode = 0

        try:

            (exitcode, info) = psc.wait_for_process()

        except Exception as e:

            logging.exception('Error during process execution: ' + str(e))

            exitcode = 1

        psc.clean()

        if exitcode > 0:

            return False