From fc9d86a4b7655f03108f8e09c67bc51f96e2e457 Mon Sep 17 00:00:00 2001 From: Simeon Gourlin Date: Thu, 10 Oct 2019 10:37:40 +0200 Subject: [PATCH 01/18] Change some volume LVM values --- puppet/manifests/volume.pp | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/puppet/manifests/volume.pp b/puppet/manifests/volume.pp index 4596ef37..c169a4f8 100644 --- a/puppet/manifests/volume.pp +++ b/puppet/manifests/volume.pp @@ -157,7 +157,8 @@ class oci::volume( # Clear volumes on delete (for data security) class { '::cinder::volume': - volume_clear => 'zero', + volume_clear => 'zero', + volume_clear_ionice => '-c3', } # A cinder-backup service on each volume nodes @@ -200,8 +201,11 @@ class oci::volume( volume_group => $vgname, iscsi_protocol => 'iscsi', extra_options => { - 'LVM_1/reserved_percentage' => { 'value' => '10' }, - 'LVM_1/volume_clear_size' => { 'value' => '90' } + 'LVM_1/reserved_percentage' => { 'value' => '10' }, + 'LVM_1/volume_clear_size' => { 'value' => '0' }, + 'LVM_1/max_over_subscription_ratio' => { 'value' => '4.0' }, + 'LVM_1/max_luns_per_storage_group' => { 'value' => '200' }, + 'LVM_1/check_max_pool_luns_threshold' => { 'value' => true } }, manage_volume_type => true, } -- GitLab From fcf183ce658803533dd222865444b3cad099c769 Mon Sep 17 00:00:00 2001 From: Simeon Gourlin Date: Wed, 23 Oct 2019 14:20:12 +0200 Subject: [PATCH 02/18] Added nested virtualisation with kvm_intel --- puppet/manifests/compute.pp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/puppet/manifests/compute.pp b/puppet/manifests/compute.pp index 0fbbaec3..7cf79ef8 100644 --- a/puppet/manifests/compute.pp +++ b/puppet/manifests/compute.pp @@ -298,6 +298,10 @@ class oci::compute( match => '.*START_DELAY.*=.*', line => 'START_DELAY=4', } + # Enable nested virtualisation with intel_kvm + file { '/etc/modprobe.d/kvm.conf': + content => 'options kvm_intel nested=1', + } if $openstack_release == 'rocky'{ class { '::nova::network::neutron': -- GitLab From 62d2cb583f81806da99bd0d11642ce9a106298e5 Mon Sep 17 00:00:00 2001 From: Thomas Goirand Date: Wed, 23 Oct 2019 17:48:16 +0200 Subject: [PATCH 03/18] Revert "Added nested virtualisation with kvm_intel" This reverts commit fcf183ce658803533dd222865444b3cad099c769. --- puppet/manifests/compute.pp | 4 ---- 1 file changed, 4 deletions(-) diff --git a/puppet/manifests/compute.pp b/puppet/manifests/compute.pp index 7cf79ef8..0fbbaec3 100644 --- a/puppet/manifests/compute.pp +++ b/puppet/manifests/compute.pp @@ -298,10 +298,6 @@ class oci::compute( match => '.*START_DELAY.*=.*', line => 'START_DELAY=4', } - # Enable nested virtualisation with intel_kvm - file { '/etc/modprobe.d/kvm.conf': - content => 'options kvm_intel nested=1', - } if $openstack_release == 'rocky'{ class { '::nova::network::neutron': -- GitLab From b67395c4ee1960c1a54dcb324111d646a6eae0e4 Mon Sep 17 00:00:00 2001 From: Simeon Gourlin Date: Mon, 4 Nov 2019 13:37:43 +0100 Subject: [PATCH 04/18] Add chmod after copy function for ssh private key on controllers --- src/inc/slave_actions.php | 1 + 1 file changed, 1 insertion(+) diff --git a/src/inc/slave_actions.php b/src/inc/slave_actions.php index 1173e3f0..5c7d3d6e 100644 --- a/src/inc/slave_actions.php +++ b/src/inc/slave_actions.php @@ -1674,6 +1674,7 @@ ExecStartPre=/usr/bin/oci-fix-ceph-dm-perms %i mkdir("$template_path/oci-in-target/root"); mkdir("$template_path/oci-in-target/root/.ssh", 0700); copy("$ssh_key_dir/id_rsa", "$template_path/oci-in-target/root/.ssh/id_rsa"); + chmod("$template_path/oci-in-target/root/.ssh/id_rsa", 0600); if(file_exists("$ssh_key_dir/id_rsa.pub")){ copy("$ssh_key_dir/id_rsa.pub", "$template_path/oci-in-target/root/.ssh/id_rsa.pub"); } -- GitLab From 63543eff9ca8dbdf2f102981ffbe4490c214e8d6 Mon Sep 17 00:00:00 2001 From: Thomas Goirand Date: Thu, 18 Mar 2021 17:38:36 +0100 Subject: [PATCH 05/18] Configure the compute pollsters. --- puppet/manifests/compute.pp | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/puppet/manifests/compute.pp b/puppet/manifests/compute.pp index 65c12dfb..bf93fa44 100644 --- a/puppet/manifests/compute.pp +++ b/puppet/manifests/compute.pp @@ -1069,11 +1069,43 @@ vfio-pci instance_discovery_method => 'libvirt_metadata', } }else{ + $compute_pollsters_list = [ + # Libvirt / compute + 'cpu', + 'cpu.delta', + 'cpu_l3_cache', + 'vcpus', + 'memory.usage', + 'network.incoming.bytes', + 'network.incoming.packets', + 'network.incoming.bytes.rate', + 'network.outgoing.bytes', + 'network.outgoing.packets', + 'network.outgoing.bytes.rate', + 'disk.read.bytes', + 'disk.read.bytes.rate', + 'disk.read.requests', + 'disk.read.requests.rate', + 'disk.write.bytes', + 'disk.write.bytes.rate', + 'disk.write.requests', + 'disk.write.requests.rate', + 'disk.device.read.requests', + 'disk.device.read.requests.rate', + 'disk.device.write.requests', + 'disk.device.write.requests.rate', + 'disk.device.read.bytes', + 'disk.device.read.bytes.rate', + 'disk.device.write.bytes', + 'disk.device.write.bytes.rate', + ] class { '::ceilometer::agent::polling': central_namespace => false, compute_namespace => true, ipmi_namespace => false, instance_discovery_method => 'libvirt_metadata', + manage_polling => true, + polling_meters => $compute_pollsters_list, } } } -- GitLab From 6e89976483de7931eea0c3abedfe62f9ce6ea79d Mon Sep 17 00:00:00 2001 From: Thomas Goirand Date: Thu, 8 Apr 2021 19:34:37 +0200 Subject: [PATCH 06/18] Add filebeat support. --- debian/control | 1 + .../openstack-cluster-installer.conf | 4 + puppet/manifests/filebeat.pp | 450 ++++++++++++++++++ src/inc/enc.php | 146 ++++++ src/variables.json | 40 ++ 5 files changed, 641 insertions(+) create mode 100644 puppet/manifests/filebeat.pp diff --git a/debian/control b/debian/control index 8e8f192d..5b12cf66 100644 --- a/debian/control +++ b/debian/control @@ -294,6 +294,7 @@ Depends: puppet-module-etcddiscovery, puppet-module-magnum, puppet-module-tempest, + puppet-module-pcfens-filebeat, puppet-module-placement, puppet-module-puppetlabs-mysql (>= 8.1.0), puppet-module-saz-ssh, diff --git a/etc/openstack-cluster-installer/openstack-cluster-installer.conf b/etc/openstack-cluster-installer/openstack-cluster-installer.conf index bfa0c15d..db0b19b8 100644 --- a/etc/openstack-cluster-installer/openstack-cluster-installer.conf +++ b/etc/openstack-cluster-installer/openstack-cluster-installer.conf @@ -251,3 +251,7 @@ auto_install_machines_os=no # After how many runs of the hardware discovery should the # servers be installed? auto_install_machines_num_of_discovery=15 + +[filebeat] +# Repository URL for Filebeat. +filebeat_repo=https://artifacts.elastic.co/packages/6.x/apt diff --git a/puppet/manifests/filebeat.pp b/puppet/manifests/filebeat.pp new file mode 100644 index 00000000..a6dadba5 --- /dev/null +++ b/puppet/manifests/filebeat.pp @@ -0,0 +1,450 @@ +class oci::filebeat( + $machine_role = undef, + $filebeat_enable = false, + $filebeat_repo = undef, + + # What to log + $log_aodh = false, + $log_apache2 = false, + $log_barbican = false, + $log_frr = false, + $log_ceilometer = false, + $log_ceph = false, + $log_cinder = false, + $log_cloudkitty = false, + $log_corosync = false, + $log_designate = false, + $log_etcd = false, + $log_haproxy = false, + $log_horizon = false, + $log_glance = false, + $log_gnocchi = false, + $log_heat = false, + $log_keystone = false, + $log_memcached = false, + $log_magnum = false, + $log_mysql = false, + $log_neutron = false, + $log_nova = false, + $log_octavia = false, + $log_openvswitch = false, + $log_panko = false, + $log_pacemaker = false, + $log_placement = false, + $log_rabbitmq = false, + $log_swiftproxy = false, + $log_swiftaccount = false, + $log_swiftcontainer = false, + $log_swiftobject = false, + $log_zookeeper = false, + + # Filebeat params + $fields_under_root = undef, + $output_hosts = undef, + $logging_level = undef, + $logging_syslog = true, + $logging_files = undef, + $logging_files_path = undef, + $logging_files_name = undef, +){ + + if $filebeat_enable { + + + $default_inputs = [ + { + name => 'system', + paths => [ + '/var/log/zabbix/*.log*', + '/var/log/*.log*', + ], + tags => [ + 'syslog', + 'system', + ], + }, + ] + + $filebeat_repo_key = '-----BEGIN PGP PUBLIC KEY BLOCK----- +Version: GnuPG v2.0.14 (GNU/Linux) + +mQENBFI3HsoBCADXDtbNJnxbPqB1vDNtCsqhe49vFYsZN9IOZsZXgp7aHjh6CJBD +A+bGFOwyhbd7at35jQjWAw1O3cfYsKAmFy+Ar3LHCMkV3oZspJACTIgCrwnkic/9 +CUliQe324qvObU2QRtP4Fl0zWcfb/S8UYzWXWIFuJqMvE9MaRY1bwUBvzoqavLGZ +j3SF1SPO+TB5QrHkrQHBsmX+Jda6d4Ylt8/t6CvMwgQNlrlzIO9WT+YN6zS+sqHd +1YK/aY5qhoLNhp9G/HxhcSVCkLq8SStj1ZZ1S9juBPoXV1ZWNbxFNGwOh/NYGldD +2kmBf3YgCqeLzHahsAEpvAm8TBa7Q9W21C8vABEBAAG0RUVsYXN0aWNzZWFyY2gg +KEVsYXN0aWNzZWFyY2ggU2lnbmluZyBLZXkpIDxkZXZfb3BzQGVsYXN0aWNzZWFy +Y2gub3JnPokBOAQTAQIAIgUCUjceygIbAwYLCQgHAwIGFQgCCQoLBBYCAwECHgEC +F4AACgkQ0n1mbNiOQrRzjAgAlTUQ1mgo3nK6BGXbj4XAJvuZDG0HILiUt+pPnz75 +nsf0NWhqR4yGFlmpuctgCmTD+HzYtV9fp9qW/bwVuJCNtKXk3sdzYABY+Yl0Cez/ +7C2GuGCOlbn0luCNT9BxJnh4mC9h/cKI3y5jvZ7wavwe41teqG14V+EoFSn3NPKm +TxcDTFrV7SmVPxCBcQze00cJhprKxkuZMPPVqpBS+JfDQtzUQD/LSFfhHj9eD+Xe +8d7sw+XvxB2aN4gnTlRzjL1nTRp0h2/IOGkqYfIG9rWmSLNlxhB2t+c0RsjdGM4/ +eRlPWylFbVMc5pmDpItrkWSnzBfkmXL3vO2X3WvwmSFiQbkBDQRSNx7KAQgA5JUl +zcMW5/cuyZR8alSacKqhSbvoSqqbzHKcUQZmlzNMKGTABFG1yRx9r+wa/fvqP6OT +RzRDvVS/cycws8YX7Ddum7x8uI95b9ye1/Xy5noPEm8cD+hplnpU+PBQZJ5XJ2I+ +1l9Nixx47wPGXeClLqcdn0ayd+v+Rwf3/XUJrvccG2YZUiQ4jWZkoxsA07xx7Bj+ +Lt8/FKG7sHRFvePFU0ZS6JFx9GJqjSBbHRRkam+4emW3uWgVfZxuwcUCn1ayNgRt +KiFv9jQrg2TIWEvzYx9tywTCxc+FFMWAlbCzi+m4WD+QUWWfDQ009U/WM0ks0Kww +EwSk/UDuToxGnKU2dQARAQABiQEfBBgBAgAJBQJSNx7KAhsMAAoJENJ9ZmzYjkK0 +c3MIAIE9hAR20mqJWLcsxLtrRs6uNF1VrpB+4n/55QU7oxA1iVBO6IFu4qgsF12J +TavnJ5MLaETlggXY+zDef9syTPXoQctpzcaNVDmedwo1SiL03uMoblOvWpMR/Y0j +6rm7IgrMWUDXDPvoPGjMl2q1iTeyHkMZEyUJ8SKsaHh4jV9wp9KmC8C+9CwMukL7 +vM5w8cgvJoAwsp3Fn59AxWthN3XJYcnMfStkIuWgR7U2r+a210W6vnUxU4oN0PmM +cursYPyeV0NX/KQeUeNMwGTFB6QHS/anRaGQewijkrYYoTNtfllxIu9XYmiBERQ/ +qPDlGRlOgVTd9xUfHFkzB52c70E= +=92oX +-----END PGP PUBLIC KEY BLOCK-----' + + apt::source {'filebeat': + ensure => 'present', + location => $filebeat_repo, + key => { + ensure => 'present', + content => $filebeat_repo_key, + } + } + + $hosts = split($filebeat_output_hosts, '%') + + class { 'filebeat': + manage_repo => false, + manage_apt => false, + major_version => '6', + fields_under_root => $fields_under_root, + outputs => { + 'logstash' => { + 'hosts' => $hosts, + 'loadbalance' => true, + }, + }, + logging => { + 'level' => $logging_level, + 'to_syslog' => $logging_syslog, + 'to_files' => $logging_files, + 'files' => { + 'path' => $logging_files_path, + 'name' => $logging_files_name, + 'keepfiles' => '7', + 'permissions' => '0644', + }, + }, + require => Apt::Source['filebeat'], + } + + $filebeat_merged_inputs.each |Hash $input| { + # frontends uWSGI + filebeat::input { $input['name']: + fields_under_root => true, + paths => $input['paths'], + fields => $input['fields'], + tags => $input['tags'], + } + } + + if $log_aodh{ + filebeat::input { 'aodh': + fields_under_root => true, + paths => '/var/log/aodh/*.log*', + fields => { backend => $machine_role, }, + tags => [ 'aodh', 'openstack', ], + } + } + + if $log_apache2{ + filebeat::input { 'aodh': + fields_under_root => true, + paths => '/var/log/apache2/*.log*' + fields => { backend => $machine_role, }, + tags => [ 'apache2', ], + } + } + + if $log_barbican{ + filebeat::input { 'aodh': + fields_under_root => true, + paths => '/var/log/barbican/*.log*' + fields => { backend => $machine_role, }, + tags => [ 'barbican', 'openstack', ], + } + } + if $log_frr{ + filebeat::input { 'frr': + fields_under_root => true, + paths => '/var/log/frr/*.log*' + fields => { backend => $machine_role, }, + tags => [ 'frr', 'system', ], + } + } + + filebeat::input { 'chrony': + fields_under_root => true, + paths => '/var/log/chrony/*.log*' + fields => { backend => $machine_role, }, + tags => [ 'chrony', 'ntp', 'system', ], + } + + if $log_ceilometer{ + filebeat::input { 'ceilometer': + fields_under_root => true, + paths => '/var/log/ceilometer/*.log*' + fields => { backend => $machine_role, }, + tags => [ 'ceilometer', 'openstack', ], + } + } + + if $log_ceph{ + filebeat::input { 'ceph': + fields_under_root => true, + paths => '/var/log/ceph/*.log*' + fields => { backend => $machine_role, }, + tags => [ 'ceph', ], + } + } + + if $log_cinder{ + filebeat::input { 'cinder': + fields_under_root => true, + paths => '/var/log/cinder/*.log*' + fields => { backend => $machine_role, }, + tags => [ 'cinder', 'openstack', ], + } + } + + if $log_cloudkitty{ + filebeat::input { 'cloudkitty': + fields_under_root => true, + paths => '/var/log/cloudkitty/*.log*' + fields => { backend => $machine_role, }, + tags => [ 'cloudkitty', 'openstack', ], + } + } + + if $log_corosync{ + filebeat::input { 'corosync': + fields_under_root => true, + paths => '/var/log/corosync/*.log*' + fields => { backend => $machine_role, }, + tags => [ 'corosync', 'system', ], + } + } + + if $log_designate{ + filebeat::input { 'designate': + fields_under_root => true, + paths => '/var/log/designate/*.log*' + fields => { backend => $machine_role, }, + tags => [ 'designate', 'openstack', ], + } + } + + if $log_etcd{ + filebeat::input { 'etcd': + fields_under_root => true, + paths => '/var/log/etcd/*.log*' + fields => { backend => $machine_role, }, + tags => [ 'etcd', 'system', ], + } + } + + if $log_haproxy{ + filebeat::input { 'haproxy': + fields_under_root => true, + paths => '/var/log/haproxy.log*' + fields => { backend => $machine_role, }, + tags => [ 'haproxy', 'system', ], + } + } + + if $log_horizon{ + filebeat::input { 'horizon': + fields_under_root => true, + paths => ['/var/log/horizon/*.log*', '/var/log/openstack-dashboard/*.log*'], + fields => { backend => $machine_role, }, + tags => [ 'horizon', 'openstack', ], + } + } + + if $log_glance{ + filebeat::input { 'glance': + fields_under_root => true, + paths => '/var/log/glance/*.log*' + fields => { backend => $machine_role, }, + tags => [ 'glance', 'openstack', ], + } + } + + if $log_gnocchi{ + filebeat::input { 'gnocchi': + fields_under_root => true, + paths => '/var/log/gnocchi/*.log*' + fields => { backend => $machine_role, }, + tags => [ 'gnocchi', 'openstack', ], + } + } + + if $log_heat{ + filebeat::input { 'heat': + fields_under_root => true, + paths => '/var/log/heat/*.log*' + fields => { backend => $machine_role, }, + tags => [ 'heat', 'openstack', ], + } + } + + if $log_keystone{ + filebeat::input { 'keystone': + fields_under_root => true, + paths => '/var/log/keystone/*.log*' + fields => { backend => $machine_role, }, + tags => [ 'keystone', 'openstack', ], + } + } + + if $log_memcached{ + filebeat::input { 'memcached': + fields_under_root => true, + paths => '/var/log/memcached/*.log*' + fields => { backend => $machine_role, }, + tags => [ 'memcached', 'system', ], + } + } + + if $log_magnum{ + filebeat::input { 'magnum': + fields_under_root => true, + paths => '/var/log/magnum/*.log*' + fields => { backend => $machine_role, }, + tags => [ 'magnum', 'openstack', ], + } + } + + if $log_mysql{ + filebeat::input { 'mysql': + fields_under_root => true, + paths => '/var/log/mysql/*.log*' + fields => { backend => $machine_role, }, + tags => [ 'mysql', 'system', ], + } + } + + if $log_neutron{ + filebeat::input { 'neutron': + fields_under_root => true, + paths => '/var/log/neutron/*.log*' + fields => { backend => $machine_role, }, + tags => [ 'neutron', 'openstack', ], + } + } + + if $log_nova{ + filebeat::input { 'nova': + fields_under_root => true, + paths => '/var/log/nova/*.log*' + fields => { backend => $machine_role, }, + tags => [ 'nova', 'openstack', ], + } + } + + if $log_octavia{ + filebeat::input { 'octavia': + fields_under_root => true, + paths => '/var/log/octavia/*.log*' + fields => { backend => $machine_role, }, + tags => [ 'octavia', 'openstack', ], + } + } + + if $log_openvswitch{ + filebeat::input { 'openvswitch': + fields_under_root => true, + paths => '/var/log/openvswitch/*.log*' + fields => { backend => $machine_role, }, + tags => [ 'openvswitch', 'system', ], + } + } + + if $log_panko{ + filebeat::input { 'panko': + fields_under_root => true, + paths => '/var/log/panko/*.log*' + fields => { backend => $machine_role, }, + tags => [ 'panko', 'openstack', ], + } + } + + if $log_pacemaker{ + filebeat::input { 'pacemaker': + fields_under_root => true, + paths => '/var/log/pacemaker/*.log*' + fields => { backend => $machine_role, }, + tags => [ 'pacemaker', 'system', ], + } + } + + if $log_placement{ + filebeat::input { 'placement': + fields_under_root => true, + paths => '/var/log/placement/*.log*' + fields => { backend => $machine_role, }, + tags => [ 'placement', 'openstack', ], + } + } + + if $log_rabbitmq{ + filebeat::input { 'rabbitmq': + fields_under_root => true, + paths => '/var/log/rabbitmq/*.log*' + fields => { backend => $machine_role, }, + tags => [ 'rabbitmq', 'system', ], + } + } + + if $log_swiftproxy{ + filebeat::input { 'swiftproxy': + fields_under_root => true, + paths => ['/var/log/swift/swift-proxy*.log*', '/var/log/swift/proxy*.log*'] + fields => { backend => $machine_role, }, + tags => [ 'swift', 'swiftproxy', 'openstack', ], + } + } + + if $log_swiftaccount{ + filebeat::input { 'swiftproxy': + fields_under_root => true, + paths => ['/var/log/swift/swift-account*.log*', '/var/log/swift/account*.log*'] + fields => { backend => $machine_role, }, + tags => [ 'swift', 'swiftaccount', 'openstack', ], + } + } + + if $log_swiftcontainer{ + filebeat::input { 'swiftcontainer': + fields_under_root => true, + paths => ['/var/log/swift/swift-container*.log*', '/var/log/swift/container*.log*'] + fields => { backend => $machine_role, }, + tags => [ 'swift', 'swiftcontainer', 'openstack', ], + } + } + + if $log_swiftobject{ + filebeat::input { 'swiftcontainer': + fields_under_root => true, + paths => '/var/log/swift/object*.log*' + fields => { backend => $machine_role, }, + tags => [ 'swift', 'swiftobject', 'openstack', ], + } + } + + if $log_zookeeper{ + filebeat::input { 'rabbitmq': + fields_under_root => true, + paths => '/var/log/zookeeper/*.log*' + fields => { backend => $machine_role, }, + tags => [ 'zookeeper', 'system', ], + } + } + + } +} diff --git a/src/inc/enc.php b/src/inc/enc.php index 18a407fc..e1909194 100644 --- a/src/inc/enc.php +++ b/src/inc/enc.php @@ -883,6 +883,152 @@ function puppet_enc($con,$conf){ $enc_file .= "\n"; + ######################## + ### Filebeat logging ### + ######################## + $enc_file .= " oci::filebeat:\n"; + $enc_file .= " machine_role: ".$machine["role"]."\n"; + $enc_file .= " filebeat_repo: ".$conf["filebeat"]["filebeat_repo"]."\n"; + + if($cluster["filebeat_enabled"] != "yes"){ + $enc_file .= " filebeat_enable: false\n"; + }else{ + $enc_file .= " filebeat_enable: true\n"; + # Define role and subroles + # Note that logging roles may be different from actual roles, + # for example in a controller it may have "swift" subrole, but + # that's not relevant for logging. + switch($machine["role"]){ + case "controller": + if($num_compute_nodes > 0){ + $enc_file .= " log_aodh: true\n"; + $enc_file .= " log_apache2: true\n"; + } + $enc_file .= " log_barbican: true\n"; + $enc_file .= " log_frr: true\n"; + if($num_cephosd_nodes > 0){ + $enc_file .= " log_ceilometer: true\n"; + $enc_file .= " log_ceph: true\n"; + } + if($num_volume_nodes > 0 OR $num_cephosd_nodes > 0){ + $enc_file .= " log_cinder: true\n"; + } + if($num_cephosd_nodes > 0){ + $enc_file .= " log_cloudkitty: true\n"; + } + $enc_file .= " log_corosync: true\n"; + if($cluster["install_magnum"] == yes){ + $enc_file .= " log_etcd: true\n"; + } + $enc_file .= " log_haproxy: true\n"; + $enc_file .= " log_horizon: true\n"; + if($cluster["install_designate"] == "yes"){ + $enc_file .= " log_designate: true\n"; + } + if($num_compute_nodes > 0){ + $enc_file .= " log_glance: true\n"; + } + if($num_cephosd_nodes > 0){ + $enc_file .= " log_gnocchi: true\n"; + } + $enc_file .= " log_heat: true\n"; + $enc_file .= " log_horizon: true\n"; + $enc_file .= " log_keystone: true\n"; + $enc_file .= " log_memcached: true\n"; + if($cluster["install_magnum"] == yes){ + $enc_file .= " log_magnum: true\n"; + } + if($n_sql_machines == 0){ + $enc_file .= " log_mysql: true\n"; + } + if($num_compute_nodes > 0){ + $enc_file .= " log_neutron: true\n"; + $enc_file .= " log_nova: true\n"; + $enc_file .= " log_octavia: true\n"; + } + if($num_network_nodes == 0){ + $enc_file .= " log_openvswitch: true\n"; + } + if($num_cephosd_nodes > 0){ + $enc_file .= " log_panko: true\n"; + } + $enc_file .= " log_pacemaker: true\n"; + if($num_compute_nodes > 0){ + $enc_file .= " log_placement: true\n"; + } + $enc_file .= " log_rabbitmq: true\n"; + if($num_cephosd_nodes > 0 || $cluster["install_designate"] == "yes"){ + $enc_file .= " log_zookeeper: true\n"; + } + break; + case "messaging": + $enc_file .= " log_ceph: true\n"; + $enc_file .= " log_cloudkitty: true\n"; + $enc_file .= " log_haproxy: true\n"; + $enc_file .= " log_gnocchi: true\n"; + $enc_file .= " log_mysql: true\n"; + $enc_file .= " log_rabbitmq: true\n"; + break; + case "swiftproxy": + $enc_file .= " log_haproxy: true\n"; + $enc_file .= " log_swiftproxy: true\n"; + if($machine["swift_store_account"] == "yes"){ + $enc_file .= " log_swiftaccount: true\n"; + } + if($machine["swift_store_container"] == "yes"){ + $enc_file .= " log_swiftcontainer: true\n"; + } + if($machine["swift_store_object"] == "yes"){ + $enc_file .= " log_swiftobject: true\n"; + } + break; + case "swiftstore": + if($machine["swift_store_account"] == "yes"){ + $enc_file .= " log_swiftaccount: true\n"; + } + if($machine["swift_store_container"] == "yes"){ + $enc_file .= " log_swiftcontainer: true\n"; + } + if($machine["swift_store_object"] == "yes"){ + $enc_file .= " log_swiftobject: true\n"; + } + break; + case "network": + $enc_file .= " log_neutron: true\n"; + break; + case "compute": + $enc_file .= " log_neutron: true\n"; + $enc_file .= " log_nova: true\n"; + if($num_cephosd_nodes > 0){ + $enc_file .= " log_cinder: true\n"; + } + break; + case "billosd": + case "billmon": + case "cephosd": + case "cephmon": + $enc_file .= " log_ceph: true\n"; + break; + break; + case "volume": + $enc_file .= " log_cinder: true\n"; + break; + case "debmirror": + case "tempest": + case "sql": + break; + } + + $enc_file .= " fields_under_root: ".$cluster['filebeat_fields_under_root']."\n"; + $enc_file .= " output_hosts: ".$cluster['filebeat_output_hosts']."\n"; + $enc_file .= " logging_level: ".$cluster['filebeat_logging_level']."\n"; + $enc_file .= " logging_syslog: ".$cluster['filebeat_logging_syslog']."\n"; + $enc_file .= " logging_files: ".$cluster['filebeat_logging_files']."\n"; + $enc_file .= " logging_files_path: ".$cluster['filebeat_logging_files_path']."\n"; + $enc_file .= " logging_files_name: ".$cluster['filebeat_logging_files_name']."\n"; + } + $enc_file .= "\n"; + $skip_variables_dot_json_enc = "no"; ############################### diff --git a/src/variables.json b/src/variables.json index 1f4c5068..3f4cee58 100644 --- a/src/variables.json +++ b/src/variables.json @@ -213,6 +213,46 @@ "default": "20", "description": "API haproxy rate limit:", "roles": ["controller"] + }, + "filebeat_enabled": { + "type": "boolean", + "default": "no", + "description": "Filebeat enable:" + }, + "filebeat_fields_under_root": { + "type": "boolean", + "default": "yes", + "description": "Filebeat fields under root:" + }, + "filebeat_output_hosts": { + "type": "string", + "default": "host1.example.com:5044%host2.example.com:5044", + "description": "Filebeat output hosts:" + }, + "filebeat_logging_level": { + "type": "string", + "default": "info", + "description": "Filebeat logging level:" + }, + "filebeat_logging_syslog": { + "type": "boolean", + "default": "no", + "description": "Filebeat logging to syslog:" + }, + "filebeat_logging_files": { + "type": "boolean", + "default": "yes", + "description": "Filebeat logging to files:" + }, + "filebeat_logging_files_path": { + "type": "string", + "default": "/var/log/filebeat", + "description": "Filebeat logging files path:" + }, + "filebeat_logging_files_name": { + "type": "string", + "default": "filebeat", + "description": "Filebeat logging files name:" } } } -- GitLab From 5335e2b0cf843303ce2846e2fd1c95228343140a Mon Sep 17 00:00:00 2001 From: Thomas Goirand Date: Fri, 9 Apr 2021 11:01:18 +0200 Subject: [PATCH 07/18] Added filebeat support. --- debian/changelog | 1 + .../openstack-cluster-installer.conf | 1 + puppet/manifests/filebeat.pp | 165 ++++++++++++------ src/variables.json | 4 +- 4 files changed, 116 insertions(+), 55 deletions(-) diff --git a/debian/changelog b/debian/changelog index 31106568..b2ea0458 100644 --- a/debian/changelog +++ b/debian/changelog @@ -27,6 +27,7 @@ openstack-cluster-installer (41) UNRELEASED; urgency=medium * Only keep 7 days of haproxy logs instead of the default which is 52. * controller: also install python3-pankoclient. * Configure by default the number of keystone uwsgi workers. + * Added filebeat support. -- Thomas Goirand Tue, 16 Feb 2021 19:08:35 +0100 diff --git a/etc/openstack-cluster-installer/openstack-cluster-installer.conf b/etc/openstack-cluster-installer/openstack-cluster-installer.conf index db0b19b8..b2142b0f 100644 --- a/etc/openstack-cluster-installer/openstack-cluster-installer.conf +++ b/etc/openstack-cluster-installer/openstack-cluster-installer.conf @@ -254,4 +254,5 @@ auto_install_machines_num_of_discovery=15 [filebeat] # Repository URL for Filebeat. +# Our mirror: http://apt.infomaniak.ch/artifacts.elastic.co/packages/6.x/apt filebeat_repo=https://artifacts.elastic.co/packages/6.x/apt diff --git a/puppet/manifests/filebeat.pp b/puppet/manifests/filebeat.pp index a6dadba5..9a27bc7c 100644 --- a/puppet/manifests/filebeat.pp +++ b/puppet/manifests/filebeat.pp @@ -1,7 +1,7 @@ class oci::filebeat( - $machine_role = undef, - $filebeat_enable = false, - $filebeat_repo = undef, + $machine_role = undef, + $filebeat_enable = false, + $filebeat_repo = undef, # What to log $log_aodh = false, @@ -51,7 +51,7 @@ class oci::filebeat( if $filebeat_enable { - $default_inputs = [ + $filebeat_default_inputs = [ { name => 'system', paths => [ @@ -100,13 +100,15 @@ qPDlGRlOgVTd9xUfHFkzB52c70E= apt::source {'filebeat': ensure => 'present', location => $filebeat_repo, + release => 'stable', key => { + id => '46095ACC8548582C1A2699A9D27D666CD88E42B4', ensure => 'present', content => $filebeat_repo_key, } } - $hosts = split($filebeat_output_hosts, '%') + $hosts = split($output_hosts, '@') class { 'filebeat': manage_repo => false, @@ -133,54 +135,53 @@ qPDlGRlOgVTd9xUfHFkzB52c70E= require => Apt::Source['filebeat'], } - $filebeat_merged_inputs.each |Hash $input| { - # frontends uWSGI - filebeat::input { $input['name']: - fields_under_root => true, - paths => $input['paths'], - fields => $input['fields'], - tags => $input['tags'], - } - } - if $log_aodh{ filebeat::input { 'aodh': fields_under_root => true, - paths => '/var/log/aodh/*.log*', + paths => [ '/var/log/aodh/*.log*' ], fields => { backend => $machine_role, }, tags => [ 'aodh', 'openstack', ], } + }else{ + file { '/etc/filebeat/conf.d/aodh.yml': ensure => 'absent', notify => Service['filebeat'] } } if $log_apache2{ - filebeat::input { 'aodh': + filebeat::input { 'apache2': fields_under_root => true, - paths => '/var/log/apache2/*.log*' + paths => [ '/var/log/apache2/*.log*' ], fields => { backend => $machine_role, }, tags => [ 'apache2', ], } + }else{ + file { '/etc/filebeat/conf.d/apache2.yml': ensure => 'absent', notify => Service['filebeat'] } } if $log_barbican{ - filebeat::input { 'aodh': + filebeat::input { 'barbican': fields_under_root => true, - paths => '/var/log/barbican/*.log*' + paths => [ '/var/log/barbican/*.log*' ], fields => { backend => $machine_role, }, tags => [ 'barbican', 'openstack', ], } + }else{ + file { '/etc/filebeat/conf.d/barbican.yml': ensure => 'absent', notify => Service['filebeat'] } } + if $log_frr{ filebeat::input { 'frr': fields_under_root => true, - paths => '/var/log/frr/*.log*' + paths => [ '/var/log/frr/*.log*' ], fields => { backend => $machine_role, }, tags => [ 'frr', 'system', ], } + }else{ + file { '/etc/filebeat/conf.d/frr.yml': ensure => 'absent', notify => Service['filebeat'] } } filebeat::input { 'chrony': fields_under_root => true, - paths => '/var/log/chrony/*.log*' + paths => [ '/var/log/chrony/*.log*' ], fields => { backend => $machine_role, }, tags => [ 'chrony', 'ntp', 'system', ], } @@ -188,73 +189,89 @@ qPDlGRlOgVTd9xUfHFkzB52c70E= if $log_ceilometer{ filebeat::input { 'ceilometer': fields_under_root => true, - paths => '/var/log/ceilometer/*.log*' + paths => [ '/var/log/ceilometer/*.log*' ], fields => { backend => $machine_role, }, tags => [ 'ceilometer', 'openstack', ], } + }else{ + file { '/etc/filebeat/conf.d/ceilometer.yml': ensure => 'absent', notify => Service['filebeat'] } } if $log_ceph{ filebeat::input { 'ceph': fields_under_root => true, - paths => '/var/log/ceph/*.log*' + paths => [ '/var/log/ceph/*.log*' ], fields => { backend => $machine_role, }, tags => [ 'ceph', ], } + }else{ + file { '/etc/filebeat/conf.d/ceph.yml': ensure => 'absent', notify => Service['filebeat'] } } if $log_cinder{ filebeat::input { 'cinder': fields_under_root => true, - paths => '/var/log/cinder/*.log*' + paths => [ '/var/log/cinder/*.log*' ], fields => { backend => $machine_role, }, tags => [ 'cinder', 'openstack', ], } + }else{ + file { '/etc/filebeat/conf.d/cinder.yml': ensure => 'absent', notify => Service['filebeat'] } } if $log_cloudkitty{ filebeat::input { 'cloudkitty': fields_under_root => true, - paths => '/var/log/cloudkitty/*.log*' + paths => [ '/var/log/cloudkitty/*.log*' ], fields => { backend => $machine_role, }, tags => [ 'cloudkitty', 'openstack', ], } + }else{ + file { '/etc/filebeat/conf.d/cloudkitty.yml': ensure => 'absent', notify => Service['filebeat'] } } if $log_corosync{ filebeat::input { 'corosync': fields_under_root => true, - paths => '/var/log/corosync/*.log*' + paths => [ '/var/log/corosync/*.log*' ], fields => { backend => $machine_role, }, tags => [ 'corosync', 'system', ], } + }else{ + file { '/etc/filebeat/conf.d/corosync.yml': ensure => 'absent', notify => Service['filebeat'] } } if $log_designate{ filebeat::input { 'designate': fields_under_root => true, - paths => '/var/log/designate/*.log*' + paths => [ '/var/log/designate/*.log*' ], fields => { backend => $machine_role, }, tags => [ 'designate', 'openstack', ], } + }else{ + file { '/etc/filebeat/conf.d/designate.yml': ensure => 'absent', notify => Service['filebeat'] } } if $log_etcd{ filebeat::input { 'etcd': fields_under_root => true, - paths => '/var/log/etcd/*.log*' + paths => [ '/var/log/etcd/*.log*' ], fields => { backend => $machine_role, }, tags => [ 'etcd', 'system', ], } + }else{ + file { '/etc/filebeat/conf.d/etcd.yml': ensure => 'absent', notify => Service['filebeat'] } } if $log_haproxy{ filebeat::input { 'haproxy': fields_under_root => true, - paths => '/var/log/haproxy.log*' + paths => [ '/var/log/haproxy.log*' ], fields => { backend => $machine_role, }, tags => [ 'haproxy', 'system', ], } + }else{ + file { '/etc/filebeat/conf.d/haproxy.yml': ensure => 'absent', notify => Service['filebeat'] } } if $log_horizon{ @@ -264,186 +281,228 @@ qPDlGRlOgVTd9xUfHFkzB52c70E= fields => { backend => $machine_role, }, tags => [ 'horizon', 'openstack', ], } + }else{ + file { '/etc/filebeat/conf.d/horizon.yml': ensure => 'absent', notify => Service['filebeat'] } } if $log_glance{ filebeat::input { 'glance': fields_under_root => true, - paths => '/var/log/glance/*.log*' + paths => [ '/var/log/glance/*.log*' ], fields => { backend => $machine_role, }, tags => [ 'glance', 'openstack', ], } + }else{ + file { '/etc/filebeat/conf.d/glance.yml': ensure => 'absent', notify => Service['filebeat'] } } if $log_gnocchi{ filebeat::input { 'gnocchi': fields_under_root => true, - paths => '/var/log/gnocchi/*.log*' + paths => [ '/var/log/gnocchi/*.log*' ], fields => { backend => $machine_role, }, tags => [ 'gnocchi', 'openstack', ], } + }else{ + file { '/etc/filebeat/conf.d/gnocchi.yml': ensure => 'absent', notify => Service['filebeat'] } } if $log_heat{ filebeat::input { 'heat': fields_under_root => true, - paths => '/var/log/heat/*.log*' + paths => [ '/var/log/heat/*.log*' ], fields => { backend => $machine_role, }, tags => [ 'heat', 'openstack', ], } + }else{ + file { '/etc/filebeat/conf.d/heat.yml': ensure => 'absent', notify => Service['filebeat'] } } if $log_keystone{ filebeat::input { 'keystone': fields_under_root => true, - paths => '/var/log/keystone/*.log*' + paths => [ '/var/log/keystone/*.log*' ], fields => { backend => $machine_role, }, tags => [ 'keystone', 'openstack', ], } + }else{ + file { '/etc/filebeat/conf.d/keystone.yml': ensure => 'absent', notify => Service['filebeat'] } } if $log_memcached{ filebeat::input { 'memcached': fields_under_root => true, - paths => '/var/log/memcached/*.log*' + paths => [ '/var/log/memcached/*.log*' ], fields => { backend => $machine_role, }, tags => [ 'memcached', 'system', ], } + }else{ + file { '/etc/filebeat/conf.d/memcached.yml': ensure => 'absent', notify => Service['filebeat'] } } if $log_magnum{ filebeat::input { 'magnum': fields_under_root => true, - paths => '/var/log/magnum/*.log*' + paths => [ '/var/log/magnum/*.log*' ], fields => { backend => $machine_role, }, tags => [ 'magnum', 'openstack', ], } + }else{ + file { '/etc/filebeat/conf.d/magnum.yml': ensure => 'absent', notify => Service['filebeat'] } } if $log_mysql{ filebeat::input { 'mysql': fields_under_root => true, - paths => '/var/log/mysql/*.log*' + paths => [ '/var/log/mysql/*.log*' ], fields => { backend => $machine_role, }, tags => [ 'mysql', 'system', ], } + }else{ + file { '/etc/filebeat/conf.d/mysql.yml': ensure => 'absent', notify => Service['filebeat'] } } if $log_neutron{ filebeat::input { 'neutron': fields_under_root => true, - paths => '/var/log/neutron/*.log*' + paths => [ '/var/log/neutron/*.log*' ], fields => { backend => $machine_role, }, tags => [ 'neutron', 'openstack', ], } + }else{ + file { '/etc/filebeat/conf.d/neutron.yml': ensure => 'absent', notify => Service['filebeat'] } } if $log_nova{ filebeat::input { 'nova': fields_under_root => true, - paths => '/var/log/nova/*.log*' + paths => [ '/var/log/nova/*.log*' ], fields => { backend => $machine_role, }, tags => [ 'nova', 'openstack', ], } + }else{ + file { '/etc/filebeat/conf.d/nova.yml': ensure => 'absent', notify => Service['filebeat'] } } if $log_octavia{ filebeat::input { 'octavia': fields_under_root => true, - paths => '/var/log/octavia/*.log*' + paths => [ '/var/log/octavia/*.log*' ], fields => { backend => $machine_role, }, tags => [ 'octavia', 'openstack', ], } + }else{ + file { '/etc/filebeat/conf.d/octavia.yml': ensure => 'absent', notify => Service['filebeat'] } } if $log_openvswitch{ filebeat::input { 'openvswitch': fields_under_root => true, - paths => '/var/log/openvswitch/*.log*' + paths => [ '/var/log/openvswitch/*.log*' ], fields => { backend => $machine_role, }, tags => [ 'openvswitch', 'system', ], } + }else{ + file { '/etc/filebeat/conf.d/openvswitch.yml': ensure => 'absent', notify => Service['filebeat'] } } if $log_panko{ filebeat::input { 'panko': fields_under_root => true, - paths => '/var/log/panko/*.log*' + paths => [ '/var/log/panko/*.log*' ], fields => { backend => $machine_role, }, tags => [ 'panko', 'openstack', ], } + }else{ + file { '/etc/filebeat/conf.d/panko.yml': ensure => 'absent', notify => Service['filebeat'] } } if $log_pacemaker{ filebeat::input { 'pacemaker': fields_under_root => true, - paths => '/var/log/pacemaker/*.log*' + paths => [ '/var/log/pacemaker/*.log*' ], fields => { backend => $machine_role, }, tags => [ 'pacemaker', 'system', ], } + }else{ + file { '/etc/filebeat/conf.d/pacemaker.yml': ensure => 'absent', notify => Service['filebeat'] } } if $log_placement{ filebeat::input { 'placement': fields_under_root => true, - paths => '/var/log/placement/*.log*' + paths => [ '/var/log/placement/*.log*' ], fields => { backend => $machine_role, }, tags => [ 'placement', 'openstack', ], } + }else{ + file { '/etc/filebeat/conf.d/placement.yml': ensure => 'absent', notify => Service['filebeat'] } } if $log_rabbitmq{ filebeat::input { 'rabbitmq': fields_under_root => true, - paths => '/var/log/rabbitmq/*.log*' + paths => [ '/var/log/rabbitmq/*.log*' ], fields => { backend => $machine_role, }, tags => [ 'rabbitmq', 'system', ], } + }else{ + file { '/etc/filebeat/conf.d/rabbitmq.yml': ensure => 'absent', notify => Service['filebeat'] } } if $log_swiftproxy{ filebeat::input { 'swiftproxy': fields_under_root => true, - paths => ['/var/log/swift/swift-proxy*.log*', '/var/log/swift/proxy*.log*'] + paths => ['/var/log/swift/swift-proxy*.log*', '/var/log/swift/proxy*.log*'], fields => { backend => $machine_role, }, tags => [ 'swift', 'swiftproxy', 'openstack', ], } + }else{ + file { '/etc/filebeat/conf.d/swiftproxy.yml': ensure => 'absent', notify => Service['filebeat'] } } if $log_swiftaccount{ - filebeat::input { 'swiftproxy': + filebeat::input { 'swiftaccount': fields_under_root => true, - paths => ['/var/log/swift/swift-account*.log*', '/var/log/swift/account*.log*'] + paths => ['/var/log/swift/swift-account*.log*', '/var/log/swift/account*.log*'], fields => { backend => $machine_role, }, tags => [ 'swift', 'swiftaccount', 'openstack', ], } + }else{ + file { '/etc/filebeat/conf.d/swiftaccount.yml': ensure => 'absent', notify => Service['filebeat'] } } if $log_swiftcontainer{ filebeat::input { 'swiftcontainer': fields_under_root => true, - paths => ['/var/log/swift/swift-container*.log*', '/var/log/swift/container*.log*'] + paths => ['/var/log/swift/swift-container*.log*', '/var/log/swift/container*.log*'], fields => { backend => $machine_role, }, tags => [ 'swift', 'swiftcontainer', 'openstack', ], } + }else{ + file { '/etc/filebeat/conf.d/swiftcontainer.yml': ensure => 'absent', notify => Service['filebeat'] } } if $log_swiftobject{ - filebeat::input { 'swiftcontainer': + filebeat::input { 'swiftobject': fields_under_root => true, - paths => '/var/log/swift/object*.log*' + paths => [ '/var/log/swift/object*.log*' ], fields => { backend => $machine_role, }, tags => [ 'swift', 'swiftobject', 'openstack', ], } + }else{ + file { '/etc/filebeat/conf.d/swiftobject.yml': ensure => 'absent', notify => Service['filebeat'] } } if $log_zookeeper{ - filebeat::input { 'rabbitmq': + filebeat::input { 'zookeeper': fields_under_root => true, - paths => '/var/log/zookeeper/*.log*' + paths => [ '/var/log/zookeeper/*.log*' ], fields => { backend => $machine_role, }, tags => [ 'zookeeper', 'system', ], } + }else{ + file { '/etc/filebeat/conf.d/zookeeper.yml': ensure => 'absent', notify => Service['filebeat'] } } } diff --git a/src/variables.json b/src/variables.json index 3f4cee58..081cb695 100644 --- a/src/variables.json +++ b/src/variables.json @@ -226,8 +226,8 @@ }, "filebeat_output_hosts": { "type": "string", - "default": "host1.example.com:5044%host2.example.com:5044", - "description": "Filebeat output hosts:" + "default": "host1.example.com:5044@host2.example.com:5044", + "description": "Filebeat output hosts (separated by @):" }, "filebeat_logging_level": { "type": "string", -- GitLab From da375eeafcd81d800e08f5b9f4125d4f5aa9a2ad Mon Sep 17 00:00:00 2001 From: Thomas Goirand Date: Thu, 8 Apr 2021 19:34:37 +0200 Subject: [PATCH 08/18] Add filebeat support. --- debian/control | 1 + .../openstack-cluster-installer.conf | 4 + puppet/manifests/filebeat.pp | 450 ++++++++++++++++++ src/inc/enc.php | 146 ++++++ src/variables.json | 40 ++ 5 files changed, 641 insertions(+) create mode 100644 puppet/manifests/filebeat.pp diff --git a/debian/control b/debian/control index 8e8f192d..5b12cf66 100644 --- a/debian/control +++ b/debian/control @@ -294,6 +294,7 @@ Depends: puppet-module-etcddiscovery, puppet-module-magnum, puppet-module-tempest, + puppet-module-pcfens-filebeat, puppet-module-placement, puppet-module-puppetlabs-mysql (>= 8.1.0), puppet-module-saz-ssh, diff --git a/etc/openstack-cluster-installer/openstack-cluster-installer.conf b/etc/openstack-cluster-installer/openstack-cluster-installer.conf index bfa0c15d..db0b19b8 100644 --- a/etc/openstack-cluster-installer/openstack-cluster-installer.conf +++ b/etc/openstack-cluster-installer/openstack-cluster-installer.conf @@ -251,3 +251,7 @@ auto_install_machines_os=no # After how many runs of the hardware discovery should the # servers be installed? auto_install_machines_num_of_discovery=15 + +[filebeat] +# Repository URL for Filebeat. +filebeat_repo=https://artifacts.elastic.co/packages/6.x/apt diff --git a/puppet/manifests/filebeat.pp b/puppet/manifests/filebeat.pp new file mode 100644 index 00000000..a6dadba5 --- /dev/null +++ b/puppet/manifests/filebeat.pp @@ -0,0 +1,450 @@ +class oci::filebeat( + $machine_role = undef, + $filebeat_enable = false, + $filebeat_repo = undef, + + # What to log + $log_aodh = false, + $log_apache2 = false, + $log_barbican = false, + $log_frr = false, + $log_ceilometer = false, + $log_ceph = false, + $log_cinder = false, + $log_cloudkitty = false, + $log_corosync = false, + $log_designate = false, + $log_etcd = false, + $log_haproxy = false, + $log_horizon = false, + $log_glance = false, + $log_gnocchi = false, + $log_heat = false, + $log_keystone = false, + $log_memcached = false, + $log_magnum = false, + $log_mysql = false, + $log_neutron = false, + $log_nova = false, + $log_octavia = false, + $log_openvswitch = false, + $log_panko = false, + $log_pacemaker = false, + $log_placement = false, + $log_rabbitmq = false, + $log_swiftproxy = false, + $log_swiftaccount = false, + $log_swiftcontainer = false, + $log_swiftobject = false, + $log_zookeeper = false, + + # Filebeat params + $fields_under_root = undef, + $output_hosts = undef, + $logging_level = undef, + $logging_syslog = true, + $logging_files = undef, + $logging_files_path = undef, + $logging_files_name = undef, +){ + + if $filebeat_enable { + + + $default_inputs = [ + { + name => 'system', + paths => [ + '/var/log/zabbix/*.log*', + '/var/log/*.log*', + ], + tags => [ + 'syslog', + 'system', + ], + }, + ] + + $filebeat_repo_key = '-----BEGIN PGP PUBLIC KEY BLOCK----- +Version: GnuPG v2.0.14 (GNU/Linux) + +mQENBFI3HsoBCADXDtbNJnxbPqB1vDNtCsqhe49vFYsZN9IOZsZXgp7aHjh6CJBD +A+bGFOwyhbd7at35jQjWAw1O3cfYsKAmFy+Ar3LHCMkV3oZspJACTIgCrwnkic/9 +CUliQe324qvObU2QRtP4Fl0zWcfb/S8UYzWXWIFuJqMvE9MaRY1bwUBvzoqavLGZ +j3SF1SPO+TB5QrHkrQHBsmX+Jda6d4Ylt8/t6CvMwgQNlrlzIO9WT+YN6zS+sqHd +1YK/aY5qhoLNhp9G/HxhcSVCkLq8SStj1ZZ1S9juBPoXV1ZWNbxFNGwOh/NYGldD +2kmBf3YgCqeLzHahsAEpvAm8TBa7Q9W21C8vABEBAAG0RUVsYXN0aWNzZWFyY2gg +KEVsYXN0aWNzZWFyY2ggU2lnbmluZyBLZXkpIDxkZXZfb3BzQGVsYXN0aWNzZWFy +Y2gub3JnPokBOAQTAQIAIgUCUjceygIbAwYLCQgHAwIGFQgCCQoLBBYCAwECHgEC +F4AACgkQ0n1mbNiOQrRzjAgAlTUQ1mgo3nK6BGXbj4XAJvuZDG0HILiUt+pPnz75 +nsf0NWhqR4yGFlmpuctgCmTD+HzYtV9fp9qW/bwVuJCNtKXk3sdzYABY+Yl0Cez/ +7C2GuGCOlbn0luCNT9BxJnh4mC9h/cKI3y5jvZ7wavwe41teqG14V+EoFSn3NPKm +TxcDTFrV7SmVPxCBcQze00cJhprKxkuZMPPVqpBS+JfDQtzUQD/LSFfhHj9eD+Xe +8d7sw+XvxB2aN4gnTlRzjL1nTRp0h2/IOGkqYfIG9rWmSLNlxhB2t+c0RsjdGM4/ +eRlPWylFbVMc5pmDpItrkWSnzBfkmXL3vO2X3WvwmSFiQbkBDQRSNx7KAQgA5JUl +zcMW5/cuyZR8alSacKqhSbvoSqqbzHKcUQZmlzNMKGTABFG1yRx9r+wa/fvqP6OT +RzRDvVS/cycws8YX7Ddum7x8uI95b9ye1/Xy5noPEm8cD+hplnpU+PBQZJ5XJ2I+ +1l9Nixx47wPGXeClLqcdn0ayd+v+Rwf3/XUJrvccG2YZUiQ4jWZkoxsA07xx7Bj+ +Lt8/FKG7sHRFvePFU0ZS6JFx9GJqjSBbHRRkam+4emW3uWgVfZxuwcUCn1ayNgRt +KiFv9jQrg2TIWEvzYx9tywTCxc+FFMWAlbCzi+m4WD+QUWWfDQ009U/WM0ks0Kww +EwSk/UDuToxGnKU2dQARAQABiQEfBBgBAgAJBQJSNx7KAhsMAAoJENJ9ZmzYjkK0 +c3MIAIE9hAR20mqJWLcsxLtrRs6uNF1VrpB+4n/55QU7oxA1iVBO6IFu4qgsF12J +TavnJ5MLaETlggXY+zDef9syTPXoQctpzcaNVDmedwo1SiL03uMoblOvWpMR/Y0j +6rm7IgrMWUDXDPvoPGjMl2q1iTeyHkMZEyUJ8SKsaHh4jV9wp9KmC8C+9CwMukL7 +vM5w8cgvJoAwsp3Fn59AxWthN3XJYcnMfStkIuWgR7U2r+a210W6vnUxU4oN0PmM +cursYPyeV0NX/KQeUeNMwGTFB6QHS/anRaGQewijkrYYoTNtfllxIu9XYmiBERQ/ +qPDlGRlOgVTd9xUfHFkzB52c70E= +=92oX +-----END PGP PUBLIC KEY BLOCK-----' + + apt::source {'filebeat': + ensure => 'present', + location => $filebeat_repo, + key => { + ensure => 'present', + content => $filebeat_repo_key, + } + } + + $hosts = split($filebeat_output_hosts, '%') + + class { 'filebeat': + manage_repo => false, + manage_apt => false, + major_version => '6', + fields_under_root => $fields_under_root, + outputs => { + 'logstash' => { + 'hosts' => $hosts, + 'loadbalance' => true, + }, + }, + logging => { + 'level' => $logging_level, + 'to_syslog' => $logging_syslog, + 'to_files' => $logging_files, + 'files' => { + 'path' => $logging_files_path, + 'name' => $logging_files_name, + 'keepfiles' => '7', + 'permissions' => '0644', + }, + }, + require => Apt::Source['filebeat'], + } + + $filebeat_merged_inputs.each |Hash $input| { + # frontends uWSGI + filebeat::input { $input['name']: + fields_under_root => true, + paths => $input['paths'], + fields => $input['fields'], + tags => $input['tags'], + } + } + + if $log_aodh{ + filebeat::input { 'aodh': + fields_under_root => true, + paths => '/var/log/aodh/*.log*', + fields => { backend => $machine_role, }, + tags => [ 'aodh', 'openstack', ], + } + } + + if $log_apache2{ + filebeat::input { 'aodh': + fields_under_root => true, + paths => '/var/log/apache2/*.log*' + fields => { backend => $machine_role, }, + tags => [ 'apache2', ], + } + } + + if $log_barbican{ + filebeat::input { 'aodh': + fields_under_root => true, + paths => '/var/log/barbican/*.log*' + fields => { backend => $machine_role, }, + tags => [ 'barbican', 'openstack', ], + } + } + if $log_frr{ + filebeat::input { 'frr': + fields_under_root => true, + paths => '/var/log/frr/*.log*' + fields => { backend => $machine_role, }, + tags => [ 'frr', 'system', ], + } + } + + filebeat::input { 'chrony': + fields_under_root => true, + paths => '/var/log/chrony/*.log*' + fields => { backend => $machine_role, }, + tags => [ 'chrony', 'ntp', 'system', ], + } + + if $log_ceilometer{ + filebeat::input { 'ceilometer': + fields_under_root => true, + paths => '/var/log/ceilometer/*.log*' + fields => { backend => $machine_role, }, + tags => [ 'ceilometer', 'openstack', ], + } + } + + if $log_ceph{ + filebeat::input { 'ceph': + fields_under_root => true, + paths => '/var/log/ceph/*.log*' + fields => { backend => $machine_role, }, + tags => [ 'ceph', ], + } + } + + if $log_cinder{ + filebeat::input { 'cinder': + fields_under_root => true, + paths => '/var/log/cinder/*.log*' + fields => { backend => $machine_role, }, + tags => [ 'cinder', 'openstack', ], + } + } + + if $log_cloudkitty{ + filebeat::input { 'cloudkitty': + fields_under_root => true, + paths => '/var/log/cloudkitty/*.log*' + fields => { backend => $machine_role, }, + tags => [ 'cloudkitty', 'openstack', ], + } + } + + if $log_corosync{ + filebeat::input { 'corosync': + fields_under_root => true, + paths => '/var/log/corosync/*.log*' + fields => { backend => $machine_role, }, + tags => [ 'corosync', 'system', ], + } + } + + if $log_designate{ + filebeat::input { 'designate': + fields_under_root => true, + paths => '/var/log/designate/*.log*' + fields => { backend => $machine_role, }, + tags => [ 'designate', 'openstack', ], + } + } + + if $log_etcd{ + filebeat::input { 'etcd': + fields_under_root => true, + paths => '/var/log/etcd/*.log*' + fields => { backend => $machine_role, }, + tags => [ 'etcd', 'system', ], + } + } + + if $log_haproxy{ + filebeat::input { 'haproxy': + fields_under_root => true, + paths => '/var/log/haproxy.log*' + fields => { backend => $machine_role, }, + tags => [ 'haproxy', 'system', ], + } + } + + if $log_horizon{ + filebeat::input { 'horizon': + fields_under_root => true, + paths => ['/var/log/horizon/*.log*', '/var/log/openstack-dashboard/*.log*'], + fields => { backend => $machine_role, }, + tags => [ 'horizon', 'openstack', ], + } + } + + if $log_glance{ + filebeat::input { 'glance': + fields_under_root => true, + paths => '/var/log/glance/*.log*' + fields => { backend => $machine_role, }, + tags => [ 'glance', 'openstack', ], + } + } + + if $log_gnocchi{ + filebeat::input { 'gnocchi': + fields_under_root => true, + paths => '/var/log/gnocchi/*.log*' + fields => { backend => $machine_role, }, + tags => [ 'gnocchi', 'openstack', ], + } + } + + if $log_heat{ + filebeat::input { 'heat': + fields_under_root => true, + paths => '/var/log/heat/*.log*' + fields => { backend => $machine_role, }, + tags => [ 'heat', 'openstack', ], + } + } + + if $log_keystone{ + filebeat::input { 'keystone': + fields_under_root => true, + paths => '/var/log/keystone/*.log*' + fields => { backend => $machine_role, }, + tags => [ 'keystone', 'openstack', ], + } + } + + if $log_memcached{ + filebeat::input { 'memcached': + fields_under_root => true, + paths => '/var/log/memcached/*.log*' + fields => { backend => $machine_role, }, + tags => [ 'memcached', 'system', ], + } + } + + if $log_magnum{ + filebeat::input { 'magnum': + fields_under_root => true, + paths => '/var/log/magnum/*.log*' + fields => { backend => $machine_role, }, + tags => [ 'magnum', 'openstack', ], + } + } + + if $log_mysql{ + filebeat::input { 'mysql': + fields_under_root => true, + paths => '/var/log/mysql/*.log*' + fields => { backend => $machine_role, }, + tags => [ 'mysql', 'system', ], + } + } + + if $log_neutron{ + filebeat::input { 'neutron': + fields_under_root => true, + paths => '/var/log/neutron/*.log*' + fields => { backend => $machine_role, }, + tags => [ 'neutron', 'openstack', ], + } + } + + if $log_nova{ + filebeat::input { 'nova': + fields_under_root => true, + paths => '/var/log/nova/*.log*' + fields => { backend => $machine_role, }, + tags => [ 'nova', 'openstack', ], + } + } + + if $log_octavia{ + filebeat::input { 'octavia': + fields_under_root => true, + paths => '/var/log/octavia/*.log*' + fields => { backend => $machine_role, }, + tags => [ 'octavia', 'openstack', ], + } + } + + if $log_openvswitch{ + filebeat::input { 'openvswitch': + fields_under_root => true, + paths => '/var/log/openvswitch/*.log*' + fields => { backend => $machine_role, }, + tags => [ 'openvswitch', 'system', ], + } + } + + if $log_panko{ + filebeat::input { 'panko': + fields_under_root => true, + paths => '/var/log/panko/*.log*' + fields => { backend => $machine_role, }, + tags => [ 'panko', 'openstack', ], + } + } + + if $log_pacemaker{ + filebeat::input { 'pacemaker': + fields_under_root => true, + paths => '/var/log/pacemaker/*.log*' + fields => { backend => $machine_role, }, + tags => [ 'pacemaker', 'system', ], + } + } + + if $log_placement{ + filebeat::input { 'placement': + fields_under_root => true, + paths => '/var/log/placement/*.log*' + fields => { backend => $machine_role, }, + tags => [ 'placement', 'openstack', ], + } + } + + if $log_rabbitmq{ + filebeat::input { 'rabbitmq': + fields_under_root => true, + paths => '/var/log/rabbitmq/*.log*' + fields => { backend => $machine_role, }, + tags => [ 'rabbitmq', 'system', ], + } + } + + if $log_swiftproxy{ + filebeat::input { 'swiftproxy': + fields_under_root => true, + paths => ['/var/log/swift/swift-proxy*.log*', '/var/log/swift/proxy*.log*'] + fields => { backend => $machine_role, }, + tags => [ 'swift', 'swiftproxy', 'openstack', ], + } + } + + if $log_swiftaccount{ + filebeat::input { 'swiftproxy': + fields_under_root => true, + paths => ['/var/log/swift/swift-account*.log*', '/var/log/swift/account*.log*'] + fields => { backend => $machine_role, }, + tags => [ 'swift', 'swiftaccount', 'openstack', ], + } + } + + if $log_swiftcontainer{ + filebeat::input { 'swiftcontainer': + fields_under_root => true, + paths => ['/var/log/swift/swift-container*.log*', '/var/log/swift/container*.log*'] + fields => { backend => $machine_role, }, + tags => [ 'swift', 'swiftcontainer', 'openstack', ], + } + } + + if $log_swiftobject{ + filebeat::input { 'swiftcontainer': + fields_under_root => true, + paths => '/var/log/swift/object*.log*' + fields => { backend => $machine_role, }, + tags => [ 'swift', 'swiftobject', 'openstack', ], + } + } + + if $log_zookeeper{ + filebeat::input { 'rabbitmq': + fields_under_root => true, + paths => '/var/log/zookeeper/*.log*' + fields => { backend => $machine_role, }, + tags => [ 'zookeeper', 'system', ], + } + } + + } +} diff --git a/src/inc/enc.php b/src/inc/enc.php index 18a407fc..e1909194 100644 --- a/src/inc/enc.php +++ b/src/inc/enc.php @@ -883,6 +883,152 @@ function puppet_enc($con,$conf){ $enc_file .= "\n"; + ######################## + ### Filebeat logging ### + ######################## + $enc_file .= " oci::filebeat:\n"; + $enc_file .= " machine_role: ".$machine["role"]."\n"; + $enc_file .= " filebeat_repo: ".$conf["filebeat"]["filebeat_repo"]."\n"; + + if($cluster["filebeat_enabled"] != "yes"){ + $enc_file .= " filebeat_enable: false\n"; + }else{ + $enc_file .= " filebeat_enable: true\n"; + # Define role and subroles + # Note that logging roles may be different from actual roles, + # for example in a controller it may have "swift" subrole, but + # that's not relevant for logging. + switch($machine["role"]){ + case "controller": + if($num_compute_nodes > 0){ + $enc_file .= " log_aodh: true\n"; + $enc_file .= " log_apache2: true\n"; + } + $enc_file .= " log_barbican: true\n"; + $enc_file .= " log_frr: true\n"; + if($num_cephosd_nodes > 0){ + $enc_file .= " log_ceilometer: true\n"; + $enc_file .= " log_ceph: true\n"; + } + if($num_volume_nodes > 0 OR $num_cephosd_nodes > 0){ + $enc_file .= " log_cinder: true\n"; + } + if($num_cephosd_nodes > 0){ + $enc_file .= " log_cloudkitty: true\n"; + } + $enc_file .= " log_corosync: true\n"; + if($cluster["install_magnum"] == yes){ + $enc_file .= " log_etcd: true\n"; + } + $enc_file .= " log_haproxy: true\n"; + $enc_file .= " log_horizon: true\n"; + if($cluster["install_designate"] == "yes"){ + $enc_file .= " log_designate: true\n"; + } + if($num_compute_nodes > 0){ + $enc_file .= " log_glance: true\n"; + } + if($num_cephosd_nodes > 0){ + $enc_file .= " log_gnocchi: true\n"; + } + $enc_file .= " log_heat: true\n"; + $enc_file .= " log_horizon: true\n"; + $enc_file .= " log_keystone: true\n"; + $enc_file .= " log_memcached: true\n"; + if($cluster["install_magnum"] == yes){ + $enc_file .= " log_magnum: true\n"; + } + if($n_sql_machines == 0){ + $enc_file .= " log_mysql: true\n"; + } + if($num_compute_nodes > 0){ + $enc_file .= " log_neutron: true\n"; + $enc_file .= " log_nova: true\n"; + $enc_file .= " log_octavia: true\n"; + } + if($num_network_nodes == 0){ + $enc_file .= " log_openvswitch: true\n"; + } + if($num_cephosd_nodes > 0){ + $enc_file .= " log_panko: true\n"; + } + $enc_file .= " log_pacemaker: true\n"; + if($num_compute_nodes > 0){ + $enc_file .= " log_placement: true\n"; + } + $enc_file .= " log_rabbitmq: true\n"; + if($num_cephosd_nodes > 0 || $cluster["install_designate"] == "yes"){ + $enc_file .= " log_zookeeper: true\n"; + } + break; + case "messaging": + $enc_file .= " log_ceph: true\n"; + $enc_file .= " log_cloudkitty: true\n"; + $enc_file .= " log_haproxy: true\n"; + $enc_file .= " log_gnocchi: true\n"; + $enc_file .= " log_mysql: true\n"; + $enc_file .= " log_rabbitmq: true\n"; + break; + case "swiftproxy": + $enc_file .= " log_haproxy: true\n"; + $enc_file .= " log_swiftproxy: true\n"; + if($machine["swift_store_account"] == "yes"){ + $enc_file .= " log_swiftaccount: true\n"; + } + if($machine["swift_store_container"] == "yes"){ + $enc_file .= " log_swiftcontainer: true\n"; + } + if($machine["swift_store_object"] == "yes"){ + $enc_file .= " log_swiftobject: true\n"; + } + break; + case "swiftstore": + if($machine["swift_store_account"] == "yes"){ + $enc_file .= " log_swiftaccount: true\n"; + } + if($machine["swift_store_container"] == "yes"){ + $enc_file .= " log_swiftcontainer: true\n"; + } + if($machine["swift_store_object"] == "yes"){ + $enc_file .= " log_swiftobject: true\n"; + } + break; + case "network": + $enc_file .= " log_neutron: true\n"; + break; + case "compute": + $enc_file .= " log_neutron: true\n"; + $enc_file .= " log_nova: true\n"; + if($num_cephosd_nodes > 0){ + $enc_file .= " log_cinder: true\n"; + } + break; + case "billosd": + case "billmon": + case "cephosd": + case "cephmon": + $enc_file .= " log_ceph: true\n"; + break; + break; + case "volume": + $enc_file .= " log_cinder: true\n"; + break; + case "debmirror": + case "tempest": + case "sql": + break; + } + + $enc_file .= " fields_under_root: ".$cluster['filebeat_fields_under_root']."\n"; + $enc_file .= " output_hosts: ".$cluster['filebeat_output_hosts']."\n"; + $enc_file .= " logging_level: ".$cluster['filebeat_logging_level']."\n"; + $enc_file .= " logging_syslog: ".$cluster['filebeat_logging_syslog']."\n"; + $enc_file .= " logging_files: ".$cluster['filebeat_logging_files']."\n"; + $enc_file .= " logging_files_path: ".$cluster['filebeat_logging_files_path']."\n"; + $enc_file .= " logging_files_name: ".$cluster['filebeat_logging_files_name']."\n"; + } + $enc_file .= "\n"; + $skip_variables_dot_json_enc = "no"; ############################### diff --git a/src/variables.json b/src/variables.json index 1f4c5068..3f4cee58 100644 --- a/src/variables.json +++ b/src/variables.json @@ -213,6 +213,46 @@ "default": "20", "description": "API haproxy rate limit:", "roles": ["controller"] + }, + "filebeat_enabled": { + "type": "boolean", + "default": "no", + "description": "Filebeat enable:" + }, + "filebeat_fields_under_root": { + "type": "boolean", + "default": "yes", + "description": "Filebeat fields under root:" + }, + "filebeat_output_hosts": { + "type": "string", + "default": "host1.example.com:5044%host2.example.com:5044", + "description": "Filebeat output hosts:" + }, + "filebeat_logging_level": { + "type": "string", + "default": "info", + "description": "Filebeat logging level:" + }, + "filebeat_logging_syslog": { + "type": "boolean", + "default": "no", + "description": "Filebeat logging to syslog:" + }, + "filebeat_logging_files": { + "type": "boolean", + "default": "yes", + "description": "Filebeat logging to files:" + }, + "filebeat_logging_files_path": { + "type": "string", + "default": "/var/log/filebeat", + "description": "Filebeat logging files path:" + }, + "filebeat_logging_files_name": { + "type": "string", + "default": "filebeat", + "description": "Filebeat logging files name:" } } } -- GitLab From 8abf3222f2764a3c7f209db031d6b556f1b11a3f Mon Sep 17 00:00:00 2001 From: Thomas Goirand Date: Fri, 9 Apr 2021 11:01:18 +0200 Subject: [PATCH 09/18] Added filebeat support. --- debian/changelog | 1 + .../openstack-cluster-installer.conf | 1 + puppet/manifests/filebeat.pp | 165 ++++++++++++------ src/variables.json | 4 +- 4 files changed, 116 insertions(+), 55 deletions(-) diff --git a/debian/changelog b/debian/changelog index 31106568..b2ea0458 100644 --- a/debian/changelog +++ b/debian/changelog @@ -27,6 +27,7 @@ openstack-cluster-installer (41) UNRELEASED; urgency=medium * Only keep 7 days of haproxy logs instead of the default which is 52. * controller: also install python3-pankoclient. * Configure by default the number of keystone uwsgi workers. + * Added filebeat support. -- Thomas Goirand Tue, 16 Feb 2021 19:08:35 +0100 diff --git a/etc/openstack-cluster-installer/openstack-cluster-installer.conf b/etc/openstack-cluster-installer/openstack-cluster-installer.conf index db0b19b8..b2142b0f 100644 --- a/etc/openstack-cluster-installer/openstack-cluster-installer.conf +++ b/etc/openstack-cluster-installer/openstack-cluster-installer.conf @@ -254,4 +254,5 @@ auto_install_machines_num_of_discovery=15 [filebeat] # Repository URL for Filebeat. +# Our mirror: http://apt.infomaniak.ch/artifacts.elastic.co/packages/6.x/apt filebeat_repo=https://artifacts.elastic.co/packages/6.x/apt diff --git a/puppet/manifests/filebeat.pp b/puppet/manifests/filebeat.pp index a6dadba5..9a27bc7c 100644 --- a/puppet/manifests/filebeat.pp +++ b/puppet/manifests/filebeat.pp @@ -1,7 +1,7 @@ class oci::filebeat( - $machine_role = undef, - $filebeat_enable = false, - $filebeat_repo = undef, + $machine_role = undef, + $filebeat_enable = false, + $filebeat_repo = undef, # What to log $log_aodh = false, @@ -51,7 +51,7 @@ class oci::filebeat( if $filebeat_enable { - $default_inputs = [ + $filebeat_default_inputs = [ { name => 'system', paths => [ @@ -100,13 +100,15 @@ qPDlGRlOgVTd9xUfHFkzB52c70E= apt::source {'filebeat': ensure => 'present', location => $filebeat_repo, + release => 'stable', key => { + id => '46095ACC8548582C1A2699A9D27D666CD88E42B4', ensure => 'present', content => $filebeat_repo_key, } } - $hosts = split($filebeat_output_hosts, '%') + $hosts = split($output_hosts, '@') class { 'filebeat': manage_repo => false, @@ -133,54 +135,53 @@ qPDlGRlOgVTd9xUfHFkzB52c70E= require => Apt::Source['filebeat'], } - $filebeat_merged_inputs.each |Hash $input| { - # frontends uWSGI - filebeat::input { $input['name']: - fields_under_root => true, - paths => $input['paths'], - fields => $input['fields'], - tags => $input['tags'], - } - } - if $log_aodh{ filebeat::input { 'aodh': fields_under_root => true, - paths => '/var/log/aodh/*.log*', + paths => [ '/var/log/aodh/*.log*' ], fields => { backend => $machine_role, }, tags => [ 'aodh', 'openstack', ], } + }else{ + file { '/etc/filebeat/conf.d/aodh.yml': ensure => 'absent', notify => Service['filebeat'] } } if $log_apache2{ - filebeat::input { 'aodh': + filebeat::input { 'apache2': fields_under_root => true, - paths => '/var/log/apache2/*.log*' + paths => [ '/var/log/apache2/*.log*' ], fields => { backend => $machine_role, }, tags => [ 'apache2', ], } + }else{ + file { '/etc/filebeat/conf.d/apache2.yml': ensure => 'absent', notify => Service['filebeat'] } } if $log_barbican{ - filebeat::input { 'aodh': + filebeat::input { 'barbican': fields_under_root => true, - paths => '/var/log/barbican/*.log*' + paths => [ '/var/log/barbican/*.log*' ], fields => { backend => $machine_role, }, tags => [ 'barbican', 'openstack', ], } + }else{ + file { '/etc/filebeat/conf.d/barbican.yml': ensure => 'absent', notify => Service['filebeat'] } } + if $log_frr{ filebeat::input { 'frr': fields_under_root => true, - paths => '/var/log/frr/*.log*' + paths => [ '/var/log/frr/*.log*' ], fields => { backend => $machine_role, }, tags => [ 'frr', 'system', ], } + }else{ + file { '/etc/filebeat/conf.d/frr.yml': ensure => 'absent', notify => Service['filebeat'] } } filebeat::input { 'chrony': fields_under_root => true, - paths => '/var/log/chrony/*.log*' + paths => [ '/var/log/chrony/*.log*' ], fields => { backend => $machine_role, }, tags => [ 'chrony', 'ntp', 'system', ], } @@ -188,73 +189,89 @@ qPDlGRlOgVTd9xUfHFkzB52c70E= if $log_ceilometer{ filebeat::input { 'ceilometer': fields_under_root => true, - paths => '/var/log/ceilometer/*.log*' + paths => [ '/var/log/ceilometer/*.log*' ], fields => { backend => $machine_role, }, tags => [ 'ceilometer', 'openstack', ], } + }else{ + file { '/etc/filebeat/conf.d/ceilometer.yml': ensure => 'absent', notify => Service['filebeat'] } } if $log_ceph{ filebeat::input { 'ceph': fields_under_root => true, - paths => '/var/log/ceph/*.log*' + paths => [ '/var/log/ceph/*.log*' ], fields => { backend => $machine_role, }, tags => [ 'ceph', ], } + }else{ + file { '/etc/filebeat/conf.d/ceph.yml': ensure => 'absent', notify => Service['filebeat'] } } if $log_cinder{ filebeat::input { 'cinder': fields_under_root => true, - paths => '/var/log/cinder/*.log*' + paths => [ '/var/log/cinder/*.log*' ], fields => { backend => $machine_role, }, tags => [ 'cinder', 'openstack', ], } + }else{ + file { '/etc/filebeat/conf.d/cinder.yml': ensure => 'absent', notify => Service['filebeat'] } } if $log_cloudkitty{ filebeat::input { 'cloudkitty': fields_under_root => true, - paths => '/var/log/cloudkitty/*.log*' + paths => [ '/var/log/cloudkitty/*.log*' ], fields => { backend => $machine_role, }, tags => [ 'cloudkitty', 'openstack', ], } + }else{ + file { '/etc/filebeat/conf.d/cloudkitty.yml': ensure => 'absent', notify => Service['filebeat'] } } if $log_corosync{ filebeat::input { 'corosync': fields_under_root => true, - paths => '/var/log/corosync/*.log*' + paths => [ '/var/log/corosync/*.log*' ], fields => { backend => $machine_role, }, tags => [ 'corosync', 'system', ], } + }else{ + file { '/etc/filebeat/conf.d/corosync.yml': ensure => 'absent', notify => Service['filebeat'] } } if $log_designate{ filebeat::input { 'designate': fields_under_root => true, - paths => '/var/log/designate/*.log*' + paths => [ '/var/log/designate/*.log*' ], fields => { backend => $machine_role, }, tags => [ 'designate', 'openstack', ], } + }else{ + file { '/etc/filebeat/conf.d/designate.yml': ensure => 'absent', notify => Service['filebeat'] } } if $log_etcd{ filebeat::input { 'etcd': fields_under_root => true, - paths => '/var/log/etcd/*.log*' + paths => [ '/var/log/etcd/*.log*' ], fields => { backend => $machine_role, }, tags => [ 'etcd', 'system', ], } + }else{ + file { '/etc/filebeat/conf.d/etcd.yml': ensure => 'absent', notify => Service['filebeat'] } } if $log_haproxy{ filebeat::input { 'haproxy': fields_under_root => true, - paths => '/var/log/haproxy.log*' + paths => [ '/var/log/haproxy.log*' ], fields => { backend => $machine_role, }, tags => [ 'haproxy', 'system', ], } + }else{ + file { '/etc/filebeat/conf.d/haproxy.yml': ensure => 'absent', notify => Service['filebeat'] } } if $log_horizon{ @@ -264,186 +281,228 @@ qPDlGRlOgVTd9xUfHFkzB52c70E= fields => { backend => $machine_role, }, tags => [ 'horizon', 'openstack', ], } + }else{ + file { '/etc/filebeat/conf.d/horizon.yml': ensure => 'absent', notify => Service['filebeat'] } } if $log_glance{ filebeat::input { 'glance': fields_under_root => true, - paths => '/var/log/glance/*.log*' + paths => [ '/var/log/glance/*.log*' ], fields => { backend => $machine_role, }, tags => [ 'glance', 'openstack', ], } + }else{ + file { '/etc/filebeat/conf.d/glance.yml': ensure => 'absent', notify => Service['filebeat'] } } if $log_gnocchi{ filebeat::input { 'gnocchi': fields_under_root => true, - paths => '/var/log/gnocchi/*.log*' + paths => [ '/var/log/gnocchi/*.log*' ], fields => { backend => $machine_role, }, tags => [ 'gnocchi', 'openstack', ], } + }else{ + file { '/etc/filebeat/conf.d/gnocchi.yml': ensure => 'absent', notify => Service['filebeat'] } } if $log_heat{ filebeat::input { 'heat': fields_under_root => true, - paths => '/var/log/heat/*.log*' + paths => [ '/var/log/heat/*.log*' ], fields => { backend => $machine_role, }, tags => [ 'heat', 'openstack', ], } + }else{ + file { '/etc/filebeat/conf.d/heat.yml': ensure => 'absent', notify => Service['filebeat'] } } if $log_keystone{ filebeat::input { 'keystone': fields_under_root => true, - paths => '/var/log/keystone/*.log*' + paths => [ '/var/log/keystone/*.log*' ], fields => { backend => $machine_role, }, tags => [ 'keystone', 'openstack', ], } + }else{ + file { '/etc/filebeat/conf.d/keystone.yml': ensure => 'absent', notify => Service['filebeat'] } } if $log_memcached{ filebeat::input { 'memcached': fields_under_root => true, - paths => '/var/log/memcached/*.log*' + paths => [ '/var/log/memcached/*.log*' ], fields => { backend => $machine_role, }, tags => [ 'memcached', 'system', ], } + }else{ + file { '/etc/filebeat/conf.d/memcached.yml': ensure => 'absent', notify => Service['filebeat'] } } if $log_magnum{ filebeat::input { 'magnum': fields_under_root => true, - paths => '/var/log/magnum/*.log*' + paths => [ '/var/log/magnum/*.log*' ], fields => { backend => $machine_role, }, tags => [ 'magnum', 'openstack', ], } + }else{ + file { '/etc/filebeat/conf.d/magnum.yml': ensure => 'absent', notify => Service['filebeat'] } } if $log_mysql{ filebeat::input { 'mysql': fields_under_root => true, - paths => '/var/log/mysql/*.log*' + paths => [ '/var/log/mysql/*.log*' ], fields => { backend => $machine_role, }, tags => [ 'mysql', 'system', ], } + }else{ + file { '/etc/filebeat/conf.d/mysql.yml': ensure => 'absent', notify => Service['filebeat'] } } if $log_neutron{ filebeat::input { 'neutron': fields_under_root => true, - paths => '/var/log/neutron/*.log*' + paths => [ '/var/log/neutron/*.log*' ], fields => { backend => $machine_role, }, tags => [ 'neutron', 'openstack', ], } + }else{ + file { '/etc/filebeat/conf.d/neutron.yml': ensure => 'absent', notify => Service['filebeat'] } } if $log_nova{ filebeat::input { 'nova': fields_under_root => true, - paths => '/var/log/nova/*.log*' + paths => [ '/var/log/nova/*.log*' ], fields => { backend => $machine_role, }, tags => [ 'nova', 'openstack', ], } + }else{ + file { '/etc/filebeat/conf.d/nova.yml': ensure => 'absent', notify => Service['filebeat'] } } if $log_octavia{ filebeat::input { 'octavia': fields_under_root => true, - paths => '/var/log/octavia/*.log*' + paths => [ '/var/log/octavia/*.log*' ], fields => { backend => $machine_role, }, tags => [ 'octavia', 'openstack', ], } + }else{ + file { '/etc/filebeat/conf.d/octavia.yml': ensure => 'absent', notify => Service['filebeat'] } } if $log_openvswitch{ filebeat::input { 'openvswitch': fields_under_root => true, - paths => '/var/log/openvswitch/*.log*' + paths => [ '/var/log/openvswitch/*.log*' ], fields => { backend => $machine_role, }, tags => [ 'openvswitch', 'system', ], } + }else{ + file { '/etc/filebeat/conf.d/openvswitch.yml': ensure => 'absent', notify => Service['filebeat'] } } if $log_panko{ filebeat::input { 'panko': fields_under_root => true, - paths => '/var/log/panko/*.log*' + paths => [ '/var/log/panko/*.log*' ], fields => { backend => $machine_role, }, tags => [ 'panko', 'openstack', ], } + }else{ + file { '/etc/filebeat/conf.d/panko.yml': ensure => 'absent', notify => Service['filebeat'] } } if $log_pacemaker{ filebeat::input { 'pacemaker': fields_under_root => true, - paths => '/var/log/pacemaker/*.log*' + paths => [ '/var/log/pacemaker/*.log*' ], fields => { backend => $machine_role, }, tags => [ 'pacemaker', 'system', ], } + }else{ + file { '/etc/filebeat/conf.d/pacemaker.yml': ensure => 'absent', notify => Service['filebeat'] } } if $log_placement{ filebeat::input { 'placement': fields_under_root => true, - paths => '/var/log/placement/*.log*' + paths => [ '/var/log/placement/*.log*' ], fields => { backend => $machine_role, }, tags => [ 'placement', 'openstack', ], } + }else{ + file { '/etc/filebeat/conf.d/placement.yml': ensure => 'absent', notify => Service['filebeat'] } } if $log_rabbitmq{ filebeat::input { 'rabbitmq': fields_under_root => true, - paths => '/var/log/rabbitmq/*.log*' + paths => [ '/var/log/rabbitmq/*.log*' ], fields => { backend => $machine_role, }, tags => [ 'rabbitmq', 'system', ], } + }else{ + file { '/etc/filebeat/conf.d/rabbitmq.yml': ensure => 'absent', notify => Service['filebeat'] } } if $log_swiftproxy{ filebeat::input { 'swiftproxy': fields_under_root => true, - paths => ['/var/log/swift/swift-proxy*.log*', '/var/log/swift/proxy*.log*'] + paths => ['/var/log/swift/swift-proxy*.log*', '/var/log/swift/proxy*.log*'], fields => { backend => $machine_role, }, tags => [ 'swift', 'swiftproxy', 'openstack', ], } + }else{ + file { '/etc/filebeat/conf.d/swiftproxy.yml': ensure => 'absent', notify => Service['filebeat'] } } if $log_swiftaccount{ - filebeat::input { 'swiftproxy': + filebeat::input { 'swiftaccount': fields_under_root => true, - paths => ['/var/log/swift/swift-account*.log*', '/var/log/swift/account*.log*'] + paths => ['/var/log/swift/swift-account*.log*', '/var/log/swift/account*.log*'], fields => { backend => $machine_role, }, tags => [ 'swift', 'swiftaccount', 'openstack', ], } + }else{ + file { '/etc/filebeat/conf.d/swiftaccount.yml': ensure => 'absent', notify => Service['filebeat'] } } if $log_swiftcontainer{ filebeat::input { 'swiftcontainer': fields_under_root => true, - paths => ['/var/log/swift/swift-container*.log*', '/var/log/swift/container*.log*'] + paths => ['/var/log/swift/swift-container*.log*', '/var/log/swift/container*.log*'], fields => { backend => $machine_role, }, tags => [ 'swift', 'swiftcontainer', 'openstack', ], } + }else{ + file { '/etc/filebeat/conf.d/swiftcontainer.yml': ensure => 'absent', notify => Service['filebeat'] } } if $log_swiftobject{ - filebeat::input { 'swiftcontainer': + filebeat::input { 'swiftobject': fields_under_root => true, - paths => '/var/log/swift/object*.log*' + paths => [ '/var/log/swift/object*.log*' ], fields => { backend => $machine_role, }, tags => [ 'swift', 'swiftobject', 'openstack', ], } + }else{ + file { '/etc/filebeat/conf.d/swiftobject.yml': ensure => 'absent', notify => Service['filebeat'] } } if $log_zookeeper{ - filebeat::input { 'rabbitmq': + filebeat::input { 'zookeeper': fields_under_root => true, - paths => '/var/log/zookeeper/*.log*' + paths => [ '/var/log/zookeeper/*.log*' ], fields => { backend => $machine_role, }, tags => [ 'zookeeper', 'system', ], } + }else{ + file { '/etc/filebeat/conf.d/zookeeper.yml': ensure => 'absent', notify => Service['filebeat'] } } } diff --git a/src/variables.json b/src/variables.json index 3f4cee58..081cb695 100644 --- a/src/variables.json +++ b/src/variables.json @@ -226,8 +226,8 @@ }, "filebeat_output_hosts": { "type": "string", - "default": "host1.example.com:5044%host2.example.com:5044", - "description": "Filebeat output hosts:" + "default": "host1.example.com:5044@host2.example.com:5044", + "description": "Filebeat output hosts (separated by @):" }, "filebeat_logging_level": { "type": "string", -- GitLab From 66e1ff1e33b47aba794a9ebe3f7661418c2b8476 Mon Sep 17 00:00:00 2001 From: Thomas Goirand Date: Sat, 17 Apr 2021 10:30:25 +0200 Subject: [PATCH 10/18] Fix ENC for messaging nodes --- src/inc/enc.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/inc/enc.php b/src/inc/enc.php index bea73238..a0fa0356 100644 --- a/src/inc/enc.php +++ b/src/inc/enc.php @@ -610,7 +610,7 @@ function puppet_enc($con,$conf){ $enc_rabbits .= $enc_rabbit_ips; // Fetch all other messaging (ie: all but this machine we're setting-up) - $qnfmsg = "SELECT machines.hostname AS hostname, INET_NTOA(ips.ip) AS ipaddr FROM machines, ips, networks WHERE machines.role='messaging' AND machines.cluster='$cluster_id' AND machines.id != '".$cluster["first_master_machine_id"]."' AND machines.id=ips.machine AND ips.network=networks.id AND networks.is_public='no' AND networks.role!='vm-net' AND networks.role!='vip' AND networks.role!='ovs-bridge' AND networks.role!='ceph-cluster' AND networks.role!='ipmi'"; + $qnfmsg = "SELECT machines.hostname AS hostname, INET_NTOA(ips.ip) AS ipaddr FROM machines, ips, networks WHERE machines.role='messaging' AND machines.cluster='$cluster_id' AND machines.id != '".$machine["id"]."' AND machines.id=ips.machine AND ips.network=networks.id AND networks.is_public='no' AND networks.role!='vm-net' AND networks.role!='vip' AND networks.role!='ovs-bridge' AND networks.role!='ceph-cluster' AND networks.role!='ipmi'"; $rnfmsg = mysqli_query($con, $qnfmsg); if($rnfmsg === FALSE){ $json["status"] = "error"; -- GitLab From fee3621d6c1e5c7efe8839f353ba76640b644023 Mon Sep 17 00:00:00 2001 From: Thomas Goirand Date: Sat, 17 Apr 2021 10:31:10 +0200 Subject: [PATCH 11/18] Add code to force a debian suite name for the MegaCli repo. --- bin/openstack-cluster-installer-build-live-image | 14 +++++++++++--- debian/changelog | 1 + .../openstack-cluster-installer.conf | 7 +++++++ 3 files changed, 19 insertions(+), 3 deletions(-) diff --git a/bin/openstack-cluster-installer-build-live-image b/bin/openstack-cluster-installer-build-live-image index d2651661..5b4fb21b 100755 --- a/bin/openstack-cluster-installer-build-live-image +++ b/bin/openstack-cluster-installer-build-live-image @@ -351,8 +351,16 @@ lXleArf9oJqxDO3TsV5zcLyxsIuRuxyP0+AKdSQUqv0dFi4Jf79OmvOmgwydhHjY -----END PGP PUBLIC KEY BLOCK-----" >config/archives/megacli.key.chroot pkgos_inifile get /etc/openstack-cluster-installer/openstack-cluster-installer.conf megacli megacli_repo MEGACLI_REPO=$RET - echo "deb ${MEGACLI_REPO} ${debian_release} main" >config/archives/megacli.list.chroot - echo "deb ${MEGACLI_REPO} ${debian_release} main" >config/archives/megacli.list.binary + pkgos_inifile get /etc/openstack-cluster-installer/openstack-cluster-installer.conf megacli megacli_force_debian_suite + MEGACLI_FORCE_DEBIAN_SUITE=$RET + if [ "${MEGACLI_FORCE_DEBIAN_SUITE}" = "yes" ] ; then + pkgos_inifile get /etc/openstack-cluster-installer/openstack-cluster-installer.conf megacli megacli_debian_suite + MEGACLI_DEBIAN_SUITE=$RET + else + MEGACLI_DEBIAN_SUITE=${debian_release} + fi + echo "deb ${MEGACLI_REPO} ${MEGACLI_DEBIAN_SUITE} main" >config/archives/megacli.list.chroot + echo "deb ${MEGACLI_REPO} ${MEGACLI_DEBIAN_SUITE} main" >config/archives/megacli.list.binary echo "megaraid-status megactl megamgr @@ -365,7 +373,7 @@ smartmontools" >config/package-lists/megacli.list.chroot if [ "${MEGACLI_IN_TARGET}" ] ; then echo "yes" >config/includes.chroot/etc/oci/setup-megacli-intarget cp config/archives/megacli.key.chroot config/includes.chroot/etc/oci/setup-megacli-intarget-key - echo "deb ${MEGACLI_REPO} ${debian_release} main" >config/includes.chroot/etc/oci/setup-megacli-intarget-repo + echo "deb ${MEGACLI_REPO} ${MEGACLI_DEBIAN_SUITE} main" >config/includes.chroot/etc/oci/setup-megacli-intarget-repo fi fi diff --git a/debian/changelog b/debian/changelog index 53f6cfef..cb1d8111 100644 --- a/debian/changelog +++ b/debian/changelog @@ -32,6 +32,7 @@ openstack-cluster-installer (41) UNRELEASED; urgency=medium package (the agent uses it). * Manage the number of API workers with $::os_workers and the new classes foo::wsgi::uwsgi contributed to puppet-openstack. + * Add code to force a debian suite name for the MegaCli repo. -- Thomas Goirand Tue, 16 Feb 2021 19:08:35 +0100 diff --git a/etc/openstack-cluster-installer/openstack-cluster-installer.conf b/etc/openstack-cluster-installer/openstack-cluster-installer.conf index b2142b0f..85b02c19 100644 --- a/etc/openstack-cluster-installer/openstack-cluster-installer.conf +++ b/etc/openstack-cluster-installer/openstack-cluster-installer.conf @@ -101,6 +101,13 @@ live_image_install_megacli=no # http://apt.infomaniak.ch/hwraid.le-vert.net/debian/ megacli_repo=http://hwraid.le-vert.net/debian +# Force a given suite name rather than the one defined +# as debian_release= +megacli_force_debian_suite=no + +# What suite to use if force_debian_suite is set to yes? +megacli_debian_suite=sid + # Should we automatically clear the MegaCli configuration after N # run of the hardware discovery? megacli_auto_clear=no -- GitLab From 633c22c3065beb771ab908703eb042293cb3a521 Mon Sep 17 00:00:00 2001 From: Thomas Goirand Date: Wed, 21 Apr 2021 22:13:48 +0200 Subject: [PATCH 12/18] Fix ownership of /var/lib/openstack-dashboard/secret-key/.secret_key_store --- puppet/manifests/controller.pp | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/puppet/manifests/controller.pp b/puppet/manifests/controller.pp index 12c1d3b5..5be77cbc 100644 --- a/puppet/manifests/controller.pp +++ b/puppet/manifests/controller.pp @@ -3507,6 +3507,13 @@ test -e \$mon_data/done cache_server_url => "${machine_hostname}:11211", } } + file { '/var/lib/openstack-dashboard/secret-key/.secret_key_store': + ensure => 'file', + mode => '0600', + owner => 'www-data', + group => 'root', + require => Anchor['horizon::service::end'], + } } } ###################### -- GitLab From f13831faf353d0df1180466e84aae2f344d2cf07 Mon Sep 17 00:00:00 2001 From: Thomas Goirand Date: Thu, 6 May 2021 00:04:59 +0200 Subject: [PATCH 13/18] Also set the NumaNodesPerSocket for R7525 nodes. --- common/usr/bin/oci-dell-set-default-racadm-config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/usr/bin/oci-dell-set-default-racadm-config b/common/usr/bin/oci-dell-set-default-racadm-config index 87fe023d..af47ee10 100755 --- a/common/usr/bin/oci-dell-set-default-racadm-config +++ b/common/usr/bin/oci-dell-set-default-racadm-config @@ -133,7 +133,7 @@ if [ "${IPMI_TYPE}" = "iDRAC6" ] || [ "${IPMI_TYPE}" = "iDRAC7" ] || [ "${IPMI_T ${RACADM} set BIOS.SataSettings.EmbSata AhciMode # NUMA Node Per Socket (NPS): - if [ "${PRODUCT_NAME}" = "PowerEdge R6525" ] ; then + if [ "${PRODUCT_NAME}" = "PowerEdge R6525" ] || [ "${PRODUCT_NAME}" = "PowerEdge R7525" ]; then CPU_MODEL=$(lscpu | grep "Model name:" | sed -e 's/Model name://') if echo "${CPU_MODEL}" | grep -q "AMD EPYC" ; then AMD_CPU_NAME=$(echo "${CPU_MODEL}" | sed -e 's/AMD EPYC //' | awk '{print $1}') -- GitLab From 6c4415cf100acc0df3824b9f2582f4f711cc1940 Mon Sep 17 00:00:00 2001 From: Thomas Goirand Date: Wed, 19 May 2021 11:41:25 +0200 Subject: [PATCH 14/18] ocicli cluster-install: install billmon and cephmon machines at the same time, same with OSDs. --- ocicli/ocicli | 67 ++++++++++++--------------------------------------- 1 file changed, 16 insertions(+), 51 deletions(-) diff --git a/ocicli/ocicli b/ocicli/ocicli index 8b330540..c54d585f 100755 --- a/ocicli/ocicli +++ b/ocicli/ocicli @@ -1160,34 +1160,38 @@ ocicli_cluster_install () { echo "-> swiftstore" SWIFTSTORES=$(ocicli -csv machine-list -a | q -H -d, "SELECT serial FROM - WHERE role='swiftstore' AND cluster='${CLUSTER_NAME}'" | tr '\n' ' ') - if [ -n "${BILLMONS}" ] ; then - echo "Installing Billing CEPH MON machines: ${BILLMONS}" - for i in ${BILLMONS} ; do + # Install the Ceph MON machines, they must be up before OSDs. + if [ -n "${BILLMONS}" ] || [ -n "${CEPH_MONS}" ] ; then + echo "Installing CEPH MON machines: ${BILLMONS} ${CEPH_MONS}" + for i in ${BILLMONS} ${CEPH_MONS} ; do STATUS=$(ocicli -csv machine-list -a | q -H -d, "SELECT status FROM - WHERE serial='${i}'") if [ "${STATUS}" = "live" ] ; then ocicli machine-install-os $i fi done - for i in ${BILLMONS} ; do - echo -n "Waiting for ${i} to be installed:" + for i in ${BILLMONS} ${CEPH_MONS} ; do + echo -n "Waiting for ${i} (Ceph MON) to be installed:" ocicli_cluster_wait_for $i status installed done echo "" fi - if [ -n "${BILLOSDS}" ] ; then - echo "Installing Billing CEPH OSD machines: ${BILLOSDS}" - for i in ${BILLOSDS} ; do + # Install the Ceph OSD machines, they must be up before messaging and controllers + if [ -n "${BILLOSDS}" ] || [ -n "${CEPH_OSDS}" ] || [ -n "${COMPUTE_WITH_CEPH_OSDS}" ] ; then + echo "Installing Billing CEPH OSD machines: ${BILLOSDS} ${CEPH_OSDS} ${COMPUTE_WITH_CEPH_OSDS}" + for i in ${BILLOSDS} ${CEPH_OSDS} ${COMPUTE_WITH_CEPH_OSDS} ; do STATUS=$(ocicli -csv machine-list -a | q -H -d, "SELECT status FROM - WHERE serial='${i}'") if [ "${STATUS}" = "live" ] ; then ocicli machine-install-os $i fi done - for i in ${BILLOSDS} ; do + for i in ${BILLOSDS} ${CEPH_OSDS} ${COMPUTE_WITH_CEPH_OSDS} ; do echo -n "Waiting for ${i} to be installed:" ocicli_cluster_wait_for $i status installed done echo "" fi + # Install the messaging machines, they must be up before the controllers, + # otherwise, keystone bootstrap fails. if [ -n "${MESSAGINGS}" ] ; then echo "Installing MESSAGING machines: ${MESSAGINGS}" for i in ${MESSAGINGS} ; do @@ -1197,48 +1201,7 @@ ocicli_cluster_install () { fi done fi - if [ -n "${CEPH_MONS}" ] ; then - echo "Installing CEPH MON machines: ${CEPH_MONS}" - for i in ${CEPH_MONS} ; do - STATUS=$(ocicli -csv machine-list -a | q -H -d, "SELECT status FROM - WHERE serial='${i}'") - if [ "${STATUS}" = "live" ] ; then - ocicli machine-install-os $i - fi - done - for i in ${CEPH_MONS} ; do - echo -n "Waiting for ${i} to be installed:" - ocicli_cluster_wait_for $i status installed - done - echo "" - fi - if [ -n "${CEPH_OSDS}" ] ; then - echo "Installing CEPH OSD machines: ${CEPH_OSDS}" - for i in ${CEPH_OSDS} ; do - STATUS=$(ocicli -csv machine-list -a | q -H -d, "SELECT status FROM - WHERE serial='${i}'") - if [ "${STATUS}" = "live" ] ; then - ocicli machine-install-os $i - fi - done - for i in ${CEPH_OSDS} ; do - echo -n "Waiting for ${i} to be installed:" - ocicli_cluster_wait_for $i status installed - done - echo "" - fi - if [ -n "${COMPUTE_WITH_CEPH_OSDS}" ] ; then - echo "Installing compute with CEPH OSD machines: ${COMPUTE_WITH_CEPH_OSDS}" - for i in ${COMPUTE_WITH_CEPH_OSDS} ; do - STATUS=$(ocicli -csv machine-list -a | q -H -d, "SELECT status FROM - WHERE serial='${i}'") - if [ "${STATUS}" = "live" ] ; then - ocicli machine-install-os $i - fi - done - for i in ${COMPUTE_WITH_CEPH_OSDS} ; do - echo -n "Waiting for ${i} to be installed:" - ocicli_cluster_wait_for $i status installed - done - echo "" - fi + # Install the controllers. They must be up before any other types. if [ -n "${CONTROLLERS}" ] ; then echo "Installing CONTROLLER machines: ${CONTROLLERS}" for i in ${CONTROLLERS} ; do @@ -1256,6 +1219,7 @@ ocicli_cluster_install () { ocicli_cluster_wait_for_puppet $i done fi + # Install the network machines. They must be up before compute nodes. if [ -n "${NETWORKS}" ] ; then echo "Installing NETWORK machines: ${NETWORKS}" for i in ${NETWORKS} ; do @@ -1273,6 +1237,7 @@ ocicli_cluster_install () { ocicli_cluster_wait_for_puppet $i done fi + # Install all the other machines, in no particular order. if [ -n "${COMPUTES}" ] || [ -n "${VOLUMES}" ] || [ -n "${SWIFTPROXIES}" ] || [ -n "${SWIFTSTORES}" ] ; then echo "Installing COMPUTE, VOLUME, SWIFTPROXY and SWIFTSTORE machines: ${COMPUTES} ${VOLUMES} ${SWIFTPROXIES} ${SWIFTSTORES}" for i in ${COMPUTES} ${VOLUMES} ${SWIFTPROXIES} ${SWIFTSTORES} ; do -- GitLab From 1f2d62367563f81437b29ed04abbbcdad4366ef9 Mon Sep 17 00:00:00 2001 From: Thomas Goirand Date: Tue, 1 Jun 2021 17:45:04 +0200 Subject: [PATCH 15/18] Fix-up oci-block-device-udev-sorting for newer HP DL385 Serial Attached SCSI controller: Adaptec Smart Storage PQI 12G SAS/PCIe 3 (rev 01) --- common/usr/bin/oci-block-device-udev-sorting | 149 +++++++++---------- 1 file changed, 71 insertions(+), 78 deletions(-) diff --git a/common/usr/bin/oci-block-device-udev-sorting b/common/usr/bin/oci-block-device-udev-sorting index b86041eb..55ede4f6 100755 --- a/common/usr/bin/oci-block-device-udev-sorting +++ b/common/usr/bin/oci-block-device-udev-sorting @@ -15,6 +15,8 @@ else exit 0 fi +logger -t "udev" "===> DEVPATH=${DEVPATH} ID_BUS=${ID_BUS}" + # Script param (sent as env var): # ID_BUS (example: scsi, ata) # DEVPATH (example: /devices/pci0000:80/0000:80:08.2/0000:81:00.0/ata8/host7/target7:0:0/7:0:0:0/block/sdc) @@ -95,91 +97,82 @@ case "${CHASSIS_MANUFACTURER}" in case "${CHASSIS_MODEL}" in "ProLiant DL385 Gen10 Plus") card=$(echo $DEVPATH | cut -d/ -f6) + target=$(echo $DEVPATH | awk -F '/' '{ print $9 }') # We expect here 2 SATA SSDs and 12 SAS HDD - if [ -n "$ID_BUS" ] && [ "$ID_BUS" = "ata" ]; then - target=$(echo $DEVPATH | awk -F '/' '{ print $9 }') - if [ "$card" = "ata7" ] && [ "$target" = "6:0:0:0" ]; then - disk="sda" - elif [ "$card" = "ata8" ] && [ "$target" = "7:0:0:0" ]; then - disk="sdb" - else - logger -t "udev" "Device $DEVPATH detected but not supported." - exit 1 - fi - elif [ -n "$ID_BUS" ] && [ "$ID_BUS" = "scsi" ]; then - if [ "${card}" = "host8" ] ; then - usecache=false - # read a cache file, if exist and recent - if [ -f $CACHE_FILE ]; then - lastentry=$(stat -c %Y $CACHE_FILE) - lastvalid=$(date -d "now -3 min" +%s) - if [ $lastentry -gt $lastvalid ]; then - usecache=true - else - rm $CACHE_FILE - fi - elif [ -e $CACHE_FILE ]; then + if [ "${card}" = "ata7" ] && [ "${target}" = "6:0:0:0" ]; then + disk="sda" + elif [ "${card}" = "ata8" ] && [ "${target}" = "7:0:0:0" ]; then + disk="sdb" + elif [ "${card}" = "host8" ] ; then + usecache=false + # read a cache file, if exist and recent + if [ -f $CACHE_FILE ]; then + lastentry=$(stat -c %Y $CACHE_FILE) + lastvalid=$(date -d "now -3 min" +%s) + if [ $lastentry -gt $lastvalid ]; then + usecache=true + else rm $CACHE_FILE fi - # cache is recent and a file - if $usecache; then - # Just read from cache file... - disk=$(grep $DEVNAME $CACHE_FILE | awk '{ print $2 }') - if ! [ -n $disk ]; then - logger -t "udev" "[Cache mode] Device $DEVPATH detected but not found." - exit 1 - fi - else - disk="" + elif [ -e $CACHE_FILE ]; then + rm $CACHE_FILE + fi + # cache is recent and a file + if $usecache; then + # Just read from cache file... + disk=$(grep $DEVNAME $CACHE_FILE | awk '{ print $2 }') + if ! [ -n $disk ]; then + logger -t "udev" "[Cache mode] Device $DEVPATH detected but not found." + exit 1 + fi + else + disk="" - # Parse the output of "ssacli controller slot=0 physicaldrive all show detail" - # and build a cache file... - TMPFILE=$(mktemp -t parse-hpe-drives.XXXXXX) - # The grep here just helps because we later have less lines to parse. - SLOT_NUM=$(ssacli controller all show | head -n 2 | tail -n 1 | sed -r 's/.*(Slot [0-9]+).*/\1/' | sed 's/Slot //') - ssacli controller slot=${SLOT_NUM} physicaldrive all show detail | grep -E 'physicaldrive|Disk Name: /dev/' >$TMPFILE - while IFS='' read -r LINE || [ -n "${LINE}" ]; do - if echo ${LINE} | grep -q physicaldrive ; then - PHYSDRIVE=$(echo ${LINE} | sed 's/^.*physicaldrive //') - case "${PHYSDRIVE}" in - # The format is: - # ${SAS_PORT}:${BOX}:${BAY} - # Since we don't care about ports, but where the drive is physically located - # we use ${BOX}:${BAY}, so the physical location of the drives is deterministic - # and we don't care the order of the ports. - "1I:1:1"|"2I:1:1"|"3I:1:1") PHYDISK="sdc" ;; - "1I:1:2"|"2I:1:2"|"3I:1:2") PHYDISK="sdd" ;; - "1I:1:3"|"2I:1:3"|"3I:1:3") PHYDISK="sde" ;; - "1I:1:4"|"2I:1:4"|"3I:1:4") PHYDISK="sdf" ;; - "1I:2:1"|"2I:2:1"|"3I:2:1") PHYDISK="sdg" ;; - "1I:2:2"|"2I:2:2"|"3I:2:2") PHYDISK="sdh" ;; - "1I:2:3"|"2I:2:3"|"3I:2:3") PHYDISK="sdi" ;; - "1I:2:4"|"2I:2:4"|"3I:2:4") PHYDISK="sdj" ;; - "1I:3:1"|"2I:3:1"|"3I:3:1") PHYDISK="sdk" ;; - "1I:3:2"|"2I:3:2"|"3I:3:2") PHYDISK="sdl" ;; - "1I:3:3"|"2I:3:3"|"3I:3:3") PHYDISK="sdm" ;; - "1I:3:4"|"2I:3:4"|"3I:3:4") PHYDISK="sdn" ;; - esac - elif echo ${LINE} | grep -q 'Disk Name: /dev/' ; then - MY_DEV_NAME=$(echo ${LINE} | sed 's#^.*Disk Name: ##') - if [ -n "${PHYDISK}" ] ; then - echo "$MY_DEV_NAME $PHYDISK" >> $CACHE_FILE - unset PHYDISK - fi - unset MY_DEV_NAME + # Parse the output of "ssacli controller slot=0 physicaldrive all show detail" + # and build a cache file... + TMPFILE=$(mktemp -t parse-hpe-drives.XXXXXX) + # The grep here just helps because we later have less lines to parse. + SLOT_NUM=$(ssacli controller all show | head -n 2 | tail -n 1 | sed -r 's/.*(Slot [0-9]+).*/\1/' | sed 's/Slot //') + ssacli controller slot=${SLOT_NUM} physicaldrive all show detail | grep -E 'physicaldrive|Disk Name: /dev/' >$TMPFILE + while IFS='' read -r LINE || [ -n "${LINE}" ]; do + if echo ${LINE} | grep -q physicaldrive ; then + PHYSDRIVE=$(echo ${LINE} | sed 's/^.*physicaldrive //') + case "${PHYSDRIVE}" in + # The format is: + # ${SAS_PORT}:${BOX}:${BAY} + # Since we don't care about ports, but where the drive is physically located + # we use ${BOX}:${BAY}, so the physical location of the drives is deterministic + # and we don't care the order of the ports. + "1I:1:1"|"2I:1:1"|"3I:1:1") PHYDISK="sdc" ;; + "1I:1:2"|"2I:1:2"|"3I:1:2") PHYDISK="sdd" ;; + "1I:1:3"|"2I:1:3"|"3I:1:3") PHYDISK="sde" ;; + "1I:1:4"|"2I:1:4"|"3I:1:4") PHYDISK="sdf" ;; + "1I:2:1"|"2I:2:1"|"3I:2:1") PHYDISK="sdg" ;; + "1I:2:2"|"2I:2:2"|"3I:2:2") PHYDISK="sdh" ;; + "1I:2:3"|"2I:2:3"|"3I:2:3") PHYDISK="sdi" ;; + "1I:2:4"|"2I:2:4"|"3I:2:4") PHYDISK="sdj" ;; + "1I:3:1"|"2I:3:1"|"3I:3:1") PHYDISK="sdk" ;; + "1I:3:2"|"2I:3:2"|"3I:3:2") PHYDISK="sdl" ;; + "1I:3:3"|"2I:3:3"|"3I:3:3") PHYDISK="sdm" ;; + "1I:3:4"|"2I:3:4"|"3I:3:4") PHYDISK="sdn" ;; + esac + elif echo ${LINE} | grep -q 'Disk Name: /dev/' ; then + MY_DEV_NAME=$(echo ${LINE} | sed 's#^.*Disk Name: ##') + if [ -n "${PHYDISK}" ] ; then + echo "$MY_DEV_NAME $PHYDISK" >> $CACHE_FILE + unset PHYDISK fi - done <$TMPFILE - rm -f ${TMPFILE} - - # Once it's built, just read from the cache file. - disk=$(grep $DEVNAME $CACHE_FILE | awk '{ print $2 }') - if ! [ -n $disk ]; then - logger -t "udev" "[Normal mode] Device $DEVPATH detected but not found." - exit 1 + unset MY_DEV_NAME fi + done <$TMPFILE + rm -f ${TMPFILE} + + # Once it's built, just read from the cache file. + disk=$(grep $DEVNAME $CACHE_FILE | awk '{ print $2 }') + if ! [ -n $disk ]; then + logger -t "udev" "[Normal mode] Device $DEVPATH detected but not found." + exit 1 fi - else - logger -t "udev" "Device $DEVPATH detected but not supported"; exit 1 fi else logger -t "udev" "Device $DEVPATH detected but not supported"; exit 1 -- GitLab From fdbd056579b6c54f174e68d33f8b682cf1d37d1b Mon Sep 17 00:00:00 2001 From: Thomas Goirand Date: Thu, 15 Jul 2021 09:53:36 +0200 Subject: [PATCH 16/18] compute.pp: remove haproxy in non-DVR mode, add haproxy healthcheck listen in DVR mode --- puppet/manifests/compute.pp | 119 ++++++++++++++++++++---------------- 1 file changed, 68 insertions(+), 51 deletions(-) diff --git a/puppet/manifests/compute.pp b/puppet/manifests/compute.pp index f4e2582b..bc581f55 100644 --- a/puppet/manifests/compute.pp +++ b/puppet/manifests/compute.pp @@ -208,59 +208,76 @@ class oci::compute( ############################################################ ### HAProxy (used for metadata server proxying and such) ### ############################################################ - class { 'haproxy': - global_options => { - 'log' => '/dev/log local0', - 'maxconn' => '256', - 'user' => 'haproxy', - 'group' => 'haproxy', - 'daemon' => '', - 'nbthread' => '4', - }, - defaults_options => { - 'mode' => 'http', - 'option' => [ - 'httplog', - ], - }, - merge_options => true, - }-> - haproxy::listen { 'hapstats': - section_name => 'statshttp', - bind => { "${machine_ip}:8088" => 'user root'}, - mode => 'http', - options => { - 'stats' => [ 'uri /', "auth admin:${pass_haproxy_stats}", 'refresh 15', 'admin if TRUE', 'realm Haproxy Statistics', ], + if $neutron_use_dvr { + class { 'haproxy': + global_options => { + 'log' => '/dev/log local0', + 'maxconn' => '256', + 'user' => 'haproxy', + 'group' => 'haproxy', + 'daemon' => '', + 'nbthread' => '4', }, - } + defaults_options => { + 'mode' => 'http', + 'option' => [ + 'httplog', + ], + }, + merge_options => true, + }-> + haproxy::listen { 'hapstats': + section_name => 'statshttp', + bind => { "${machine_ip}:8088" => 'user root'}, + mode => 'http', + options => { + 'stats' => [ 'uri /', "auth admin:${pass_haproxy_stats}", 'refresh 15', 'admin if TRUE', 'realm Haproxy Statistics', ], + }, + }-> + haproxy::listen { 'haphealthcheck': + section_name => 'healthcheck', + bind => { "${machine_ip}:8081" => 'user root'}, + mode => 'health', + options => [ + { 'option' => 'httpchk' }, + { 'option' => 'http-keep-alive' }, - $haproxy_options_for_metadata_proxy = [ - { 'use_backend' => 'metadatabe' }, - ] - haproxy::frontend { 'openstackfe': - mode => 'http', - bind => { "127.0.0.1:8775" => [] }, - options => $haproxy_options_for_metadata_proxy, - } - haproxy::backend { 'metadatabe': - options => [ - { 'option' => 'forwardfor' }, - { 'option' => 'httpchk GET /healthcheck' }, - { 'mode' => 'http' }, - { 'balance' => 'roundrobin' }, - ], - } - if $openstack_release == 'rocky'{ - $metadatabm_options = 'check' - }else{ - $metadatabm_options = 'check check-ssl ssl verify required ca-file /etc/ssl/certs/oci-pki-oci-ca-chain.pem' - } - haproxy::balancermember { 'metadatabm': - listening_service => 'metadatabe', - ipaddresses => $all_masters_ip, - server_names => $all_masters, - ports => 8775, - options => $metadatabm_options + ] + } + + $haproxy_options_for_metadata_proxy = [ + { 'use_backend' => 'metadatabe' }, + ] + haproxy::frontend { 'openstackfe': + mode => 'http', + bind => { "127.0.0.1:8775" => [] }, + options => $haproxy_options_for_metadata_proxy, + } + haproxy::backend { 'metadatabe': + options => [ + { 'option' => 'forwardfor' }, + { 'option' => 'httpchk GET /healthcheck' }, + { 'mode' => 'http' }, + { 'balance' => 'roundrobin' }, + ], + } + if $openstack_release == 'rocky'{ + $metadatabm_options = 'check' + }else{ + $metadatabm_options = 'check check-ssl ssl verify required ca-file /etc/ssl/certs/oci-pki-oci-ca-chain.pem' + } + haproxy::balancermember { 'metadatabm': + listening_service => 'metadatabe', + ipaddresses => $all_masters_ip, + server_names => $all_masters, + ports => 8775, + options => $metadatabm_options + } + } else { + package { 'remove-haproxy-in-non-dvr': + name => 'haproxy', + ensure => purged, + } } ############ -- GitLab From 0b357da311a5c1bcc8f0aeebb9f9b1d14ebb8fc3 Mon Sep 17 00:00:00 2001 From: Thomas Goirand Date: Fri, 20 Aug 2021 17:19:38 +0200 Subject: [PATCH 17/18] Now also report CPU from machines. --- .../usr/bin/openstack-cluster-installer-agent | 21 ++++-- ocicli/ocicli | 15 ++++ src/oci_db.php | 6 ++ src/report.php | 70 ++++++++++++++++++- 4 files changed, 103 insertions(+), 9 deletions(-) diff --git a/common/usr/bin/openstack-cluster-installer-agent b/common/usr/bin/openstack-cluster-installer-agent index 912f565d..cd2394db 100755 --- a/common/usr/bin/openstack-cluster-installer-agent +++ b/common/usr/bin/openstack-cluster-installer-agent @@ -182,6 +182,14 @@ fi PRODUCT_NAME=$(dmidecode -t 1 | grep "Product Name" | sed -e "s/[ \t]*Product Name:[ ]*//" -e "s/,//g") BIOS_VERSION=$(dmidecode -s bios-version) +echo "===> Fetching CPU info" +CPU_VENDOR=$(lscpu | grep "^Vendor ID:" | sed 's/Vendor ID:[ ]*//') +CPU_THREADS=$(lscpu | grep '^CPU(s):' | sed 's/CPU(s):[ ]*//') +CPU_CORE_PER_SOCKET=$(lscpu | grep '^Core(s) per socket:' | sed 's/Core(s) per socket:[ ]*//') +CPU_SOCKETS=$(lscpu | grep '^Socket(s):' | sed 's/Socket(s):[ ]*//') +CPU_MODEL_NAME=$(lscpu | grep '^Model name:' | sed 's/Model name:[ ]*//') +CPU_MHZ=$(lscpu | grep '^CPU MHz:' | sed 's/CPU MHz:[ ]*//') + echo "===> Fetching storage controllers" TMPSTRAGE_CTRL=$(mktemp -t openstack-cluster-installer-agent_storage_ctrl.XXXXXX) lshw -class storage -json 2>/dev/null >${TMPSTRAGE_CTRL} @@ -201,6 +209,11 @@ for store_ctrl_id in $(cat ${TMPSTRAGE_CTRL} | jq -r '.[]["id"]') ; do if echo "${PRODUCT}" | grep -q -E 'MegaRAID' ; then CTRL_TYPE="LSI-raid" BLKCTRL_FIRMWARE_VERSION=$(megacli -Version -Ctrl -aall | grep "Fw Package Build :" | sed -e 's/Fw Package Build : //') + elif [ "${PRODUCT}" = "Smart Storage PQI SAS" ] || [ ""$(cat ${TMPSTRAGE_CTRL} | jq -r '.[] | select(.id == "'${store_ctrl_id}'") | .["configuration"]["driver"]') = "smartpqi" -a "${PRODUCT}" = "Adaptec" ] ; then + CTRL_TYPE="SSA" + PRODUCT="Smart Storage PQI SAS" + SSA_SLOT_NUMBER=$(ssacli controller all show | head -n 2 | tail -n 1 | sed -r 's/.*(Slot [0-9]+).*/\1/' | sed 's/Slot //') + BLKCTRL_FIRMWARE_VERSION=$(ssacli controller slot=${SSA_SLOT_NUMBER} show | grep "Firmware Version" | sed -e 's/ Firmware Version: //') else case "${PRODUCT}" in # r740xd with PERC H330-mini @@ -208,11 +221,6 @@ for store_ctrl_id in $(cat ${TMPSTRAGE_CTRL} | jq -r '.[]["id"]') ; do CTRL_TYPE="LSI-noraid" BLKCTRL_FIRMWARE_VERSION=$(perccli show J | jq -r '.["Controllers"][]["Response Data"]["StoreLib IR3 Version"]') ;; - "Smart Storage PQI SAS") - CTRL_TYPE="SSA" - SSA_SLOT_NUMBER=$(ssacli controller all show | head -n 2 | tail -n 1 | sed -r 's/.*(Slot [0-9]+).*/\1/' | sed 's/Slot //') - BLKCTRL_FIRMWARE_VERSION=$(ssacli controller slot=${SSA_SLOT_NUMBER} show | grep "Firmware Version" | sed -e 's/ Firmware Version: //') - ;; "Lewisburg SATA Controller [AHCI mode]") CTRL_TYPE="HP-JBOD" BLKCTRL_FIRMWARE_VERSION="0.0.0" @@ -327,10 +335,9 @@ else fi echo ' "machine": [' >>${TMPFILE2} -echo ' {"serial": "'${SYSTEM_SERIAL}'", "productname": "'${PRODUCT_NAME}'", "bios_version": "'${BIOS_VERSION}'", "dell_lifecycle_version": "'${LIFECYCLE_VERSION}'", "ipmi_firmware_version": "'${IPMI_FIRMWARE_VERSION}'", "ipmi_detected_ip": "'${IPMI_DETECTED_IP}'", "ladvd_report": "none", "use_oci_sort_dev":"'${USE_OCI_SORT_DEV}'"}' >>${TMPFILE2} +echo ' {"serial": "'${SYSTEM_SERIAL}'", "productname": "'${PRODUCT_NAME}'", "bios_version": "'${BIOS_VERSION}'", "dell_lifecycle_version": "'${LIFECYCLE_VERSION}'", "ipmi_firmware_version": "'${IPMI_FIRMWARE_VERSION}'", "ipmi_detected_ip": "'${IPMI_DETECTED_IP}'", "ladvd_report": "none", "use_oci_sort_dev":"'${USE_OCI_SORT_DEV}'", "cpu_vendor": "'${CPU_VENDOR}'", "cpu_threads": "'${CPU_THREADS}'", "cpu_core_per_socket": "'${CPU_CORE_PER_SOCKET}'", "cpu_sockets": "'${CPU_SOCKETS}'", "cpu_model_name": "'${CPU_MODEL_NAME}'", "cpu_mhz": "'${CPU_MHZ}'"}' >>${TMPFILE2} echo ' ],' >>${TMPFILE2} - ### Get total amount of RAM TOTAL_RAM=$(dmidecode -t memory | awk '/Size: [0-9]/ {tmp=$2 ; if($3 == "GB") { tmp = $2 * 1024 }; t+=tmp;} END { print t}') RAM_TYPE=$(dmidecode -t 17 | grep Type | head -n 1 | awk '{print $2}') diff --git a/ocicli/ocicli b/ocicli/ocicli index ea56e283..8f52ac33 100755 --- a/ocicli/ocicli +++ b/ocicli/ocicli @@ -307,6 +307,14 @@ ocicli_machine_show (){ OCICLI_MACHINE_SHOW_DELL_LIFECYCLE_VER=$(cat ${API_RESPONSE_FILE} | jq -r '.["data"]| "\(.dell_lifecycle_version)"') OCICLI_MACHINE_SHOW_LADVD_REPORT=$(cat ${API_RESPONSE_FILE} | jq -r '.["data"]| "\(.ladvd_report)"') + # CPU infos + OCICLI_MACHINE_SHOW_CPU_VENDOR=$(cat ${API_RESPONSE_FILE} | jq -r '.["data"]| "\(.cpu_vendor)"') + OCICLI_MACHINE_SHOW_CPU_THREADS=$(cat ${API_RESPONSE_FILE} | jq -r '.["data"]| "\(.cpu_threads)"') + OCICLI_MACHINE_SHOW_CPU_CORE_PER_SOCKET=$(cat ${API_RESPONSE_FILE} | jq -r '.["data"]| "\(.cpu_core_per_socket)"') + OCICLI_MACHINE_SHOW_CPU_SOCKETS=$(cat ${API_RESPONSE_FILE} | jq -r '.["data"]| "\(.cpu_sockets)"') + OCICLI_MACHINE_SHOW_CPU_MODEL_NAME=$(cat ${API_RESPONSE_FILE} | jq -r '.["data"]| "\(.cpu_model_name)"') + OCICLI_MACHINE_SHOW_CPU_MHZ=$(cat ${API_RESPONSE_FILE} | jq -r '.["data"]| "\(.cpu_mhz)"') + # Location attributes OCICLI_MACHINE_SHOW_LOC_DC=$(cat ${API_RESPONSE_FILE} | jq -r '.["data"]| "\(.loc_dc)"') OCICLI_MACHINE_SHOW_LOC_ROW=$(cat ${API_RESPONSE_FILE} | jq -r '.["data"]| "\(.loc_row)"') @@ -458,6 +466,13 @@ ocicli_machine_show (){ done echo "" + echo "==== CPU =============,===============================================" + echo "Vendor:,${OCICLI_MACHINE_SHOW_CPU_VENDOR}" + echo "Model name:,${OCICLI_MACHINE_SHOW_CPU_MODEL_NAME}" + echo "MHz:,${OCICLI_MACHINE_SHOW_CPU_MHZ}" + echo "Threads:,${OCICLI_MACHINE_SHOW_CPU_THREADS}" + echo "Core per socket:,${OCICLI_MACHINE_SHOW_CPU_CORE_PER_SOCKET}" + echo "Sockets:,${OCICLI_MACHINE_SHOW_CPU_SOCKETS}" echo "==== NETWORKING ======,===============================================" echo "LADVD report:,${OCICLI_MACHINE_SHOW_LADVD_REPORT}" diff --git a/src/oci_db.php b/src/oci_db.php index 9b078d98..1ee224c0 100644 --- a/src/oci_db.php +++ b/src/oci_db.php @@ -197,6 +197,12 @@ $database = array( "ceph_osd_initial_setup" => "enum('yes','no') NOT NULL default 'yes'", "force_no_bgp2host" => "enum('yes','no') NOT NULL default 'no'", "use_oci_sort_dev" => "enum('yes','no') NOT NULL default 'no'", + "cpu_vendor" => "varchar(32) NOT NULL default 'none'", + "cpu_model_name" => "varchar(64) NOT NULL default 'none'", + "cpu_mhz" => "varchar(16) NOT NULL default '0'", + "cpu_threads" => "int(11) NOT NULL default '0'", + "cpu_core_per_socket" => "int(11) NOT NULL default '0'", + "cpu_sockets" => "int(11) NOT NULL default '0'", ), "primary" => "(id)", "keys" => array( diff --git a/src/report.php b/src/report.php index 3c880d81..aa092ddc 100644 --- a/src/report.php +++ b/src/report.php @@ -122,10 +122,76 @@ if (!isset($data["machine"][0]["use_oci_sort_dev"])){ } } +if (!isset($data["machine"][0]["cpu_vendor"])){ + $cpu_vendor = "none"; +}else{ + $reg = '/^[ @:.0-9a-zA-Z\(\)]{1,120}$/'; + if(!preg_match($reg, $data["machine"][0]["cpu_vendor"])){ + $cpu_vendor = "none"; + }else{ + $cpu_vendor = $data["machine"][0]["cpu_vendor"]; + } +} + +if (!isset($data["machine"][0]["cpu_model_name"])){ + $cpu_model_name = "none"; +}else{ + $reg = '/^[ @:.0-9a-zA-Z\(\)]{1,120}$/'; + if(!preg_match($reg, $data["machine"][0]["cpu_model_name"])){ + $cpu_model_name = "none"; + }else{ + $cpu_model_name = $data["machine"][0]["cpu_model_name"]; + } +} + +if (!isset($data["machine"][0]["cpu_mhz"])){ + $cpu_mhz = "none"; +}else{ + $reg = '/^[ :.0-9a-fA-F]{1,120}$/'; + if(!preg_match($reg, $data["machine"][0]["cpu_mhz"])){ + $cpu_mhz = "none"; + }else{ + $cpu_mhz = $data["machine"][0]["cpu_mhz"]; + } +} + +if (!isset($data["machine"][0]["cpu_threads"])){ + $cpu_threads = 0; +}else{ + $reg = '/^[0-9]{1,9}$/'; + if(!preg_match($reg, $data["machine"][0]["cpu_threads"])){ + $cpu_threads = 0; + }else{ + $cpu_threads = $data["machine"][0]["cpu_threads"]; + } +} + +if (!isset($data["machine"][0]["cpu_core_per_socket"])){ + $cpu_core_per_socket = 0; +}else{ + $reg = '/^[0-9]{1,6}$/'; + if(!preg_match($reg, $data["machine"][0]["cpu_core_per_socket"])){ + $cpu_core_per_socket = 0; + }else{ + $cpu_core_per_socket = $data["machine"][0]["cpu_core_per_socket"]; + } +} + +if (!isset($data["machine"][0]["cpu_sockets"])){ + $cpu_sockets = 0; +}else{ + $reg = '/^[0-9]{1,6}$/'; + if(!preg_match($reg, $data["machine"][0]["cpu_sockets"])){ + $cpu_sockets = 0; + }else{ + $cpu_sockets = $data["machine"][0]["cpu_sockets"]; + } +} + $r = mysqli_query($con, "SELECT * FROM machines WHERE serial='" . $safe_serial_number . "'"); $n = mysqli_num_rows($r); if($n == 0){ - $r = mysqli_query($con, "INSERT INTO machines (ipaddr, dhcp_ip, serial, product_name, ladvd_report, bios_version, dell_lifecycle_version, ipmi_firmware_version, ipmi_detected_ip, memory, use_oci_sort_dev, status) VALUES ('" . $remote_addr. "', '" . $remote_addr. "', '$safe_serial_number' , '$prod_name', '$ladvd_report', '$bios_version', '$dell_lifecycle_version', '$ipmi_version', '$ipmi_detected_ip', '$safe_memory_size', '$use_oci_sort_dev', 'live')"); + $r = mysqli_query($con, "INSERT INTO machines (ipaddr, dhcp_ip, serial, product_name, ladvd_report, bios_version, dell_lifecycle_version, ipmi_firmware_version, ipmi_detected_ip, memory, use_oci_sort_dev, cpu_vendor, cpu_model_name, cpu_mhz, cpu_threads, cpu_core_per_socket, cpu_sockets, status) VALUES ('" . $remote_addr. "', '" . $remote_addr. "', '$safe_serial_number' , '$prod_name', '$ladvd_report', '$bios_version', '$dell_lifecycle_version', '$ipmi_version', '$ipmi_detected_ip', '$safe_memory_size', '$use_oci_sort_dev', '$cpu_vendor', '$cpu_model_name', '$cpu_mhz', '$cpu_threads', '$cpu_core_per_socket', '$cpu_sockets', 'live')"); if($r === FALSE){ $error = mysqli_error($con); printf("Error inserting: %s\n", mysqli_error($con)); @@ -262,7 +328,7 @@ if($n == 0){ $report_counter = $a["report_counter"] + 1; // We keep the machine id... - $req = "UPDATE machines SET report_counter='$report_counter', ipaddr='$remote_addr', memory='".$safe_memory_size."', serial='$safe_serial_number', product_name='$prod_name', ladvd_report='$ladvd_report', bios_version='$bios_version', dell_lifecycle_version='$dell_lifecycle_version', ipmi_firmware_version='$ipmi_version', ipmi_detected_ip='$ipmi_detected_ip', use_oci_sort_dev='$use_oci_sort_dev', lastseen=NOW() $dhcp_update WHERE serial='$safe_serial_number'"; + $req = "UPDATE machines SET report_counter='$report_counter', ipaddr='$remote_addr', memory='".$safe_memory_size."', serial='$safe_serial_number', product_name='$prod_name', ladvd_report='$ladvd_report', bios_version='$bios_version', dell_lifecycle_version='$dell_lifecycle_version', ipmi_firmware_version='$ipmi_version', ipmi_detected_ip='$ipmi_detected_ip', use_oci_sort_dev='$use_oci_sort_dev', cpu_vendor='$cpu_vendor', cpu_model_name='$cpu_model_name', cpu_mhz='$cpu_mhz', cpu_threads='$cpu_threads', cpu_core_per_socket='$cpu_core_per_socket', cpu_sockets='$cpu_sockets', lastseen=NOW() $dhcp_update WHERE serial='$safe_serial_number'"; $r = mysqli_query($con, $req); if (isset($data["blockdevices"])){ $blkdev_array = array(); -- GitLab From 25fe5917dc3d9a083225ffa4be936420d54b6446 Mon Sep 17 00:00:00 2001 From: Thomas Goirand Date: Tue, 28 Sep 2021 09:13:15 +0200 Subject: [PATCH 18/18] swift-calculate-ring: works with older client and newer OCI. --- src/api.php | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/src/api.php b/src/api.php index 12b1740e..557fc54d 100644 --- a/src/api.php +++ b/src/api.php @@ -3602,12 +3602,10 @@ function api_actions($con,$conf){ $cluster = mysqli_fetch_array($r); - if(isset($_REQUEST["ec_only"])){ - if($_REQUEST["ec_only"] == "yes"){ - $safe_ec_only = "yes"; - }else{ - $safe_ec_only = "no"; - } + if(isset($_REQUEST["ec_only"]) && $_REQUEST["ec_only"] == "yes"){ + $safe_ec_only = "yes"; + }else{ + $safe_ec_only = "no"; } if(isset($_REQUEST["initial_account_weight"])){ -- GitLab