From f614102b8f69b1ff26943addd6819d0c81c200ef Mon Sep 17 00:00:00 2001 From: Thomas Goirand Date: Fri, 15 Oct 2021 14:48:45 +0200 Subject: [PATCH] Use separate table and API for filebeat output host list. --- ocicli/completions/ocicli | 38 +++++++- ocicli/ocicli | 17 ++++ ocicli/ocicli.1 | 22 +++++ puppet/manifests/filebeat.pp | 4 +- src/api.php | 175 +++++++++++++++++++++++++++++++++++ src/inc/enc.php | 12 ++- src/oci_db.php | 11 ++- src/variables.json | 5 - 8 files changed, 272 insertions(+), 12 deletions(-) diff --git a/ocicli/completions/ocicli b/ocicli/completions/ocicli index 16d965aa..950a6118 100644 --- a/ocicli/completions/ocicli +++ b/ocicli/completions/ocicli @@ -13,7 +13,7 @@ _ocicli() { prev="${COMP_WORDS[COMP_CWORD-1]}" cmd_name="${COMP_WORDS[1]}" - opts="machine-list machine-console machine-wipe machine-show profile-list machine-guessed-profile machine-check-megacli-applied machine-megacli-apply machine-megacli-reset-raid machine-guess-racking machine-auto-add machine-auto-rack machine-to-dns machine-out-of-dns machine-out-of-monitoring machine-to-monitoring machine-gen-root-pass machine-forget-root-pass machine-report-counter-reset machine-set machine-add machine-remove machine-destroy machine-reboot-on-hdd machine-reboot-on-live machine-ipmi-reboot-on-hdd machine-ipmi-reboot-on-live machine-hpe-ilo-license-install machine-hpe-activate-ipmi-over-lan machine-ipmi-cmd machine-install-os machine-display-install-cmd machine-install-log machine-set-ipmi machine-apply-ipmi machine-ip-add machine-ip-list machine-ip-remove machine-renew-ipmi-password ipmi-assign-check check-all-ipmi location-list location-create location-delete network-create network-list network-delete network-add network-remove network-set cluster-list cluster-create cluster-delete cluster-set cluster-show cluster-show-networks cluster-show-machines cluster-show-ips cluster-install cluster-reset cluster-rolecounts-list cluster-rolecounts-set swift-region-list swift-region-create swift-region-delete swift-calculate-ring swift-publish-ring role-list role-create role-delete" + opts="machine-list machine-console machine-wipe machine-show profile-list machine-guessed-profile machine-check-megacli-applied machine-megacli-apply machine-megacli-reset-raid machine-guess-racking machine-auto-add machine-auto-rack machine-to-dns machine-out-of-dns machine-out-of-monitoring machine-to-monitoring machine-gen-root-pass machine-forget-root-pass machine-report-counter-reset machine-set machine-add machine-remove machine-destroy machine-reboot-on-hdd machine-reboot-on-live machine-ipmi-reboot-on-hdd machine-ipmi-reboot-on-live machine-hpe-ilo-license-install machine-hpe-activate-ipmi-over-lan machine-ipmi-cmd machine-install-os machine-display-install-cmd machine-install-log machine-set-ipmi machine-apply-ipmi machine-ip-add machine-ip-list machine-ip-remove machine-renew-ipmi-password ipmi-assign-check check-all-ipmi location-list location-create location-delete network-create network-list network-delete network-add network-remove network-set cluster-list cluster-create cluster-delete cluster-set cluster-show cluster-show-networks cluster-show-machines cluster-show-ips cluster-install cluster-reset cluster-rolecounts-list cluster-rolecounts-set swift-region-list swift-region-create swift-region-delete swift-calculate-ring swift-publish-ring role-list role-create role-delete filebeat-output-host-list filebeat-output-host-create filebeat-output-host-delete" iface_names="none eth0 eth1 eth2 eth3 eth4 eth5 10m1 10m2 10m3 10m4 100m1 100m2 100m3 100m4 1g1 1g2 1g3 1g4 10g1 10g2 10g3 10g4" case "${cmd_name}" in @@ -300,7 +300,41 @@ _ocicli() { ;; esac ;; - cluster-show-machines|cluster-show-networks|cluster-show-ips|cluster-delete|cluster-show|cluster-install|cluster-reset|cluster-rolecounts-list|swift-publish-ring) + filebeat-output-host-create) + case ${COMP_CWORD} in + 2) + local cluster_list=$(ocicli -csv cluster-list | grep -v "id,name,domain" | cut -d, -f2 | tr \\n " ") + COMPREPLY=( $(compgen -W "${cluster_list}" -- ${cur}) ) + return 0 + ;; + *) + return 0 + ;; + esac + ;; + filebeat-output-host-delete) + case ${COMP_CWORD} in + 2) + local cluster_list=$(ocicli -csv cluster-list | grep -v "id,name,domain" | cut -d, -f2 | tr \\n " ") + COMPREPLY=( $(compgen -W "${cluster_list}" -- ${cur}) ) + return 0 + ;; + 3) + local cluster_list=$(ocicli -csv filebeat-output-host-list ${COMP_WORDS[COMP_CWORD-1]} | q -H -d, "SELECT hostname FROM -" | tr '\n' ' ') + COMPREPLY=( $(compgen -W "${cluster_list}" -- ${cur}) ) + return 0 + ;; + 4) + local cluster_list=$(ocicli -csv filebeat-output-host-list ${COMP_WORDS[COMP_CWORD-2]} | q -H -d, "SELECT port FROM -" | tr '\n' ' ') + COMPREPLY=( $(compgen -W "${cluster_list}" -- ${cur}) ) + return 0 + ;; + *) + return 0 + ;; + esac + ;; + cluster-show-machines|cluster-show-networks|cluster-show-ips|cluster-delete|cluster-show|cluster-install|cluster-reset|cluster-rolecounts-list|swift-publish-ring|filebeat-output-host-list) local cluster_list=$(ocicli -csv cluster-list | grep -v "id,name,domain" | cut -d, -f2 | tr \\n " ") COMPREPLY=( $(compgen -W "${cluster_list}" -- ${cur}) ) return 0 diff --git a/ocicli/ocicli b/ocicli/ocicli index feb931fb..1a7e512e 100755 --- a/ocicli/ocicli +++ b/ocicli/ocicli @@ -91,6 +91,10 @@ usage (){ echo " role-list" echo " role-create " echo " role-delete " + echo "" + echo " filebeat-output-host-list " + echo " filebeat-output-host-create " + echo " filebeat-output-host-delete " exit 1 } @@ -2399,6 +2403,19 @@ case "${ACTION}" in ocicli_call "action=cluster_rolecounts_set&name=${1}&role=${2}&count=${3}" ocicli_display_status ;; + "filebeat-output-host-list") + ocicli_call "action=filebeat_output_host_list&cluster_name=${1}" + echo "cluster,hostname,port" + cat ${API_RESPONSE_FILE} | jq -r '.["data"]|=sort_by(.cluster_name,.hostname,.port)|.["data"][] | "\(.cluster_name),\(.hostname),\(.port)"' + ;; + "filebeat-output-host-create") + ocicli_call "action=filebeat_output_host_create&cluster_name=${1}&filebeat_host=${2}&filebeat_port=${3}" + ocicli_display_status + ;; + "filebeat-output-host-delete") + ocicli_call "action=filebeat_output_host_delete&cluster_name=${1}&filebeat_host=${2}&filebeat_port=${3}" + ocicli_display_status + ;; "swift-region-list") ocicli_swiftregion_list ;; diff --git a/ocicli/ocicli.1 b/ocicli/ocicli.1 index 35819a41..65d937c3 100644 --- a/ocicli/ocicli.1 +++ b/ocicli/ocicli.1 @@ -1089,6 +1089,28 @@ Create a new role. .IP Delete a role. +.LP +.B filebeat\-output\-host\-list +.I +.IP +List all filebeat output hosts for a given cluster. + +.LP +.B filebeat\-output\-host\-create +.I +.I +.I +.IP +Add a new filebeat output host to a cluster. + +.LP +.B filebeat\-output\-host\-delete +.I +.I +.I +.IP +Delete a new filebeat output host from a cluster. + .SH AUTHORS ocicli has been written by Thomas Goirand . diff --git a/puppet/manifests/filebeat.pp b/puppet/manifests/filebeat.pp index 4ba783d8..ce21fc13 100644 --- a/puppet/manifests/filebeat.pp +++ b/puppet/manifests/filebeat.pp @@ -94,8 +94,6 @@ qPDlGRlOgVTd9xUfHFkzB52c70E= } } - $hosts = split($output_hosts, '@') - class { 'filebeat': manage_repo => false, manage_apt => false, @@ -103,7 +101,7 @@ qPDlGRlOgVTd9xUfHFkzB52c70E= fields_under_root => $fields_under_root, outputs => { 'logstash' => { - 'hosts' => $hosts, + 'hosts' => $output_hosts, 'loadbalance' => true, }, }, diff --git a/src/api.php b/src/api.php index 557fc54d..fde16112 100644 --- a/src/api.php +++ b/src/api.php @@ -2730,6 +2730,181 @@ function api_actions($con,$conf){ return $json; break; + ### Filebeat hosts management + case "filebeat_output_host_list": + $safe_cluster_name = safe_fqdn("cluster_name"); + if($safe_cluster_name === FALSE){ + $json["status"] = "error"; + $json["message"] = "Error: not valid cluster name."; + return $json; + } + $q = "SELECT * FROM clusters WHERE name='$safe_cluster_name'"; + $r = mysqli_query($con, $q); + if($r === FALSE){ + $json["status"] = "error"; + $json["message"] = mysqli_error($con); + return $json; + } + $n = mysqli_num_rows($r); + if($n != 1){ + $json["status"] = "error"; + $json["message"] = "Error: cluster name doesn't exist."; + return $json; + } + $cluster = mysqli_fetch_array($r); + $safe_cluster_id = $cluster["id"]; + + $q = "SELECT * FROM filebeathosts WHERE cluster='$safe_cluster_id'"; + $r = mysqli_query($con, $q); + if($r === FALSE){ + $json["status"] = "error"; + $json["message"] = mysqli_error($con); + return $json; + } + $n = mysqli_num_rows($r); + if($n == 0){ + $json["data"] = Array(); + }else{ + for($i=0;$i<$n;$i++){ + $a = mysqli_fetch_array($r); + $cluster_id = $a["cluster"]; + $qc = "SELECT name FROM clusters WHERE id='$cluster_id'"; + $rc = mysqli_query($con, $qc); + if($rc === FALSE){ + $json["status"] = "error"; + $json["message"] = mysqli_error($con); + return $json; + } + $nc = mysqli_num_rows($rc); + if($nc != 1){ + $json["status"] = "error"; + $json["message"] = "Cannot find cluster with ID $cluster_id: please clean-up your filebeathosts table."; + return $json; + } + $ac = mysqli_fetch_array($rc); + $a["cluster_name"] = $ac["name"]; + + $json["data"][] = $a; + } + } + + return $json; + break; + case "filebeat_output_host_create": + $safe_cluster_name = safe_fqdn("cluster_name"); + if($safe_cluster_name === FALSE){ + $json["status"] = "error"; + $json["message"] = "Error: not valid cluster name."; + return $json; + } + + $safe_filebeat_host = safe_fqdn("filebeat_host"); + if($safe_filebeat_host === FALSE){ + $json["status"] = "error"; + $json["message"] = "Error: not valid filebeat host."; + return $json; + } + + $safe_filebeat_port = safe_int("filebeat_port"); + if($safe_filebeat_port === FALSE){ + $json["status"] = "error"; + $json["message"] = "Error: not valid filebeat host."; + return $json; + } + + $q = "SELECT * FROM clusters WHERE name='$safe_cluster_name'"; + $r = mysqli_query($con, $q); + if($r === FALSE){ + $json["status"] = "error"; + $json["message"] = mysqli_error($con); + return $json; + } + $n = mysqli_num_rows($r); + if($n != 1){ + $json["status"] = "error"; + $json["message"] = "Error: cluster name doesn't exist."; + return $json; + } + $cluster = mysqli_fetch_array($r); + $safe_cluster_id = $cluster["id"]; + + $q = "SELECT * FROM filebeathosts WHERE cluster='$safe_cluster_id' AND hostname='$safe_filebeat_host' AND port='$safe_filebeat_port'"; + $r = mysqli_query($con, $q); + if($r === FALSE){ + $json["status"] = "error"; + $json["message"] = mysqli_error($con); + return $json; + } + $n = mysqli_num_rows($r); + if($n >= 1){ + $json["status"] = "error"; + $json["message"] = "Error: filebeat host already exist for this cluster."; + return $json; + } + + $q = "INSERT INTO filebeathosts (cluster, hostname, port) VALUES ('$safe_cluster_id', '$safe_filebeat_host', '$safe_filebeat_port')"; + $r = mysqli_query($con, $q); + + return $json; + break; + case "filebeat_output_host_delete": + $safe_cluster_name = safe_fqdn("cluster_name"); + if($safe_cluster_name === FALSE){ + $json["status"] = "error"; + $json["message"] = "Error: not valid cluster name."; + return $json; + } + + $safe_filebeat_host = safe_fqdn("filebeat_host"); + if($safe_filebeat_host === FALSE){ + $json["status"] = "error"; + $json["message"] = "Error: not valid filebeat host."; + return $json; + } + + $safe_filebeat_port = safe_int("filebeat_port"); + if($safe_filebeat_port === FALSE){ + $json["status"] = "error"; + $json["message"] = "Error: not valid filebeat host."; + return $json; + } + + $q = "SELECT * FROM clusters WHERE name='$safe_cluster_name'"; + $r = mysqli_query($con, $q); + if($r === FALSE){ + $json["status"] = "error"; + $json["message"] = mysqli_error($con); + return $json; + } + $n = mysqli_num_rows($r); + if($n != 1){ + $json["status"] = "error"; + $json["message"] = "Error: cluster name doesn't exist."; + return $json; + } + $cluster = mysqli_fetch_array($r); + $safe_cluster_id = $cluster["id"]; + + $q = "SELECT * FROM filebeathosts WHERE cluster='$safe_cluster_id' AND hostname='$safe_filebeat_host' AND port='$safe_filebeat_port'"; + $r = mysqli_query($con, $q); + if($r === FALSE){ + $json["status"] = "error"; + $json["message"] = mysqli_error($con); + return $json; + } + $n = mysqli_num_rows($r); + if($n != 1){ + $json["status"] = "error"; + $json["message"] = "Error: filebeat host does not exist: cannot delete."; + return $json; + } + + $q = "DELETE FROM filebeathosts WHERE cluster='$safe_cluster_id' AND hostname='$safe_filebeat_host' AND port='$safe_filebeat_port'"; + $r = mysqli_query($con, $q); + + return $json; + break; + case "network_list": $q = "SELECT networks.id AS id,networks.name AS name,networks.ip AS ip,networks.cidr AS cidr,networks.is_public AS is_public,networks.cluster AS cluster,networks.role AS role,networks.iface1 AS iface1,networks.iface2 AS iface2,networks.bridgename AS bridgename,networks.vlan AS vlan,networks.mtu AS mtu,locations.name AS location, networks.ipmi_match_addr AS ipmi_match_addr, networks.ipmi_match_cidr AS ipmi_match_cidr, networks.first_ip AS first_ip, networks.last_ip AS last_ip FROM networks,locations WHERE locations.id=networks.location_id"; $r = mysqli_query($con, $q); diff --git a/src/inc/enc.php b/src/inc/enc.php index f9e5064a..89126a55 100644 --- a/src/inc/enc.php +++ b/src/inc/enc.php @@ -1076,7 +1076,17 @@ function puppet_enc($con,$conf){ } $enc_file .= " fields_under_root: ".$cluster['filebeat_fields_under_root']."\n"; - $enc_file .= " output_hosts: ".$cluster['filebeat_output_hosts']."\n"; +# $enc_file .= " output_hosts: ".$cluster['filebeat_output_hosts']."\n"; + $enc_file .= " output_hosts:\n"; + $qfb = "SELECT hostname,port FROM filebeathosts WHERE cluster='$cluster_id'"; + $rfb = $r = mysqli_query($con, $qfb); + $nfb = mysqli_num_rows($rfb); + for($i=0;$i<$nfb;$i++){ + $afb = mysqli_fetch_array($rfb); + $fb_host = $afb["hostname"]; + $fb_port = $afb["port"]; + $enc_file .= " - $fb_host:$fb_port\n"; + } $enc_file .= " logging_level: ".$cluster['filebeat_logging_level']."\n"; $enc_file .= " logging_syslog: ".$cluster['filebeat_logging_syslog']."\n"; $enc_file .= " logging_files: ".$cluster['filebeat_logging_files']."\n"; diff --git a/src/oci_db.php b/src/oci_db.php index 1ee224c0..44066a9e 100644 --- a/src/oci_db.php +++ b/src/oci_db.php @@ -315,7 +315,16 @@ $database = array( "keys" => array( "name" => "(login)" ) - ) + ), + "filebeathosts" => array( + "vars" => array( + "id" => "int(11) NOT NULL auto_increment", + "cluster" => "int(11) NULL default NULL", + "hostname" => "varchar(128) NOT NULL default ''", + "port" => "int(6) NOT NULL default '5044'", + ), + "primary" => "(id)", + ), ) ); ?> diff --git a/src/variables.json b/src/variables.json index 71232149..1364e4a6 100644 --- a/src/variables.json +++ b/src/variables.json @@ -349,11 +349,6 @@ "default": "yes", "description": "Filebeat fields under root:" }, - "filebeat_output_hosts": { - "type": "string", - "default": "host1.example.com:5044@host2.example.com:5044", - "description": "Filebeat output hosts (separated by @):" - }, "filebeat_logging_level": { "type": "string", "default": "info", -- GitLab