mgmt/monitoring/Icinga@MariaDB-Apache-RHEL7.md

ICINGA installation

update 2019 11 12

#

yum install icinga2 icinga2-selinux systemctl enable icinga2 && systemctl start icinga2 icinga2 feature list

yum install nagios-plugins-all

#

installing DB for icinga

# yum install mariadb-server mariadb systemctl enable mariadb && systemctl start mariadb mysql_secure_installation yum install icinga2-ido-mysql mysql -u root -p

#(created by GRANT command, single run) CREATE USER icinga@localhost IDENTIFIED BY 'newpass';
CREATE DATABASE icinga;
GRANT SELECT, INSERT, UPDATE, DELETE, DROP, CREATE VIEW, INDEX, EXECUTE ON icinga.* TO 'icinga'@'localhost' IDENTIFIED BY '(newpass)';
# GRANT SELECT, INSERT, UPDATE, DELETE, DROP, CREATE VIEW, INDEX, EXECUTE ON icinga.* TO 'icinga'@'localhost';
quit

mysql -u root -p icinga < /usr/share/icinga2-ido-mysql/schema/mysql.sql

vi /etc/icinga2/features-available/ido-mysql.conf

uncomment and update credentials

ln -s /etc/icinga2/features-available/ido-mysql.conf /etc/icinga2/features-enabled/ido-mysql.conf systemctl restart icinga2

yum install httpd systemctl enable httpd && systemctl start httpd

open firewall 80,443

################## setup

clean if re-setup

rm /var/lib/icinga2/ca/ca.*
rm /var/lib/icinga2/certs/*.key
rm /etc/icinga2/conf.d/api-users.conf
rm /etc/icinga2/constants.conf
rm -R /var/lib/icinga2/api/packages/director
# re-enable setup module
icingacli module enable setup
https://(host)/icingaweb2/setup

changes passes for root

icinga2 api setup

#

run NODE WIZARD

# icinga2 node wizard Please specify if this is an agent/satellite setup ('n' installs a master setup) [Y/n]: n Please specify the common name (CN) [(host)]: (host) Master zone name [master]: zon1 Do you want to specify additional global zones? [y/N]: n Please specify the API bind host/port (optional): Bind Host []: Bind Port []: Do you want to disable the inclusion of the conf.d directory [Y/n]: y icinga2 api setup systemctl restart icinga2

debug

icinga2 daemon -C

add salt (should generated by setup)

generate ticketsalt and modify file

vi /etc/icinga2/constants.conf

vi /etc/icinga2/conf.d/api-users.conf

add another user

object ApiUser "icingaweb2" { password = "newpass" // permissions = [ "status/query", "actions/", "objects/modify/", "objects/query/" ] permissions = [ "" ] } systemctl restart icinga2

yum install rh-php71 rh-php71-php-mysqlnd yum install icingaweb2 icingaweb2-selinux icingacli systemctl enable rh-php71-php-fpm.service && systemctl start rh-php71-php-fpm.service

check for FilesMatch

vi /etc/httpd/conf.d/icingaweb2.conf

systemctl restart rh-php71-php-fpm.service systemctl status rh-php71-php-fpm.service

icingacli setup token create The newly generated setup token is: 512233xxxxa90f12

to recall token:

icingacli setup token show

#

create DB for icingaweb2

# mysql -u root -p

CREATE DATABASE icingaweb2;
GRANT ALL ON icingaweb2.* TO icingaweb2@localhost IDENTIFIED BY 'newpass';

#

to recreate manually schema of icingaweb2

# mysql -u root -p icingaweb2 < /usr/share/doc/icingaweb2/schema/mysql.schema.sql

#

change pass for icingaweb2, in case it is forgotten

# mysql -u root -p

USE icingaweb2;

install ImageMagick

yum install ImageMagick ImageMagick-devel

################ server firewall

open firewall

##

Icinga welcomes agents.

##

-A INPUT -m state --state NEW -m tcp -p tcp -s xxx.xxx.xxx.0/xx --dport 5665 -j ACCEPT #? -A INPUT -m state -m tcp --state NEW -p tcp --dport 5665 -j ACCEPT

github?

for github

(better create ipset)

-A OUTPUT -m multiport -m tcp -p tcp -d xxx.xxx.xx.xx --dports 80,443 -j ACCEPT

#################### installing director (adding hosts/services) yum install git

dependencies

https://github.com/Icinga/icingaweb2-module-reactbundle/archive/v0.7.0.tar.gz https://github.com/Icinga/icingaweb2-module-ipl/archive/v0.3.0.tar.gz https://github.com/Icinga/icingaweb2-module-incubator/archive/v0.5.0.tar.gz

need manually download, cause firewall is blocking github, otherwise use script

check for latest version

https://github.com/Icinga/icingaweb2-module-director/releases https://github.com/icinga/icingaweb2-module-director/archive/v1.7.0.tar.gz

modify script, according to latest version number

ICINGAWEB_MODULEPATH="/usr/share/icingaweb2/modules"

install -d -m 0755 "${ICINGAWEB_MODULEPATH}/reactbundle" tar xfz icingaweb2-module-reactbundle-0.6.0.tar.gz -C ${ICINGAWEB_MODULEPATH}/reactbundle --strip-components 1

install -d -m 0755 "${ICINGAWEB_MODULEPATH}/ipl" tar xfz icingaweb2-module-ipl-0.3.0.tar.gz -C ${ICINGAWEB_MODULEPATH}/ipl --strip-components 1

install -d -m 0755 "${ICINGAWEB_MODULEPATH}/incubator" tar xfz icingaweb2-module-incubator-0.3.0.tar.gz -C ${ICINGAWEB_MODULEPATH}/incubator --strip-components 1

install -d -m 0755 "${ICINGAWEB_MODULEPATH}/director" tar xfz icingaweb2-module-director-1.7.0.tar.gz -C ${ICINGAWEB_MODULEPATH}/director --strip-components 1

create database for director

mysql -u root -p CREATE DATABASE director CHARACTER SET 'utf8'; GRANT ALL ON director.* TO director@localhost IDENTIFIED BY 'newpass';

add resource (specify character set is lowercase 'utf8'):

Configuration / Application / Resources

configure icinga director

Configuration / Modules / director / Configuration / DB resource = icingaweb_director_db , create database schema endpoint: (host) Icinga Host: (host) Port: 5665 API user: icingaweb2 password: (password)

configuring daemon

useradd -r -g icingaweb2 -d /var/lib/icingadirector -s /bin/false icingadirector install -d -o icingadirector -g icingaweb2 -m 0750 /var/lib/icingadirector MODULE_PATH=/usr/share/icingaweb2/modules/director cp "${MODULE_PATH}/contrib/systemd/icinga-director.service" /etc/systemd/system/ systemctl daemon-reload systemctl enable icinga-director.service Created symlink from /etc/systemd/system/multi-user.target.wants/icinga-director.service to /etc/systemd/system/icinga-director.service. systemctl start icinga-director.service

disable default checks

mv /etc/icinga2/conf.d/services.conf /etc/icinga2/conf.d/services.conf.20191021

uncomment and enable, set 'true'

vi /etc/icinga2/features-enabled/api.conf

enable features

icinga2 feature enable command perfdata

consider: install and enable InfluxDB and disable perfdata (writing to files), instructions below:

#

module: reporting

# download https://github.com/Icinga/icingaweb2-module-reporting/archive/master.zip upload cd /usr/share/icingaweb2/modules ICINGAWEB_MODULEPATH="/usr/share/icingaweb2/modules" install -d -m 0755 "${ICINGAWEB_MODULEPATH}/reporting" unzip mysql -u root -p

CREATE DATABASE reporting;
GRANT SELECT, INSERT, UPDATE, DELETE, DROP, CREATE VIEW, INDEX, EXECUTE ON reporting.* TO reporting@localhost IDENTIFIED BY '(newpass)';

create template table first, otherwise 1005 error, cause key does not exist

mysql -p -u root reporting < schema/mysql.sql Configuration -> Application -> Resources > create new resouce. icingaweb_reporting_db, db:reporting, l:reporting, p:(newpass), utf8mb4, [validate configuration], [save changes] Configuration -> Modules -> reporting -> Backend, icingaweb_reporting_db, [save changes] -> Mail > From: icinga@(host), [save changes] cp /usr/share/icingaweb2/modules/reporting/config/systemd/icinga-reporting.service /etc/systemd/system/icinga-reporting.service systemctl enable icinga-reporting && systemctl start icinga-reporting (pdfexport requires https://github.com/Icinga/icingaweb2-module-pdfexport/blob/master/doc/02-Installation.md)

test

reporting > reports > availability > send > (type_email), [send]

#

module: InfluxDB Writer

enabling perfdata writing into database (for later access with other tools)

#

By default the InfluxdbWriter feature expects the InfluxDB daemon to listen at 127.0.0.1 on port 8086.

icinga2 feature enable influxdb
systemctl restart icinga2

If SELinux is enabled, it will not allow access for Icinga 2 to InfluxDB until the boolean icinga2_can_connect_all is set

getsebool -a | grep icinga setsebool -P icinga2_can_connect_all true

more policies

vi icinga_allow_getattr.te

module icinga_allow_getattr 1.0;

require {
        type icinga2_t;
        type fs_t;
        class filesystem getattr;
}

#============= icinga2_t ==============
allow icinga2_t fs_t:filesystem getattr;

pack, check and install module

checkmodule -M -m -o icinga_allow_getattr.mod icinga_allow_getattr.te
semodule_package -m icinga_allow_getattr.mod -o icinga_allow_getattr.pp
semodule -i icinga_allow_getattr.pp

#

Visualization: InfluxDB + Grafana

# yum install influxdb systemctl start influxd netstat -ntap | grep 8086 influx

CREATE DATABASE icinga2;
CREATE USER icinga2 WITH PASSWORD '(newpass)';

exit

enable SSL in influxdb

vi /etc/influxdb/influxdb.conf

https-enabled = true
https-certificate = "<bundled-certificate-file>.pem"
https-private-key = "<bundled-certificate-file>.pem"

remember to give permissions to files, my cert is owned by grafana group, to keep everybody happy

usermod -aG grafana influxdb cat /etc/group | grep grafana systemctl restart influxdb influx -ssl -host (host)

vi /etc/icinga2/features-enabled/influxdb.conf ---snip snip---

The InfluxdbWriter type writes check result metrics and performance data to an InfluxDB HTTP API

object InfluxdbWriter "influxdb" {

  host = "127.0.0.1"
  port = 8086

  ssl_enable = true
#  ssl_cert = /data/www/conf/ssl/(cert)-crt.pem
#  ssl_key = /data/www/conf/ssl/(cert)-key.pem
  database = "icinga2"
  username = "icinga2"
  password = "(newpass)"

  enable_send_thresholds = true
  enable_send_metadata = true

  flush_threshold = 1024
  flush_interval = 10s

  host_template = {
      measurement = "$host.check_command$"
      tags = {
          hostname = "$host.name$"
      }
  }
  service_template = {
      measurement = "$service.check_command$"
      tags = {
          hostname = "$host.name$"
          service = "$service.name$"
      }
  }
}

#

installing Grafana front-end and connecting it to InfluxDB

#

https://grafana.com/grafana/plugins?orderBy=weight&direction=asc

#

yum install grafana
systemctl daemon-reload
systemctl enable grafana-server
systemctl start grafana-server
netstat -ntap | grep 3000
curl 0:3000
vi /etc/sysconfig/iptables

-A INPUT -m state --state NEW -m tcp -p tcp -s xxx.xxx.xx.xxx/xxx --dport 3000 -j ACCEPT

iptables-restore < /etc/sysconfig/iptables

http://(host):3000/login

admin:admin
add data source > influxdb
specify: name, URL, database, username, password [test and save]
http://(host):3000/dashboard/import
copy-paste dashboard config from https://raw.githubusercontent.com/Mikesch-mp/icingaweb2-module-grafana/v1.1.8/dashboards/influxdb/base-metrics.json , [import]
check settings, specify icinga2-influxdb to correct data source, [import]

add another dashboard
https://raw.githubusercontent.com/Mikesch-mp/icingaweb2-module-grafana/master/dashboards/influxdb/icinga2-default.json

vi /etc/grafana/grafana.ini protocol = https cert_file = /data/www/conf/ssl/(host)-crt.pem cert_key = /data/www/conf/ssl/(host)-key.pem reporting_enabled = false check_for_updates = false disable_initial_admin_creation = true disable_gravatar = false cookie_secure = true allow_embedding = true strict_transport_security = true

adjust to three days (over weekend)

strict_transport_security_max_age_seconds = 86400 allow_sign_up = false [auth.anonymous] enabled = false

# make grafana happy accessing certificates
chgrp grafana /data/www/conf/ssl/(host)-key.pem
chmod 0440 /data/www/conf/ssl/(host)-key.pem


#
# module: Grafana Module for Icinga Web 2
# show graphs inside of icinga ui
# https://github.com/Mikesch-mp/icingaweb2-module-grafana
#
# do not enable image rendering, use IFRAME

# replace version number from https://github.com/Mikesch-mp/icingaweb2-module-grafana/releases/latest
# with internet
MODULE_VERSION="1.3.6"
ICINGAWEB_MODULEPATH="/usr/share/icingaweb2/modules"
REPO_URL="https://github.com/Mikesch-mp/icingaweb2-module-grafana"
TARGET_DIR="${ICINGAWEB_MODULEPATH}/grafana"
URL="${REPO_URL}/archive/v${MODULE_VERSION}.tar.gz"
install -d -m 0755 "${TARGET_DIR}"
wget -q -O - "$URL" | tar xfz - -C "${TARGET_DIR}" --strip-components 1


# without internet
download https://github.com/Mikesch-mp/icingaweb2-module-grafana/archive/v${MODULE_VERSION}.tar.gz
upload to host

ICINGAWEB_MODULEPATH="/usr/share/icingaweb2/modules" TARGET_DIR="${ICINGAWEB_MODULEPATH}/grafana"

install -d -m 0755 "${TARGET_DIR}" tar xfz /data/home/(you)/icingaweb2-module-grafana-1.3.6.tar.gz -C "${TARGET_DIR}" --strip-components 1

chmod -R 0755 /usr/share/icingaweb2/modules/grafana mkdir /etc/icingaweb2/modules/grafana

vi /etc/icingaweb2/modules/grafana/config.ini

[grafana]

host = "(host):3000"

protocol = "http"
defaultdashboard = "base-metrics"
defaultorgid = "1"
defaultdashboardstore = "db"
theme = "light"
datasource = "influxdb"
accessmode = "iframe"
timerange = "6h"

vi /etc/icingaweb2/modules/grafana/graphs.ini [ping4] dashboard = "base-metrics" panelId = "1" orgId = ""

[ping6] dashboard = "base-metrics" panelId = "3" orgId = ""

[hostalive] dashboard = "base-metrics" panelId = "9" orgId = ""

[apt] dashboard = "base-metrics" panelId = "2" orgId = ""

[load] dashboard = "base-metrics" panelId = "4" orgId = ""

[swap] dashboard = "base-metrics" panelId = "5" orgId = ""

[procs] dashboard = "base-metrics" panelId = "7" orgId = ""

[ntp] dashboard = "base-metrics" panelId = "6" orgId = ""

[users] dashboard = "base-metrics" panelId = "8" orgId = ""

vi /etc/grafana/grafana.ini [auth.anonymous] enabled = true allow_embedding = true

# selinux

ls -laZ /usr/share/icingaweb2/modules/ semanage fcontext -a -t icingaweb2_content_t "/usr/share/icingaweb2/modules(/.*)?" cat /etc/selinux/targeted/contexts/files/file_contexts.local | grep icinga restorecon -R -v /usr/share/icingaweb2/modules/

systemctl restart grafana-server chown apache:icingaweb2 /etc/icingaweb2/modules/grafana/ chmod 770 /etc/icingaweb2/modules/grafana/ chmod 660 /etc/icingaweb2/modules/grafana/* icingacli module list | grep grafana icingacli module enable grafana icingacli module list | grep grafana systemctl restart httpd systemctl restart rh-php71-php-fpm chown -R apache:icingaweb2 /etc/icingaweb2

icingaweb2 > configuration > module > grafana > configuration > default-dashboard






#
# deploying by_ssh check style
#

# server side (made once)

passwd icinga vi /etc/passwd

# replace:
# icinga:x:991:990:icinga:/var/spool/icinga2:/sbin/nologin
icinga:x:991:990:icinga:/data/home/icinga:/bin/bash

mkdir /data/home/icinga chown icinga:icinga /data/home/icinga chmod 700 /data/home/icinga ls -la /data/home

# move perfdata to new new home directory

mv -R /var/spool/icinga2* /data/home/icinga systemctl restart icinga

# create ssh key pair, do not set passphrase

ssh-keygen -b 4096 -t rsa -C "icinga@$(hostname) (by_ssh check)" -f $HOME/.ssh/id_rsa

# if icinga home directory is elsewhere, move it

systemctl stop icinga2.service usermod -m -d /data/home/icinga icinga systemctl start icinga2.service systemctl status icinga2.service

#
# check_nwc_health
#

git clone

upload

yum install make yum install gcc yum install build-essential yum install automake autoreconf yum install dos2unix dos2unix * autoreconf yum install -y perl-Net-SNMP perl-Data-Dumper perl-Module-Load ./configure --libexec=/usr/lib64/nagios/plugins --with-nagios-user=icinga --with-nagios-group=icinga vi missing

:set ff=unix
:wq

./configure make cp plugins-scripts/check_nwc_health /usr/lib64/nagios/plugins/ chmod 755 /usr/lib64/nagios/plugins/check_nwc_health ```

#

common issues:

#

Remote command execution failed: @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@

Remote command execution failed: Host key verification failed.

Remote command execution failed: bash: /data/home/nagios/libexec/check_disk: No such file or directory

##

enabling API (probably from old documentation)

##

#

add ApiUser and restart

# vi /etc/icinga2/conf.d/api-users.conf

#

test

# curl -k -s -u collector:newpass 'https://127.0.0.1:5665/v1'