Icinga@MariaDB-Apache-RHEL7.md 17 KB
ICINGA installation
update 2019 11 12
#
yum install icinga2 icinga2-selinux systemctl enable icinga2 && systemctl start icinga2 icinga2 feature list
yum install nagios-plugins-all
#
installing DB for icinga
# yum install mariadb-server mariadb systemctl enable mariadb && systemctl start mariadb mysql_secure_installation yum install icinga2-ido-mysql mysql -u root -p
#(created by GRANT command, single run) CREATE USER icinga@localhost IDENTIFIED BY 'newpass';
CREATE DATABASE icinga;
GRANT SELECT, INSERT, UPDATE, DELETE, DROP, CREATE VIEW, INDEX, EXECUTE ON icinga.* TO 'icinga'@'localhost' IDENTIFIED BY '(newpass)';
# GRANT SELECT, INSERT, UPDATE, DELETE, DROP, CREATE VIEW, INDEX, EXECUTE ON icinga.* TO 'icinga'@'localhost';
quit
mysql -u root -p icinga < /usr/share/icinga2-ido-mysql/schema/mysql.sql
vi /etc/icinga2/features-available/ido-mysql.conf
uncomment and update credentials
ln -s /etc/icinga2/features-available/ido-mysql.conf /etc/icinga2/features-enabled/ido-mysql.conf systemctl restart icinga2
yum install httpd systemctl enable httpd && systemctl start httpd
open firewall 80,443
################## setup
clean if re-setup
rm /var/lib/icinga2/ca/ca.*
rm /var/lib/icinga2/certs/*.key
rm /etc/icinga2/conf.d/api-users.conf
rm /etc/icinga2/constants.conf
rm -R /var/lib/icinga2/api/packages/director
# re-enable setup module
icingacli module enable setup
https://(host)/icingaweb2/setup
changes passes for root
icinga2 api setup
#
run NODE WIZARD
# icinga2 node wizard Please specify if this is an agent/satellite setup ('n' installs a master setup) [Y/n]: n Please specify the common name (CN) [(host)]: (host) Master zone name [master]: zon1 Do you want to specify additional global zones? [y/N]: n Please specify the API bind host/port (optional): Bind Host []: Bind Port []: Do you want to disable the inclusion of the conf.d directory [Y/n]: y icinga2 api setup systemctl restart icinga2
debug
icinga2 daemon -C
add salt (should generated by setup)
generate ticketsalt and modify file
vi /etc/icinga2/constants.conf
vi /etc/icinga2/conf.d/api-users.conf
add another user
object ApiUser "icingaweb2" { password = "newpass" // permissions = [ "status/query", "actions/", "objects/modify/", "objects/query/" ] permissions = [ "" ] } systemctl restart icinga2
yum install rh-php71 rh-php71-php-mysqlnd yum install icingaweb2 icingaweb2-selinux icingacli systemctl enable rh-php71-php-fpm.service && systemctl start rh-php71-php-fpm.service
check for FilesMatch
vi /etc/httpd/conf.d/icingaweb2.conf
systemctl restart rh-php71-php-fpm.service systemctl status rh-php71-php-fpm.service
icingacli setup token create The newly generated setup token is: 512233xxxxa90f12
to recall token:
icingacli setup token show
#
create DB for icingaweb2
# mysql -u root -p
CREATE DATABASE icingaweb2;
GRANT ALL ON icingaweb2.* TO icingaweb2@localhost IDENTIFIED BY 'newpass';
#
to recreate manually schema of icingaweb2
# mysql -u root -p icingaweb2 < /usr/share/doc/icingaweb2/schema/mysql.schema.sql
#
change pass for icingaweb2, in case it is forgotten
# mysql -u root -p
USE icingaweb2;
install ImageMagick
yum install ImageMagick ImageMagick-devel
################ server firewall
open firewall
##
Icinga welcomes agents.
##
-A INPUT -m state --state NEW -m tcp -p tcp -s xxx.xxx.xxx.0/xx --dport 5665 -j ACCEPT #? -A INPUT -m state -m tcp --state NEW -p tcp --dport 5665 -j ACCEPT
github?
for github
(better create ipset)
-A OUTPUT -m multiport -m tcp -p tcp -d xxx.xxx.xx.xx --dports 80,443 -j ACCEPT
#################### installing director (adding hosts/services) yum install git
dependencies
https://github.com/Icinga/icingaweb2-module-reactbundle/archive/v0.7.0.tar.gz https://github.com/Icinga/icingaweb2-module-ipl/archive/v0.3.0.tar.gz https://github.com/Icinga/icingaweb2-module-incubator/archive/v0.5.0.tar.gz
need manually download, cause firewall is blocking github, otherwise use script
check for latest version
https://github.com/Icinga/icingaweb2-module-director/releases https://github.com/icinga/icingaweb2-module-director/archive/v1.7.0.tar.gz
modify script, according to latest version number
ICINGAWEB_MODULEPATH="/usr/share/icingaweb2/modules"
install -d -m 0755 "${ICINGAWEB_MODULEPATH}/reactbundle" tar xfz icingaweb2-module-reactbundle-0.6.0.tar.gz -C ${ICINGAWEB_MODULEPATH}/reactbundle --strip-components 1
install -d -m 0755 "${ICINGAWEB_MODULEPATH}/ipl" tar xfz icingaweb2-module-ipl-0.3.0.tar.gz -C ${ICINGAWEB_MODULEPATH}/ipl --strip-components 1
install -d -m 0755 "${ICINGAWEB_MODULEPATH}/incubator" tar xfz icingaweb2-module-incubator-0.3.0.tar.gz -C ${ICINGAWEB_MODULEPATH}/incubator --strip-components 1
install -d -m 0755 "${ICINGAWEB_MODULEPATH}/director" tar xfz icingaweb2-module-director-1.7.0.tar.gz -C ${ICINGAWEB_MODULEPATH}/director --strip-components 1
create database for director
mysql -u root -p CREATE DATABASE director CHARACTER SET 'utf8'; GRANT ALL ON director.* TO director@localhost IDENTIFIED BY 'newpass';
add resource (specify character set is lowercase 'utf8'):
Configuration / Application / Resources
configure icinga director
Configuration / Modules / director / Configuration / DB resource = icingaweb_director_db , create database schema endpoint: (host) Icinga Host: (host) Port: 5665 API user: icingaweb2 password: (password)
configuring daemon
useradd -r -g icingaweb2 -d /var/lib/icingadirector -s /bin/false icingadirector install -d -o icingadirector -g icingaweb2 -m 0750 /var/lib/icingadirector MODULE_PATH=/usr/share/icingaweb2/modules/director cp "${MODULE_PATH}/contrib/systemd/icinga-director.service" /etc/systemd/system/ systemctl daemon-reload systemctl enable icinga-director.service Created symlink from /etc/systemd/system/multi-user.target.wants/icinga-director.service to /etc/systemd/system/icinga-director.service. systemctl start icinga-director.service
disable default checks
mv /etc/icinga2/conf.d/services.conf /etc/icinga2/conf.d/services.conf.20191021
uncomment and enable, set 'true'
vi /etc/icinga2/features-enabled/api.conf
enable features
icinga2 feature enable command perfdata
consider: install and enable InfluxDB and disable perfdata (writing to files), instructions below:
#
module: reporting
# download https://github.com/Icinga/icingaweb2-module-reporting/archive/master.zip upload cd /usr/share/icingaweb2/modules ICINGAWEB_MODULEPATH="/usr/share/icingaweb2/modules" install -d -m 0755 "${ICINGAWEB_MODULEPATH}/reporting" unzip mysql -u root -p
CREATE DATABASE reporting;
GRANT SELECT, INSERT, UPDATE, DELETE, DROP, CREATE VIEW, INDEX, EXECUTE ON reporting.* TO reporting@localhost IDENTIFIED BY '(newpass)';
create template table first, otherwise 1005 error, cause key does not exist
mysql -p -u root reporting < schema/mysql.sql Configuration -> Application -> Resources > create new resouce. icingaweb_reporting_db, db:reporting, l:reporting, p:(newpass), utf8mb4, [validate configuration], [save changes] Configuration -> Modules -> reporting -> Backend, icingaweb_reporting_db, [save changes] -> Mail > From: icinga@(host), [save changes] cp /usr/share/icingaweb2/modules/reporting/config/systemd/icinga-reporting.service /etc/systemd/system/icinga-reporting.service systemctl enable icinga-reporting && systemctl start icinga-reporting (pdfexport requires https://github.com/Icinga/icingaweb2-module-pdfexport/blob/master/doc/02-Installation.md)
test
reporting > reports > availability > send > (type_email), [send]
#
module: InfluxDB Writer
enabling perfdata writing into database (for later access with other tools)
#
By default the InfluxdbWriter feature expects the InfluxDB daemon to listen at 127.0.0.1 on port 8086.
icinga2 feature enable influxdb
systemctl restart icinga2
If SELinux is enabled, it will not allow access for Icinga 2 to InfluxDB until the boolean icinga2_can_connect_all is set
getsebool -a | grep icinga setsebool -P icinga2_can_connect_all true
more policies
vi icinga_allow_getattr.te
module icinga_allow_getattr 1.0;
require {
type icinga2_t;
type fs_t;
class filesystem getattr;
}
#============= icinga2_t ==============
allow icinga2_t fs_t:filesystem getattr;
pack, check and install module
checkmodule -M -m -o icinga_allow_getattr.mod icinga_allow_getattr.te
semodule_package -m icinga_allow_getattr.mod -o icinga_allow_getattr.pp
semodule -i icinga_allow_getattr.pp
#
Visualization: InfluxDB + Grafana
# yum install influxdb systemctl start influxd netstat -ntap | grep 8086 influx
CREATE DATABASE icinga2;
CREATE USER icinga2 WITH PASSWORD '(newpass)';
exit
enable SSL in influxdb
vi /etc/influxdb/influxdb.conf
https-enabled = true
https-certificate = "<bundled-certificate-file>.pem"
https-private-key = "<bundled-certificate-file>.pem"
remember to give permissions to files, my cert is owned by grafana group, to keep everybody happy
usermod -aG grafana influxdb cat /etc/group | grep grafana systemctl restart influxdb influx -ssl -host (host)
vi /etc/icinga2/features-enabled/influxdb.conf ---snip snip---
The InfluxdbWriter type writes check result metrics and performance data to an InfluxDB HTTP API
object InfluxdbWriter "influxdb" {
host = "127.0.0.1"
port = 8086
ssl_enable = true
# ssl_cert = /data/www/conf/ssl/(cert)-crt.pem
# ssl_key = /data/www/conf/ssl/(cert)-key.pem
database = "icinga2"
username = "icinga2"
password = "(newpass)"
enable_send_thresholds = true
enable_send_metadata = true
flush_threshold = 1024
flush_interval = 10s
host_template = {
measurement = "$host.check_command$"
tags = {
hostname = "$host.name$"
}
}
service_template = {
measurement = "$service.check_command$"
tags = {
hostname = "$host.name$"
service = "$service.name$"
}
}
}
#
installing Grafana front-end and connecting it to InfluxDB
#
https://grafana.com/grafana/plugins?orderBy=weight&direction=asc
#
yum install grafana
systemctl daemon-reload
systemctl enable grafana-server
systemctl start grafana-server
netstat -ntap | grep 3000
curl 0:3000
vi /etc/sysconfig/iptables
-A INPUT -m state --state NEW -m tcp -p tcp -s xxx.xxx.xx.xxx/xxx --dport 3000 -j ACCEPT
iptables-restore < /etc/sysconfig/iptables
http://(host):3000/login
admin:admin
add data source > influxdb
specify: name, URL, database, username, password [test and save]
http://(host):3000/dashboard/import
copy-paste dashboard config from https://raw.githubusercontent.com/Mikesch-mp/icingaweb2-module-grafana/v1.1.8/dashboards/influxdb/base-metrics.json , [import]
check settings, specify icinga2-influxdb to correct data source, [import]
add another dashboard
https://raw.githubusercontent.com/Mikesch-mp/icingaweb2-module-grafana/master/dashboards/influxdb/icinga2-default.json
vi /etc/grafana/grafana.ini protocol = https cert_file = /data/www/conf/ssl/(host)-crt.pem cert_key = /data/www/conf/ssl/(host)-key.pem reporting_enabled = false check_for_updates = false disable_initial_admin_creation = true disable_gravatar = false cookie_secure = true allow_embedding = true strict_transport_security = true
adjust to three days (over weekend)
strict_transport_security_max_age_seconds = 86400 allow_sign_up = false [auth.anonymous] enabled = false
# make grafana happy accessing certificates
chgrp grafana /data/www/conf/ssl/(host)-key.pem
chmod 0440 /data/www/conf/ssl/(host)-key.pem
#
# module: Grafana Module for Icinga Web 2
# show graphs inside of icinga ui
# https://github.com/Mikesch-mp/icingaweb2-module-grafana
#
# do not enable image rendering, use IFRAME
# replace version number from https://github.com/Mikesch-mp/icingaweb2-module-grafana/releases/latest
# with internet
MODULE_VERSION="1.3.6"
ICINGAWEB_MODULEPATH="/usr/share/icingaweb2/modules"
REPO_URL="https://github.com/Mikesch-mp/icingaweb2-module-grafana"
TARGET_DIR="${ICINGAWEB_MODULEPATH}/grafana"
URL="${REPO_URL}/archive/v${MODULE_VERSION}.tar.gz"
install -d -m 0755 "${TARGET_DIR}"
wget -q -O - "$URL" | tar xfz - -C "${TARGET_DIR}" --strip-components 1
# without internet
download https://github.com/Mikesch-mp/icingaweb2-module-grafana/archive/v${MODULE_VERSION}.tar.gz
upload to host
ICINGAWEB_MODULEPATH="/usr/share/icingaweb2/modules" TARGET_DIR="${ICINGAWEB_MODULEPATH}/grafana"
install -d -m 0755 "${TARGET_DIR}" tar xfz /data/home/(you)/icingaweb2-module-grafana-1.3.6.tar.gz -C "${TARGET_DIR}" --strip-components 1
chmod -R 0755 /usr/share/icingaweb2/modules/grafana mkdir /etc/icingaweb2/modules/grafana
vi /etc/icingaweb2/modules/grafana/config.ini
[grafana]
host = "(host):3000"
protocol = "http"
defaultdashboard = "base-metrics"
defaultorgid = "1"
defaultdashboardstore = "db"
theme = "light"
datasource = "influxdb"
accessmode = "iframe"
timerange = "6h"
vi /etc/icingaweb2/modules/grafana/graphs.ini [ping4] dashboard = "base-metrics" panelId = "1" orgId = ""
[ping6] dashboard = "base-metrics" panelId = "3" orgId = ""
[hostalive] dashboard = "base-metrics" panelId = "9" orgId = ""
[apt] dashboard = "base-metrics" panelId = "2" orgId = ""
[load] dashboard = "base-metrics" panelId = "4" orgId = ""
[swap] dashboard = "base-metrics" panelId = "5" orgId = ""
[procs] dashboard = "base-metrics" panelId = "7" orgId = ""
[ntp] dashboard = "base-metrics" panelId = "6" orgId = ""
[users] dashboard = "base-metrics" panelId = "8" orgId = ""
vi /etc/grafana/grafana.ini [auth.anonymous] enabled = true allow_embedding = true
# selinux
ls -laZ /usr/share/icingaweb2/modules/ semanage fcontext -a -t icingaweb2_content_t "/usr/share/icingaweb2/modules(/.*)?" cat /etc/selinux/targeted/contexts/files/file_contexts.local | grep icinga restorecon -R -v /usr/share/icingaweb2/modules/
systemctl restart grafana-server chown apache:icingaweb2 /etc/icingaweb2/modules/grafana/ chmod 770 /etc/icingaweb2/modules/grafana/ chmod 660 /etc/icingaweb2/modules/grafana/* icingacli module list | grep grafana icingacli module enable grafana icingacli module list | grep grafana systemctl restart httpd systemctl restart rh-php71-php-fpm chown -R apache:icingaweb2 /etc/icingaweb2
icingaweb2 > configuration > module > grafana > configuration > default-dashboard
#
# deploying by_ssh check style
#
# server side (made once)
passwd icinga vi /etc/passwd
# replace:
# icinga:x:991:990:icinga:/var/spool/icinga2:/sbin/nologin
icinga:x:991:990:icinga:/data/home/icinga:/bin/bash
mkdir /data/home/icinga chown icinga:icinga /data/home/icinga chmod 700 /data/home/icinga ls -la /data/home
# move perfdata to new new home directory
mv -R /var/spool/icinga2* /data/home/icinga systemctl restart icinga
# create ssh key pair, do not set passphrase
ssh-keygen -b 4096 -t rsa -C "icinga@$(hostname) (by_ssh check)" -f $HOME/.ssh/id_rsa
# if icinga home directory is elsewhere, move it
systemctl stop icinga2.service usermod -m -d /data/home/icinga icinga systemctl start icinga2.service systemctl status icinga2.service
#
# check_nwc_health
#
git clone
upload
yum install make yum install gcc yum install build-essential yum install automake autoreconf yum install dos2unix dos2unix * autoreconf yum install -y perl-Net-SNMP perl-Data-Dumper perl-Module-Load ./configure --libexec=/usr/lib64/nagios/plugins --with-nagios-user=icinga --with-nagios-group=icinga vi missing
:set ff=unix
:wq
./configure make cp plugins-scripts/check_nwc_health /usr/lib64/nagios/plugins/ chmod 755 /usr/lib64/nagios/plugins/check_nwc_health ```
#
common issues:
#
Remote command execution failed: @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
Remote command execution failed: Host key verification failed.
Remote command execution failed: bash: /data/home/nagios/libexec/check_disk: No such file or directory
##
enabling API (probably from old documentation)
##
#
add ApiUser and restart
# vi /etc/icinga2/conf.d/api-users.conf
#
test
# curl -k -s -u collector:newpass 'https://127.0.0.1:5665/v1'