# ICINGA installation
# update 2019 11 12
#

yum install icinga2 icinga2-selinux
systemctl enable icinga2 && systemctl start icinga2
icinga2 feature list

yum install nagios-plugins-all


#
# installing DB for icinga
#
yum install mariadb-server mariadb
systemctl enable mariadb && systemctl start mariadb
mysql_secure_installation
yum install icinga2-ido-mysql
mysql -u root -p
```
#(created by GRANT command, single run) CREATE USER icinga@localhost IDENTIFIED BY 'newpass';
CREATE DATABASE icinga;
GRANT SELECT, INSERT, UPDATE, DELETE, DROP, CREATE VIEW, INDEX, EXECUTE ON icinga.* TO 'icinga'@'localhost' IDENTIFIED BY '(newpass)';
# GRANT SELECT, INSERT, UPDATE, DELETE, DROP, CREATE VIEW, INDEX, EXECUTE ON icinga.* TO 'icinga'@'localhost';
quit
```
mysql -u root -p icinga < /usr/share/icinga2-ido-mysql/schema/mysql.sql

vi /etc/icinga2/features-available/ido-mysql.conf
    uncomment and update credentials
ln -s /etc/icinga2/features-available/ido-mysql.conf /etc/icinga2/features-enabled/ido-mysql.conf
systemctl restart icinga2

yum install httpd
systemctl enable httpd && systemctl start httpd

# open firewall 80,443

################## setup

# clean if re-setup
```
rm /var/lib/icinga2/ca/ca.*
rm /var/lib/icinga2/certs/*.key
rm /etc/icinga2/conf.d/api-users.conf
rm /etc/icinga2/constants.conf
rm -R /var/lib/icinga2/api/packages/director
# re-enable setup module
icingacli module enable setup
https://(host)/icingaweb2/setup
```



# changes passes for root
icinga2 api setup

#
# run NODE WIZARD
#
icinga2 node wizard
Please specify if this is an agent/satellite setup ('n' installs a master setup) [Y/n]: n
Please specify the common name (CN) [(host)]: (host)
Master zone name [master]: zon1
Do you want to specify additional global zones? [y/N]: n
Please specify the API bind host/port (optional):
Bind Host []:
Bind Port []:
Do you want to disable the inclusion of the conf.d directory [Y/n]: y
icinga2 api setup
systemctl restart icinga2


# debug
icinga2 daemon -C

# add salt (should generated by setup)
# generate ticketsalt and modify file
vi /etc/icinga2/constants.conf


vi /etc/icinga2/conf.d/api-users.conf
# add another user
object ApiUser "icingaweb2" {
  password = "newpass"
  // permissions = [ "status/query", "actions/*", "objects/modify/*", "objects/query/*" ]
  permissions = [ "*" ]
}
systemctl restart icinga2

yum install rh-php71 rh-php71-php-mysqlnd
yum install icingaweb2 icingaweb2-selinux icingacli
systemctl enable rh-php71-php-fpm.service && systemctl start rh-php71-php-fpm.service


# check for FilesMatch
vi /etc/httpd/conf.d/icingaweb2.conf

systemctl restart rh-php71-php-fpm.service
systemctl status rh-php71-php-fpm.service

icingacli setup token create
  The newly generated setup token is: 512233xxxxa90f12
# to recall token:
icingacli setup token show



#
# create DB for icingaweb2
#
mysql -u root -p
    CREATE DATABASE icingaweb2;
    GRANT ALL ON icingaweb2.* TO icingaweb2@localhost IDENTIFIED BY 'newpass';

#
# to recreate manually schema of icingaweb2
#
mysql -u root -p icingaweb2 < /usr/share/doc/icingaweb2/schema/mysql.schema.sql


#
# change pass for icingaweb2, in case it is forgotten
#
mysql -u root -p
    USE icingaweb2;


# install ImageMagick
yum install ImageMagick ImageMagick-devel




################ server firewall
# open firewall

### ## #
# Icinga welcomes agents.
### ## #
-A INPUT -m state --state NEW -m tcp -p tcp -s xxx.xxx.xxx.0/xx --dport 5665 -j ACCEPT
#? -A INPUT -m state -m tcp --state NEW -p tcp --dport 5665 -j ACCEPT


# github?
# for github
# (better create ipset)
-A OUTPUT -m multiport -m tcp -p tcp -d xxx.xxx.xx.xx --dports 80,443 -j ACCEPT




#################### installing director (adding hosts/services)
yum install git
# dependencies
https://github.com/Icinga/icingaweb2-module-reactbundle/archive/v0.7.0.tar.gz
https://github.com/Icinga/icingaweb2-module-ipl/archive/v0.3.0.tar.gz
https://github.com/Icinga/icingaweb2-module-incubator/archive/v0.5.0.tar.gz


# need manually download, cause firewall is blocking github, otherwise use script
# check for latest version
https://github.com/Icinga/icingaweb2-module-director/releases
https://github.com/icinga/icingaweb2-module-director/archive/v1.7.0.tar.gz
# modify script, according to latest version number
ICINGAWEB_MODULEPATH="/usr/share/icingaweb2/modules"

install -d -m 0755 "${ICINGAWEB_MODULEPATH}/reactbundle"
tar xfz icingaweb2-module-reactbundle-0.6.0.tar.gz -C ${ICINGAWEB_MODULEPATH}/reactbundle --strip-components 1

install -d -m 0755 "${ICINGAWEB_MODULEPATH}/ipl"
tar xfz icingaweb2-module-ipl-0.3.0.tar.gz -C ${ICINGAWEB_MODULEPATH}/ipl --strip-components 1

install -d -m 0755 "${ICINGAWEB_MODULEPATH}/incubator"
tar xfz icingaweb2-module-incubator-0.3.0.tar.gz -C ${ICINGAWEB_MODULEPATH}/incubator --strip-components 1

install -d -m 0755 "${ICINGAWEB_MODULEPATH}/director"
tar xfz icingaweb2-module-director-1.7.0.tar.gz -C ${ICINGAWEB_MODULEPATH}/director --strip-components 1


# create database for director
mysql -u root -p
CREATE DATABASE director CHARACTER SET 'utf8';
GRANT ALL ON director.* TO director@localhost IDENTIFIED BY 'newpass';
# add resource (specify character set is lowercase 'utf8'):
Configuration / Application / Resources
# configure icinga director
Configuration / Modules / director / Configuration / DB resource = icingaweb_director_db , create database schema
endpoint: (host)
Icinga Host: (host)
Port: 5665
API user: icingaweb2
password: (password)


# configuring daemon
useradd -r -g icingaweb2 -d /var/lib/icingadirector -s /bin/false icingadirector
install -d -o icingadirector -g icingaweb2 -m 0750 /var/lib/icingadirector
MODULE_PATH=/usr/share/icingaweb2/modules/director
cp "${MODULE_PATH}/contrib/systemd/icinga-director.service" /etc/systemd/system/
systemctl daemon-reload
systemctl enable icinga-director.service
Created symlink from /etc/systemd/system/multi-user.target.wants/icinga-director.service to /etc/systemd/system/icinga-director.service.
systemctl start icinga-director.service

# disable default checks
mv /etc/icinga2/conf.d/services.conf /etc/icinga2/conf.d/services.conf.20191021

# uncomment and enable, set 'true'
vi /etc/icinga2/features-enabled/api.conf

# enable features
icinga2 feature enable command perfdata


# consider: install and enable InfluxDB and disable perfdata (writing to files), instructions below:





#
# module:  reporting
#
download https://github.com/Icinga/icingaweb2-module-reporting/archive/master.zip
upload
cd /usr/share/icingaweb2/modules
ICINGAWEB_MODULEPATH="/usr/share/icingaweb2/modules"
install -d -m 0755 "${ICINGAWEB_MODULEPATH}/reporting"
unzip
mysql -u root -p
    CREATE DATABASE reporting;
    GRANT SELECT, INSERT, UPDATE, DELETE, DROP, CREATE VIEW, INDEX, EXECUTE ON reporting.* TO reporting@localhost IDENTIFIED BY '(newpass)';
# create template table first, otherwise 1005 error, cause key does not exist
mysql -p -u root reporting < schema/mysql.sql
Configuration -> Application -> Resources > create new resouce. icingaweb_reporting_db, db:reporting, l:reporting, p:(newpass), utf8mb4, [validate configuration], [save changes]
Configuration -> Modules -> reporting -> Backend, icingaweb_reporting_db, [save changes]
  -> Mail > From: icinga@(host), [save changes]
cp /usr/share/icingaweb2/modules/reporting/config/systemd/icinga-reporting.service /etc/systemd/system/icinga-reporting.service
systemctl enable icinga-reporting && systemctl start icinga-reporting
(pdfexport requires https://github.com/Icinga/icingaweb2-module-pdfexport/blob/master/doc/02-Installation.md)

# test
reporting > reports > availability > send > (type_email), [send]



#
# module: InfluxDB Writer
# enabling perfdata writing into database (for later access with other tools)
#

By default the InfluxdbWriter feature expects the InfluxDB daemon to listen at 127.0.0.1 on port 8086.
```
icinga2 feature enable influxdb
systemctl restart icinga2
```
# If SELinux is enabled, it will not allow access for Icinga 2 to InfluxDB until the boolean icinga2_can_connect_all is set
getsebool -a | grep icinga
setsebool -P icinga2_can_connect_all true

# more policies
```
vi icinga_allow_getattr.te
```
```
module icinga_allow_getattr 1.0;

require {
        type icinga2_t;
        type fs_t;
        class filesystem getattr;
}

#============= icinga2_t ==============
allow icinga2_t fs_t:filesystem getattr;
```
pack, check and install module
```
checkmodule -M -m -o icinga_allow_getattr.mod icinga_allow_getattr.te
semodule_package -m icinga_allow_getattr.mod -o icinga_allow_getattr.pp
semodule -i icinga_allow_getattr.pp
```


#
# Visualization: InfluxDB + Grafana
#
yum install influxdb
systemctl start influxd
netstat -ntap | grep 8086
influx
    CREATE DATABASE icinga2;
    CREATE USER icinga2 WITH PASSWORD '(newpass)';
exit

# enable SSL in influxdb
vi /etc/influxdb/influxdb.conf
```
https-enabled = true
https-certificate = "<bundled-certificate-file>.pem"
https-private-key = "<bundled-certificate-file>.pem"
```
# remember to give permissions to files, my cert is owned by grafana group, to keep everybody happy
usermod -aG grafana influxdb
cat /etc/group | grep grafana
systemctl restart influxdb
influx -ssl -host (host)

vi /etc/icinga2/features-enabled/influxdb.conf
---snip snip---

The InfluxdbWriter type writes check result metrics and performance data to an InfluxDB HTTP API
```
object InfluxdbWriter "influxdb" {

  host = "127.0.0.1"
  port = 8086

  ssl_enable = true
#  ssl_cert = /data/www/conf/ssl/(cert)-crt.pem
#  ssl_key = /data/www/conf/ssl/(cert)-key.pem
  database = "icinga2"
  username = "icinga2"
  password = "(newpass)"

  enable_send_thresholds = true
  enable_send_metadata = true

  flush_threshold = 1024
  flush_interval = 10s

  host_template = {
      measurement = "$host.check_command$"
      tags = {
          hostname = "$host.name$"
      }
  }
  service_template = {
      measurement = "$service.check_command$"
      tags = {
          hostname = "$host.name$"
          service = "$service.name$"
      }
  }
}
```


#
# installing Grafana front-end and connecting it to InfluxDB
#
# https://grafana.com/grafana/plugins?orderBy=weight&direction=asc
#
```
yum install grafana
systemctl daemon-reload
systemctl enable grafana-server
systemctl start grafana-server
netstat -ntap | grep 3000
curl 0:3000
vi /etc/sysconfig/iptables
```
```
-A INPUT -m state --state NEW -m tcp -p tcp -s xxx.xxx.xx.xxx/xxx --dport 3000 -j ACCEPT
```
```
iptables-restore < /etc/sysconfig/iptables
```
http://(host):3000/login
```
admin:admin
add data source > influxdb
specify: name, URL, database, username, password [test and save]
http://(host):3000/dashboard/import
copy-paste dashboard config from https://raw.githubusercontent.com/Mikesch-mp/icingaweb2-module-grafana/v1.1.8/dashboards/influxdb/base-metrics.json , [import]
check settings, specify icinga2-influxdb to correct data source, [import]

add another dashboard
https://raw.githubusercontent.com/Mikesch-mp/icingaweb2-module-grafana/master/dashboards/influxdb/icinga2-default.json

```
vi /etc/grafana/grafana.ini
protocol = https
cert_file = /data/www/conf/ssl/(host)-crt.pem
cert_key = /data/www/conf/ssl/(host)-key.pem
reporting_enabled = false
check_for_updates = false
disable_initial_admin_creation = true
disable_gravatar = false
cookie_secure = true
allow_embedding = true
strict_transport_security = true
# adjust to three days (over weekend)
strict_transport_security_max_age_seconds = 86400
allow_sign_up = false
[auth.anonymous]
enabled = false
```

# make grafana happy accessing certificates
chgrp grafana /data/www/conf/ssl/(host)-key.pem
chmod 0440 /data/www/conf/ssl/(host)-key.pem


#
# module: Grafana Module for Icinga Web 2
# show graphs inside of icinga ui
# https://github.com/Mikesch-mp/icingaweb2-module-grafana
#
# do not enable image rendering, use IFRAME

# replace version number from https://github.com/Mikesch-mp/icingaweb2-module-grafana/releases/latest
# with internet
MODULE_VERSION="1.3.6"
ICINGAWEB_MODULEPATH="/usr/share/icingaweb2/modules"
REPO_URL="https://github.com/Mikesch-mp/icingaweb2-module-grafana"
TARGET_DIR="${ICINGAWEB_MODULEPATH}/grafana"
URL="${REPO_URL}/archive/v${MODULE_VERSION}.tar.gz"
install -d -m 0755 "${TARGET_DIR}"
wget -q -O - "$URL" | tar xfz - -C "${TARGET_DIR}" --strip-components 1


# without internet
download https://github.com/Mikesch-mp/icingaweb2-module-grafana/archive/v${MODULE_VERSION}.tar.gz
upload to host
```
ICINGAWEB_MODULEPATH="/usr/share/icingaweb2/modules"
TARGET_DIR="${ICINGAWEB_MODULEPATH}/grafana"
```
```
install -d -m 0755 "${TARGET_DIR}"
tar xfz /data/home/(you)/icingaweb2-module-grafana-1.3.6.tar.gz -C "${TARGET_DIR}" --strip-components 1
```
```
chmod -R 0755 /usr/share/icingaweb2/modules/grafana
mkdir /etc/icingaweb2/modules/grafana
```

```
vi /etc/icingaweb2/modules/grafana/config.ini
	[grafana]
  host = "(host):3000"
	protocol = "http"
	defaultdashboard = "base-metrics"
	defaultorgid = "1"
	defaultdashboardstore = "db"
	theme = "light"
	datasource = "influxdb"
	accessmode = "iframe"
	timerange = "6h"
```

```
vi /etc/icingaweb2/modules/grafana/graphs.ini
[ping4]
dashboard = "base-metrics"
panelId = "1"
orgId = ""

[ping6]
dashboard = "base-metrics"
panelId = "3"
orgId = ""

[hostalive]
dashboard = "base-metrics"
panelId = "9"
orgId = ""

[apt]
dashboard = "base-metrics"
panelId = "2"
orgId = ""

[load]
dashboard = "base-metrics"
panelId = "4"
orgId = ""

[swap]
dashboard = "base-metrics"
panelId = "5"
orgId = ""

[procs]
dashboard = "base-metrics"
panelId = "7"
orgId = ""

[ntp]
dashboard = "base-metrics"
panelId = "6"
orgId = ""

[users]
dashboard = "base-metrics"
panelId = "8"
orgId = ""
```

```
vi /etc/grafana/grafana.ini
[auth.anonymous]
enabled = true
allow_embedding = true
```
# selinux
```
ls -laZ /usr/share/icingaweb2/modules/
semanage fcontext -a -t icingaweb2_content_t "/usr/share/icingaweb2/modules(/.*)?"
cat /etc/selinux/targeted/contexts/files/file_contexts.local | grep icinga
restorecon -R -v /usr/share/icingaweb2/modules/
```

```
systemctl restart grafana-server
chown apache:icingaweb2 /etc/icingaweb2/modules/grafana/
chmod 770 /etc/icingaweb2/modules/grafana/
chmod 660 /etc/icingaweb2/modules/grafana/*
icingacli module list | grep grafana
icingacli module enable grafana
icingacli module list | grep grafana
systemctl restart httpd
systemctl restart rh-php71-php-fpm
chown -R apache:icingaweb2 /etc/icingaweb2
```

icingaweb2 > configuration > module > grafana > configuration > default-dashboard






#
# deploying by_ssh check style
#

# server side (made once)
```
passwd icinga
vi /etc/passwd
    # replace:
    # icinga:x:991:990:icinga:/var/spool/icinga2:/sbin/nologin
    icinga:x:991:990:icinga:/data/home/icinga:/bin/bash

mkdir /data/home/icinga
chown icinga:icinga /data/home/icinga
chmod 700 /data/home/icinga
ls -la /data/home
```
# move perfdata to new new home directory
```
mv -R /var/spool/icinga2* /data/home/icinga
systemctl restart icinga
```

# create ssh key pair, do not set passphrase
```
ssh-keygen -b 4096 -t rsa -C "icinga@$(hostname) (by_ssh check)" -f $HOME/.ssh/id_rsa
```

# if icinga home directory is elsewhere, move it
```
systemctl stop icinga2.service
usermod -m -d /data/home/icinga icinga
systemctl start icinga2.service
systemctl status icinga2.service
```

#
# check_nwc_health
#
```
git clone
# upload
```
```
yum install make
yum install gcc
yum install build-essential
yum install automake autoreconf
yum install dos2unix
dos2unix *
autoreconf
yum install -y perl-Net-SNMP perl-Data-Dumper perl-Module-Load
./configure --libexec=/usr/lib64/nagios/plugins --with-nagios-user=icinga --with-nagios-group=icinga
vi missing
    :set ff=unix
    :wq
./configure
make
cp plugins-scripts/check_nwc_health /usr/lib64/nagios/plugins/
chmod 755 /usr/lib64/nagios/plugins/check_nwc_health
```

#
# common issues:
#

# Remote command execution failed: @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
# Remote command execution failed: Host key verification failed.
# Remote command execution failed: bash: /data/home/nagios/libexec/check_disk: No such file or directory


##
## enabling API (probably from old documentation)
##


#
# add ApiUser and restart
#
vi /etc/icinga2/conf.d/api-users.conf


#
# test
#
curl -k -s -u collector:newpass 'https://127.0.0.1:5665/v1'