chore: copy elk setup from docker-elk git repo
https://github.com/deviantony/docker-elk
This commit is contained in:
ryan committed
1 parent
12cebe8192
commit
fc4ed05fb3
21 files changed
+783
No files matched your search
@@ -0,0 +1,12 @@
|
||||
# Ignore Docker build files
|
||||
Dockerfile
|
||||
.dockerignore
|
||||
|
||||
# Ignore OS artifacts
|
||||
**/.DS_Store
|
||||
|
||||
# Ignore Git files
|
||||
.gitignore
|
||||
|
||||
# Ignore setup state
|
||||
state/
|
||||
@@ -0,0 +1 @@
|
||||
/state/
|
||||
@@ -0,0 +1,15 @@
|
||||
ARG ELASTIC_VERSION
|
||||
|
||||
# https://www.docker.elastic.co/
|
||||
FROM docker.elastic.co/elasticsearch/elasticsearch:${ELASTIC_VERSION}
|
||||
|
||||
USER root
|
||||
|
||||
RUN set -eux; \
|
||||
mkdir /state; \
|
||||
chmod 0775 /state; \
|
||||
chown elasticsearch:root /state
|
||||
|
||||
USER elasticsearch:root
|
||||
|
||||
ENTRYPOINT ["/entrypoint.sh"]
|
||||
Executable
+134
@@ -0,0 +1,134 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
set -eu
|
||||
set -o pipefail
|
||||
|
||||
source "${BASH_SOURCE[0]%/*}"/lib.sh
|
||||
|
||||
|
||||
# --------------------------------------------------------
|
||||
# Users declarations
|
||||
|
||||
declare -A users_passwords
|
||||
users_passwords=(
|
||||
[logstash_internal]="${LOGSTASH_INTERNAL_PASSWORD:-}"
|
||||
[kibana_system]="${KIBANA_SYSTEM_PASSWORD:-}"
|
||||
[metricbeat_internal]="${METRICBEAT_INTERNAL_PASSWORD:-}"
|
||||
[filebeat_internal]="${FILEBEAT_INTERNAL_PASSWORD:-}"
|
||||
[heartbeat_internal]="${HEARTBEAT_INTERNAL_PASSWORD:-}"
|
||||
[monitoring_internal]="${MONITORING_INTERNAL_PASSWORD:-}"
|
||||
[beats_system]="${BEATS_SYSTEM_PASSWORD=:-}"
|
||||
)
|
||||
|
||||
declare -A users_roles
|
||||
users_roles=(
|
||||
[logstash_internal]='logstash_writer'
|
||||
[metricbeat_internal]='metricbeat_writer'
|
||||
[filebeat_internal]='filebeat_writer'
|
||||
[heartbeat_internal]='heartbeat_writer'
|
||||
[monitoring_internal]='remote_monitoring_collector'
|
||||
)
|
||||
|
||||
# --------------------------------------------------------
|
||||
# Roles declarations
|
||||
|
||||
declare -A roles_files
|
||||
roles_files=(
|
||||
[logstash_writer]='logstash_writer.json'
|
||||
[metricbeat_writer]='metricbeat_writer.json'
|
||||
[filebeat_writer]='filebeat_writer.json'
|
||||
[heartbeat_writer]='heartbeat_writer.json'
|
||||
)
|
||||
|
||||
# --------------------------------------------------------
|
||||
|
||||
|
||||
echo "-------- $(date --rfc-3339=seconds) --------"
|
||||
|
||||
state_file="${BASH_SOURCE[0]%/*}"/state/.done
|
||||
if [[ -e "$state_file" ]]; then
|
||||
declare state_birthtime
|
||||
state_birthtime="$(stat -c '%Y' "$state_file")"
|
||||
state_birthtime="$(date --rfc-3339=seconds --date="@${state_birthtime}")"
|
||||
|
||||
log "Setup has already run successfully on ${state_birthtime}. Skipping"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
log 'Waiting for availability of Elasticsearch. This can take several minutes.'
|
||||
|
||||
declare -i exit_code=0
|
||||
wait_for_elasticsearch || exit_code=$?
|
||||
|
||||
if ((exit_code)); then
|
||||
case $exit_code in
|
||||
6)
|
||||
suberr 'Could not resolve host. Is Elasticsearch running?'
|
||||
;;
|
||||
7)
|
||||
suberr 'Failed to connect to host. Is Elasticsearch healthy?'
|
||||
;;
|
||||
28)
|
||||
suberr 'Timeout connecting to host. Is Elasticsearch healthy?'
|
||||
;;
|
||||
*)
|
||||
suberr "Connection to Elasticsearch failed. Exit code: ${exit_code}"
|
||||
;;
|
||||
esac
|
||||
|
||||
exit $exit_code
|
||||
fi
|
||||
|
||||
sublog 'Elasticsearch is running'
|
||||
|
||||
log 'Waiting for initialization of built-in users'
|
||||
|
||||
wait_for_builtin_users || exit_code=$?
|
||||
|
||||
if ((exit_code)); then
|
||||
suberr 'Timed out waiting for condition'
|
||||
exit $exit_code
|
||||
fi
|
||||
|
||||
sublog 'Built-in users were initialized'
|
||||
|
||||
for role in "${!roles_files[@]}"; do
|
||||
log "Role '$role'"
|
||||
|
||||
declare body_file
|
||||
body_file="${BASH_SOURCE[0]%/*}/roles/${roles_files[$role]:-}"
|
||||
if [[ ! -f "${body_file:-}" ]]; then
|
||||
sublog "No role body found at '${body_file}', skipping"
|
||||
continue
|
||||
fi
|
||||
|
||||
sublog 'Creating/updating'
|
||||
ensure_role "$role" "$(<"${body_file}")"
|
||||
done
|
||||
|
||||
for user in "${!users_passwords[@]}"; do
|
||||
log "User '$user'"
|
||||
if [[ -z "${users_passwords[$user]:-}" ]]; then
|
||||
sublog 'No password defined, skipping'
|
||||
continue
|
||||
fi
|
||||
|
||||
declare -i user_exists=0
|
||||
user_exists="$(check_user_exists "$user")"
|
||||
|
||||
if ((user_exists)); then
|
||||
sublog 'User exists, setting password'
|
||||
set_user_password "$user" "${users_passwords[$user]}"
|
||||
else
|
||||
if [[ -z "${users_roles[$user]:-}" ]]; then
|
||||
suberr ' No role defined, skipping creation'
|
||||
continue
|
||||
fi
|
||||
|
||||
sublog 'User does not exist, creating'
|
||||
create_user "$user" "${users_passwords[$user]}" "${users_roles[$user]}"
|
||||
fi
|
||||
done
|
||||
|
||||
mkdir -p "${state_file%/*}"
|
||||
touch "$state_file"
|
||||
+240
@@ -0,0 +1,240 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
# Log a message.
|
||||
function log {
|
||||
echo "[+] $1"
|
||||
}
|
||||
|
||||
# Log a message at a sub-level.
|
||||
function sublog {
|
||||
echo " ⠿ $1"
|
||||
}
|
||||
|
||||
# Log an error.
|
||||
function err {
|
||||
echo "[x] $1" >&2
|
||||
}
|
||||
|
||||
# Log an error at a sub-level.
|
||||
function suberr {
|
||||
echo " ⠍ $1" >&2
|
||||
}
|
||||
|
||||
# Poll the 'elasticsearch' service until it responds with HTTP code 200.
|
||||
function wait_for_elasticsearch {
|
||||
local elasticsearch_host="${ELASTICSEARCH_HOST:-elasticsearch}"
|
||||
|
||||
local -a args=( '-s' '-D-' '-m15' '-w' '%{http_code}' "http://${elasticsearch_host}:9200/" )
|
||||
|
||||
if [[ -n "${ELASTIC_PASSWORD:-}" ]]; then
|
||||
args+=( '-u' "elastic:${ELASTIC_PASSWORD}" )
|
||||
fi
|
||||
|
||||
local -i result=1
|
||||
local output
|
||||
|
||||
# retry for max 300s (60*5s)
|
||||
for _ in $(seq 1 60); do
|
||||
local -i exit_code=0
|
||||
output="$(curl "${args[@]}")" || exit_code=$?
|
||||
|
||||
if ((exit_code)); then
|
||||
result=$exit_code
|
||||
fi
|
||||
|
||||
if [[ "${output: -3}" -eq 200 ]]; then
|
||||
result=0
|
||||
break
|
||||
fi
|
||||
|
||||
sleep 5
|
||||
done
|
||||
|
||||
if ((result)) && [[ "${output: -3}" -ne 000 ]]; then
|
||||
echo -e "\n${output::-3}"
|
||||
fi
|
||||
|
||||
return $result
|
||||
}
|
||||
|
||||
# Poll the Elasticsearch users API until it returns users.
|
||||
function wait_for_builtin_users {
|
||||
local elasticsearch_host="${ELASTICSEARCH_HOST:-elasticsearch}"
|
||||
|
||||
local -a args=( '-s' '-D-' '-m15' "http://${elasticsearch_host}:9200/_security/user?pretty" )
|
||||
|
||||
if [[ -n "${ELASTIC_PASSWORD:-}" ]]; then
|
||||
args+=( '-u' "elastic:${ELASTIC_PASSWORD}" )
|
||||
fi
|
||||
|
||||
local -i result=1
|
||||
|
||||
local line
|
||||
local -i exit_code
|
||||
local -i num_users
|
||||
|
||||
# retry for max 30s (30*1s)
|
||||
for _ in $(seq 1 30); do
|
||||
num_users=0
|
||||
|
||||
# read exits with a non-zero code if the last read input doesn't end
|
||||
# with a newline character. The printf without newline that follows the
|
||||
# curl command ensures that the final input not only contains curl's
|
||||
# exit code, but causes read to fail so we can capture the return value.
|
||||
# Ref. https://unix.stackexchange.com/a/176703/152409
|
||||
while IFS= read -r line || ! exit_code="$line"; do
|
||||
if [[ "$line" =~ _reserved.+true ]]; then
|
||||
(( num_users++ ))
|
||||
fi
|
||||
done < <(curl "${args[@]}"; printf '%s' "$?")
|
||||
|
||||
if ((exit_code)); then
|
||||
result=$exit_code
|
||||
fi
|
||||
|
||||
# we expect more than just the 'elastic' user in the result
|
||||
if (( num_users > 1 )); then
|
||||
result=0
|
||||
break
|
||||
fi
|
||||
|
||||
sleep 1
|
||||
done
|
||||
|
||||
return $result
|
||||
}
|
||||
|
||||
# Verify that the given Elasticsearch user exists.
|
||||
function check_user_exists {
|
||||
local username=$1
|
||||
|
||||
local elasticsearch_host="${ELASTICSEARCH_HOST:-elasticsearch}"
|
||||
|
||||
local -a args=( '-s' '-D-' '-m15' '-w' '%{http_code}'
|
||||
"http://${elasticsearch_host}:9200/_security/user/${username}"
|
||||
)
|
||||
|
||||
if [[ -n "${ELASTIC_PASSWORD:-}" ]]; then
|
||||
args+=( '-u' "elastic:${ELASTIC_PASSWORD}" )
|
||||
fi
|
||||
|
||||
local -i result=1
|
||||
local -i exists=0
|
||||
local output
|
||||
|
||||
output="$(curl "${args[@]}")"
|
||||
if [[ "${output: -3}" -eq 200 || "${output: -3}" -eq 404 ]]; then
|
||||
result=0
|
||||
fi
|
||||
if [[ "${output: -3}" -eq 200 ]]; then
|
||||
exists=1
|
||||
fi
|
||||
|
||||
if ((result)); then
|
||||
echo -e "\n${output::-3}"
|
||||
else
|
||||
echo "$exists"
|
||||
fi
|
||||
|
||||
return $result
|
||||
}
|
||||
|
||||
# Set password of a given Elasticsearch user.
|
||||
function set_user_password {
|
||||
local username=$1
|
||||
local password=$2
|
||||
|
||||
local elasticsearch_host="${ELASTICSEARCH_HOST:-elasticsearch}"
|
||||
|
||||
local -a args=( '-s' '-D-' '-m15' '-w' '%{http_code}'
|
||||
"http://${elasticsearch_host}:9200/_security/user/${username}/_password"
|
||||
'-X' 'POST'
|
||||
'-H' 'Content-Type: application/json'
|
||||
'-d' "{\"password\" : \"${password}\"}"
|
||||
)
|
||||
|
||||
if [[ -n "${ELASTIC_PASSWORD:-}" ]]; then
|
||||
args+=( '-u' "elastic:${ELASTIC_PASSWORD}" )
|
||||
fi
|
||||
|
||||
local -i result=1
|
||||
local output
|
||||
|
||||
output="$(curl "${args[@]}")"
|
||||
if [[ "${output: -3}" -eq 200 ]]; then
|
||||
result=0
|
||||
fi
|
||||
|
||||
if ((result)); then
|
||||
echo -e "\n${output::-3}\n"
|
||||
fi
|
||||
|
||||
return $result
|
||||
}
|
||||
|
||||
# Create the given Elasticsearch user.
|
||||
function create_user {
|
||||
local username=$1
|
||||
local password=$2
|
||||
local role=$3
|
||||
|
||||
local elasticsearch_host="${ELASTICSEARCH_HOST:-elasticsearch}"
|
||||
|
||||
local -a args=( '-s' '-D-' '-m15' '-w' '%{http_code}'
|
||||
"http://${elasticsearch_host}:9200/_security/user/${username}"
|
||||
'-X' 'POST'
|
||||
'-H' 'Content-Type: application/json'
|
||||
'-d' "{\"password\":\"${password}\",\"roles\":[\"${role}\"]}"
|
||||
)
|
||||
|
||||
if [[ -n "${ELASTIC_PASSWORD:-}" ]]; then
|
||||
args+=( '-u' "elastic:${ELASTIC_PASSWORD}" )
|
||||
fi
|
||||
|
||||
local -i result=1
|
||||
local output
|
||||
|
||||
output="$(curl "${args[@]}")"
|
||||
if [[ "${output: -3}" -eq 200 ]]; then
|
||||
result=0
|
||||
fi
|
||||
|
||||
if ((result)); then
|
||||
echo -e "\n${output::-3}\n"
|
||||
fi
|
||||
|
||||
return $result
|
||||
}
|
||||
|
||||
# Ensure that the given Elasticsearch role is up-to-date, create it if required.
|
||||
function ensure_role {
|
||||
local name=$1
|
||||
local body=$2
|
||||
|
||||
local elasticsearch_host="${ELASTICSEARCH_HOST:-elasticsearch}"
|
||||
|
||||
local -a args=( '-s' '-D-' '-m15' '-w' '%{http_code}'
|
||||
"http://${elasticsearch_host}:9200/_security/role/${name}"
|
||||
'-X' 'POST'
|
||||
'-H' 'Content-Type: application/json'
|
||||
'-d' "$body"
|
||||
)
|
||||
|
||||
if [[ -n "${ELASTIC_PASSWORD:-}" ]]; then
|
||||
args+=( '-u' "elastic:${ELASTIC_PASSWORD}" )
|
||||
fi
|
||||
|
||||
local -i result=1
|
||||
local output
|
||||
|
||||
output="$(curl "${args[@]}")"
|
||||
if [[ "${output: -3}" -eq 200 ]]; then
|
||||
result=0
|
||||
fi
|
||||
|
||||
if ((result)); then
|
||||
echo -e "\n${output::-3}\n"
|
||||
fi
|
||||
|
||||
return $result
|
||||
}
|
||||
@@ -0,0 +1,19 @@
|
||||
{
|
||||
"cluster": [
|
||||
"manage_ilm",
|
||||
"manage_index_templates",
|
||||
"monitor",
|
||||
"read_pipeline"
|
||||
],
|
||||
"indices": [
|
||||
{
|
||||
"names": [
|
||||
"filebeat-*"
|
||||
],
|
||||
"privileges": [
|
||||
"create_doc",
|
||||
"manage"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
{
|
||||
"cluster": [
|
||||
"manage_ilm",
|
||||
"manage_index_templates",
|
||||
"monitor"
|
||||
],
|
||||
"indices": [
|
||||
{
|
||||
"names": [
|
||||
"heartbeat-*"
|
||||
],
|
||||
"privileges": [
|
||||
"create_doc",
|
||||
"manage"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,33 @@
|
||||
{
|
||||
"cluster": [
|
||||
"manage_index_templates",
|
||||
"monitor",
|
||||
"manage_ilm"
|
||||
],
|
||||
"indices": [
|
||||
{
|
||||
"names": [
|
||||
"logs-generic-default",
|
||||
"logstash-*",
|
||||
"ecs-logstash-*"
|
||||
],
|
||||
"privileges": [
|
||||
"write",
|
||||
"create",
|
||||
"create_index",
|
||||
"manage",
|
||||
"manage_ilm"
|
||||
]
|
||||
},
|
||||
{
|
||||
"names": [
|
||||
"logstash",
|
||||
"ecs-logstash"
|
||||
],
|
||||
"privileges": [
|
||||
"write",
|
||||
"manage"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,19 @@
|
||||
{
|
||||
"cluster": [
|
||||
"manage_ilm",
|
||||
"manage_index_templates",
|
||||
"monitor"
|
||||
],
|
||||
"indices": [
|
||||
{
|
||||
"names": [
|
||||
".monitoring-*-mb",
|
||||
"metricbeat-*"
|
||||
],
|
||||
"privileges": [
|
||||
"create_doc",
|
||||
"manage"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
Reference in new issue
Block a user