diff --git a/ror-demo-cluster/.env-showcase b/ror-demo-cluster/.env-showcase new file mode 100644 index 0000000..61f9cd8 --- /dev/null +++ b/ror-demo-cluster/.env-showcase @@ -0,0 +1,14 @@ +# Non-interactive configuration for ror-demo-cluster. +# When this file is present and non-empty, run.sh skips all interactive prompts and uses these values directly. +# +# ES/KBN DOCKERFILE options: +# Dockerfile-use-ror-binaries-from-api - download ROR plugin from API (requires ROR_ES_VERSION / ROR_KBN_VERSION) +# Dockerfile-use-ror-binaries-from-file - use a local plugin file (requires ES_ROR_FILE / KBN_ROR_FILE) + +ES_VERSION=8.19.11 +ES_DOCKERFILE=Dockerfile-use-ror-binaries-from-file +ES_ROR_FILE=readonlyrest-1.69.0-pre01_es8.19.11.zip + +KBN_VERSION=8.19.11 +KBN_DOCKERFILE=Dockerfile-use-ror-binaries-from-api +ROR_KBN_VERSION=1.68.0 diff --git a/ror-demo-cluster/conf/es/readonlyrest.yml b/ror-demo-cluster/conf/es/readonlyrest.yml index f52599b..3b33875 100644 --- a/ror-demo-cluster/conf/es/readonlyrest.yml +++ b/ror-demo-cluster/conf/es/readonlyrest.yml @@ -18,7 +18,7 @@ readonlyrest: - name: "End users" groups: ["EndUsers"] - indices: ["frontend_logs", "kibana_sample_data_*"] + indices: ["logs-frontend-dev", "kibana_sample_data_*"] kibana: index: .kibana_end_@{user} access: rw @@ -26,10 +26,10 @@ readonlyrest: - name: "Business users" groups: ["BusinessUsers"] - indices: ["business_logs", "kibana_sample_data_*"] + indices: ["logs-business-dev", "kibana_sample_data_*"] kibana: index: .kibana_business_@{user} - access: rw + access: ro hide_apps: ["Security", "Observability"] users: diff --git a/ror-demo-cluster/conf/kbn/enterprise-ror-newplatform-kibana.yml b/ror-demo-cluster/conf/kbn/enterprise-ror-newplatform-kibana.yml index ff5adc3..5f396db 100644 --- a/ror-demo-cluster/conf/kbn/enterprise-ror-newplatform-kibana.yml +++ b/ror-demo-cluster/conf/kbn/enterprise-ror-newplatform-kibana.yml @@ -12,6 +12,8 @@ server.ssl.certificate: /usr/share/kibana/config/kibana.crt server.ssl.key: /usr/share/kibana/config/kibana.key server.ssl.redirectHttpFromPort: 80 +xpack.encryptedSavedObjects.encryptionKey: "min-32-byte-long-strong-encryption-key" + readonlyrest_kbn.logLevel: info readonlyrest_kbn.cookiePass: '12312313123213123213123abcdefghijklm' readonlyrest_kbn: diff --git a/ror-demo-cluster/conf/kbn/free-ror-newplatform-kibana.yml b/ror-demo-cluster/conf/kbn/free-ror-newplatform-kibana.yml index 3df3d8e..420783d 100644 --- a/ror-demo-cluster/conf/kbn/free-ror-newplatform-kibana.yml +++ b/ror-demo-cluster/conf/kbn/free-ror-newplatform-kibana.yml @@ -12,5 +12,7 @@ server.ssl.certificate: /usr/share/kibana/config/kibana.crt server.ssl.key: /usr/share/kibana/config/kibana.key server.ssl.redirectHttpFromPort: 80 +xpack.encryptedSavedObjects.encryptionKey: "min-32-byte-long-strong-encryption-key" + readonlyrest_kbn.logLevel: info readonlyrest_kbn.cookiePass: '12312313123213123213123abcdefghijklm' diff --git a/ror-demo-cluster/conf/kbn/pro-ror-newplatform-kibana.yml b/ror-demo-cluster/conf/kbn/pro-ror-newplatform-kibana.yml index 3df3d8e..420783d 100644 --- a/ror-demo-cluster/conf/kbn/pro-ror-newplatform-kibana.yml +++ b/ror-demo-cluster/conf/kbn/pro-ror-newplatform-kibana.yml @@ -12,5 +12,7 @@ server.ssl.certificate: /usr/share/kibana/config/kibana.crt server.ssl.key: /usr/share/kibana/config/kibana.key server.ssl.redirectHttpFromPort: 80 +xpack.encryptedSavedObjects.encryptionKey: "min-32-byte-long-strong-encryption-key" + readonlyrest_kbn.logLevel: info readonlyrest_kbn.cookiePass: '12312313123213123213123abcdefghijklm' diff --git a/ror-demo-cluster/readonlyrest-1.69.0-pre01_es8.19.11.zip b/ror-demo-cluster/readonlyrest-1.69.0-pre01_es8.19.11.zip new file mode 100644 index 0000000..a953d0a Binary files /dev/null and b/ror-demo-cluster/readonlyrest-1.69.0-pre01_es8.19.11.zip differ diff --git a/shared/init-scripts/init.sh b/shared/init-scripts/init.sh index 70164aa..183097a 100755 --- a/shared/init-scripts/init.sh +++ b/shared/init-scripts/init.sh @@ -4,37 +4,10 @@ cd "$(dirname "$0")" source utils/lib.sh -function generate_log_documents() { - if [ "$#" -ne 1 ]; then - echo "ERROR: One required: 1) number of documents to generate" - return 1 - fi +createDataStream "logs-frontend-dev" && generate_log_documents 100 | putDocument "logs-frontend-dev" +createDataStream "logs-business-dev" && generate_log_documents 100 | putDocument "logs-business-dev" +createDataStream "logs-system-dev" && generate_log_documents 100 | putDocument "logs-system-dev" - N=$1 - - for ((i = 1; i <= N; i++)); do - user_id=$((RANDOM % 10000 + 1)) - timestamp=$(date -u +"%Y-%m-%dT%H:%M:%SZ") - log_message="User $user_id login successful" - level="$(pick_randomly "INFO" "WARN" "ERROR" "DEBUG")" - - echo "{ \"message\": \"$log_message\", \"level\": \"$level\", \"timestamp\": \"$timestamp\", \"user_id\": \"$user_id\" }" - done -} - -function index_documents() { - if [ "$#" -ne 1 ]; then - echo "ERROR: One required: 1) index name" - return 1 - fi - - INDEX_NAME=$1 - - while IFS= read -r document; do - putDocument "$INDEX_NAME" "$document" - done -} - -generate_log_documents 100 | index_documents "frontend_logs" -generate_log_documents 50 | index_documents "business_logs" -generate_log_documents 60 | index_documents "system_logs" \ No newline at end of file +#createIndex "frontend_logs_index" && generate_log_documents 100 | putDocument "frontend_logs_index" +#createIndex "business_logs_index" && generate_log_documents 50 | putDocument "business_logs_index" +#createIndex "system_logs_index" && generate_log_documents 60 | putDocument "system_logs_index" \ No newline at end of file diff --git a/shared/init-scripts/utils/lib.sh b/shared/init-scripts/utils/lib.sh index fd0e7b8..be08b88 100755 --- a/shared/init-scripts/utils/lib.sh +++ b/shared/init-scripts/utils/lib.sh @@ -7,9 +7,92 @@ function pick_randomly() { echo "${OPTIONS[$RANDOM_INDEX]}" } +function createIndex() { + if [ "$#" -ne 1 ]; then + echo "ERROR: One parameter required: 1) index name" + return 1 + fi + + if ! [ -v ELASTICSEARCH_ADDRESS ] || [ -z "$ELASTICSEARCH_ADDRESS" ]; then + echo "ERROR: required variable ELASTICSEARCH_ADDRESS not set or empty" + exit 2 + fi + + if ! [ -v ELASTICSEARCH_USER ] || [ -z "$ELASTICSEARCH_USER" ]; then + echo "ERROR: required variable ELASTICSEARCH_USER not set or empty" + exit 3 + fi + + if ! [ -v ELASTICSEARCH_PASSWORD ] || [ -z "$ELASTICSEARCH_PASSWORD" ]; then + echo "ERROR: required variable ELASTICSEARCH_PASSWORD not set or empty" + exit 4 + fi + + INDEX_NAME=$1 + + response=$(curl -k -s -L -w "\n%{http_code}" -u $ELASTICSEARCH_USER:$ELASTICSEARCH_PASSWORD \ + -X PUT "$ELASTICSEARCH_ADDRESS/$INDEX_NAME" \ + -H "Content-Type: application/json" + ) + + http_status=$(echo "$response" | tail -n 1) + response_body=$(echo "$response" | sed \$d) + + if [[ "$http_status" != 2* ]]; then + echo "ERROR: Cannot create index [$INDEX_NAME]. HTTP status: $http_status, response body: $response_body" + return 5 + fi + + return 0 +} + +function createDataStream() { + if [ "$#" -ne 1 ]; then + echo "ERROR: One parameter required: 1) data stream name" + return 1 + fi + + if ! [ -v ELASTICSEARCH_ADDRESS ] || [ -z "$ELASTICSEARCH_ADDRESS" ]; then + echo "ERROR: required variable ELASTICSEARCH_ADDRESS not set or empty" + exit 2 + fi + + if ! [ -v ELASTICSEARCH_USER ] || [ -z "$ELASTICSEARCH_USER" ]; then + echo "ERROR: required variable ELASTICSEARCH_USER not set or empty" + exit 3 + fi + + if ! [ -v ELASTICSEARCH_PASSWORD ] || [ -z "$ELASTICSEARCH_PASSWORD" ]; then + echo "ERROR: required variable ELASTICSEARCH_PASSWORD not set or empty" + exit 4 + fi + + STREAM_NAME=$1 + TEMPLATE_NAME="${STREAM_NAME}-template" + + response=$(curl -k -s -L -w "\n%{http_code}" -u $ELASTICSEARCH_USER:$ELASTICSEARCH_PASSWORD \ + -X PUT "$ELASTICSEARCH_ADDRESS/_index_template/$TEMPLATE_NAME" \ + -H "Content-Type: application/json" -d "{ + \"index_patterns\": [\"$STREAM_NAME\"], + \"data_stream\": {}, + \"priority\": 500 + }" + ) + + http_status=$(echo "$response" | tail -n 1) + response_body=$(echo "$response" | sed \$d) + + if [[ "$http_status" != 2* ]]; then + echo "ERROR: Cannot create index template for data stream [$STREAM_NAME]. HTTP status: $http_status, response body: $response_body" + return 5 + fi + + return 0 +} + function putDocument() { - if [ "$#" -ne 2 ]; then - echo "ERROR: Three parameters required: 1) index name, 2) document JSON string" + if [ "$#" -lt 1 ] || [ "$#" -gt 2 ]; then + echo "ERROR: Required: 1) index name, optionally 2) document JSON string (or via stdin)" return 1 fi @@ -29,22 +112,50 @@ function putDocument() { fi INDEX_NAME=$1 - DOCUMENT_CONTENT=$2 -set -x + if [ "$#" -eq 2 ]; then + putSingleDocument "$INDEX_NAME" "$2" + else + while IFS= read -r DOCUMENT_CONTENT; do + putSingleDocument "$INDEX_NAME" "$DOCUMENT_CONTENT" + done + fi +} - respone=$(curl -k -s -L -w "\n%{http_code}" -u $ELASTICSEARCH_USER:$ELASTICSEARCH_PASSWORD \ +function putSingleDocument() { + INDEX_NAME=$1 + DOCUMENT_CONTENT=$2 + + response=$(curl -k -s -L -w "\n%{http_code}" -u $ELASTICSEARCH_USER:$ELASTICSEARCH_PASSWORD \ -X POST "$ELASTICSEARCH_ADDRESS/$INDEX_NAME/_doc/" \ -H "Content-Type: application/json" -d "$DOCUMENT_CONTENT" ) - http_status=$(echo "$respone" | tail -n 1) - response_body=$(echo "$respone" | sed \$d) + http_status=$(echo "$response" | tail -n 1) + response_body=$(echo "$response" | sed \$d) if [[ "$http_status" != 2* ]] ; then - echo "ERROR: Cannot add document [$DOCUMENT_CONTENT] to index=[$INDEX_NAME].\nHTTP status: $HTTP_STATUS, response body: $RESPONSE_BODY" + echo "ERROR: Cannot add document [$DOCUMENT_CONTENT] to index=[$INDEX_NAME].\nHTTP status: $http_status, response body: $response_body" return 5 fi return 0 } + +function generate_log_documents() { + if [ "$#" -ne 1 ]; then + echo "ERROR: One required: 1) number of documents to generate" + return 1 + fi + + N=$1 + + for ((i = 1; i <= N; i++)); do + user_id=$((RANDOM % 10000 + 1)) + timestamp=$(date -u +"%Y-%m-%dT%H:%M:%SZ") + log_message="User $user_id login successful" + level="$(pick_randomly "INFO" "WARN" "ERROR" "DEBUG")" + + echo "{ \"message\": \"$log_message\", \"level\": \"$level\", \"@timestamp\": \"$timestamp\", \"user_id\": \"$user_id\" }" + done +} \ No newline at end of file diff --git a/utils/collect-info-about-ror-es-kbn.sh b/utils/collect-info-about-ror-es-kbn.sh index cf02878..112f81a 100755 --- a/utils/collect-info-about-ror-es-kbn.sh +++ b/utils/collect-info-about-ror-es-kbn.sh @@ -2,6 +2,30 @@ echo "Preparing Elasticsearch & Kibana with ROR environment ..." +if [[ -s ".env-showcase" ]]; then + echo "Found .env-showcase - running in non-interactive mode ..." + cp .env-showcase .env + + source .env-showcase + + if [[ "$ES_DOCKERFILE" == *"from-file"* ]]; then + es_ror_info="FILE: $ES_ROR_FILE" + else + es_ror_info="API: ROR ES $ROR_ES_VERSION" + fi + + if [[ "$KBN_DOCKERFILE" == *"from-file"* ]]; then + kbn_ror_info="FILE: $KBN_ROR_FILE" + else + kbn_ror_info="API: ROR KBN $ROR_KBN_VERSION" + fi + + echo " Elasticsearch $ES_VERSION ($es_ror_info)" + echo " Kibana $KBN_VERSION ($kbn_ror_info)" + + exit 0 +fi + if ! command -v jq > /dev/null; then $(dirname "$0")/collect-info-about-ror-es-kbn-without-hints.sh else diff --git a/xpack-docker-demo-cluster/docker-compose.yml b/xpack-docker-demo-cluster/docker-compose.yml index e89e548..86eecab 100644 --- a/xpack-docker-demo-cluster/docker-compose.yml +++ b/xpack-docker-demo-cluster/docker-compose.yml @@ -9,7 +9,7 @@ services: ports: - "29200:9200" - "29300:9300" - - "5005:5005" + - "5015:5005" environment: - cluster.name=xpack-es-cluster - node.name=es-xpack-single