diff --git a/ror-demo-cluster/conf/es/readonlyrest.yml b/ror-demo-cluster/conf/es/readonlyrest.yml index f52599b..d86865c 100644 --- a/ror-demo-cluster/conf/es/readonlyrest.yml +++ b/ror-demo-cluster/conf/es/readonlyrest.yml @@ -18,7 +18,7 @@ readonlyrest: - name: "End users" groups: ["EndUsers"] - indices: ["frontend_logs", "kibana_sample_data_*"] + indices: ["frontend_logs*", "kibana_sample_data_*"] kibana: index: .kibana_end_@{user} access: rw @@ -26,10 +26,18 @@ readonlyrest: - name: "Business users" groups: ["BusinessUsers"] - indices: ["business_logs", "kibana_sample_data_*"] + indices: ["business_logs*", "kibana_sample_data_*"] kibana: index: .kibana_business_@{user} - access: rw + access: ro + hide_apps: ["Security", "Observability"] + + - name: "Test users" + groups: ["TestUsers"] + indices: ["business_logs*", "kibana_sample_data_*"] + kibana: + index: .kibana_test_@{user} + access: ro_strict hide_apps: ["Security", "Observability"] users: @@ -42,6 +50,8 @@ readonlyrest: name: "End Users" - id: "BusinessUsers" name: "Business Users" + - id: "TestUsers" + name: "Test Users" - username: user1 auth_key: user1:test @@ -50,26 +60,11 @@ readonlyrest: name: "End Users" - id: "BusinessUsers" name: "Business Users" + - id: "TestUsers" + name: "Test Users" - username: user2 auth_key: user2:test groups: - id: "EndUsers" name: "End Users" - - - username: "*" - ror_kbn_auth: - name: "kbn1" - groups: ["*"] - groups: - - local_group: - id: "EndUsers" - name: "End Users" - external_group_ids: [ "extEndUsers" ] - - local_group: - id: "BusinessUsers" - name: "Business Users" - external_group_ids: [ "extBusinessUsers" ] - ror_kbn: - - name: kbn1 - signature_key: "9yzBfnLaTYLfGPzyKW9es76RKYhUVgmuv6ZtehaScj5msGpBpa5FWpwk295uJYaaffTFnQC5tsknh2AguVDaTrqCLfM5zCTqdE4UGNL73h28Bg4dPrvTAFQyygQqv4xfgnevBED6VZYdfjXAQLc8J8ywaHQQSmprZqYCWGE6sM3vzNUEWWB3kmGrEKa4sGbXhmXZCvL6NDnEJhXPDJAzu9BMQxn8CzVLqrx6BxDgPYF8gZCxtyxMckXwCaYXrxAGbjkYH69F4wYhuAdHSWgRAQCuWwYmWCA6g39j4VPge5pv962XYvxwJpvn23Y5KvNZ5S5c6crdG4f4gTCXnU36x92fKMQzsQV9K4phcuNvMWkpqVB6xMA5aPzUeHcGytD93dG8D52P5BxsgaJJE6QqDrk3Y2vyLw9ZEbJhPRJxbuBKVCBtVx26Ldd46dq5eyyzmNEyQGLrjQ4qd978VtG8TNT5rkn4ETJQEju5HfCBbjm3urGLFVqxhGVawecT4YM9Rry4EqXWkRJGTFQWQRnweUFbKNbVTC9NxcXEp6K5rSPEy9trb5UYLYhhMJ9fWSBMuenGRjNSJxeurMRCaxPpNppBLFnp8qW5ezfHgCBpEjkSNNzP4uXMZFAXmdUfJ8XQdPTWuYfdHYc5TZWnzrdq9wcfFQRDpDB2zX5Myu96krDt9vA7wNKfYwkSczA6qUQV66jA8nV4Cs38cDAKVBXnxz22ddAVrPv8ajpu7hgBtULMURjvLt94Nc5FDKw79CTTQxffWEj9BJCDCpQnTufmT8xenywwVJvtj49yv2MP2mGECrVDRmcGUAYBKR8G6ZnFAYDVC9UhY46FGWDcyVX3HKwgtHeb45Ww7dsW8JdMnZYctaEU585GZmqTJp2LcAWRcQPH25JewnPX8pjzVpJNcy7avfA2bcU86bfASvQBDUCrhjgRmK2ECR6vzPwTsYKRgFrDqb62FeMdrKgJ9vKs435T5ACN7MNtdRXHQ4fj5pNpUMDW26Wd7tt9bkBTqEGf" diff --git a/ror-demo-cluster/conf/kbn/enterprise-ror-newplatform-kibana.yml b/ror-demo-cluster/conf/kbn/enterprise-ror-newplatform-kibana.yml index ff5adc3..6deed2f 100644 --- a/ror-demo-cluster/conf/kbn/enterprise-ror-newplatform-kibana.yml +++ b/ror-demo-cluster/conf/kbn/enterprise-ror-newplatform-kibana.yml @@ -12,53 +12,15 @@ server.ssl.certificate: /usr/share/kibana/config/kibana.crt server.ssl.key: /usr/share/kibana/config/kibana.key server.ssl.redirectHttpFromPort: 80 -readonlyrest_kbn.logLevel: info -readonlyrest_kbn.cookiePass: '12312313123213123213123abcdefghijklm' -readonlyrest_kbn: - auth: - signature_key: "9yzBfnLaTYLfGPzyKW9es76RKYhUVgmuv6ZtehaScj5msGpBpa5FWpwk295uJYaaffTFnQC5tsknh2AguVDaTrqCLfM5zCTqdE4UGNL73h28Bg4dPrvTAFQyygQqv4xfgnevBED6VZYdfjXAQLc8J8ywaHQQSmprZqYCWGE6sM3vzNUEWWB3kmGrEKa4sGbXhmXZCvL6NDnEJhXPDJAzu9BMQxn8CzVLqrx6BxDgPYF8gZCxtyxMckXwCaYXrxAGbjkYH69F4wYhuAdHSWgRAQCuWwYmWCA6g39j4VPge5pv962XYvxwJpvn23Y5KvNZ5S5c6crdG4f4gTCXnU36x92fKMQzsQV9K4phcuNvMWkpqVB6xMA5aPzUeHcGytD93dG8D52P5BxsgaJJE6QqDrk3Y2vyLw9ZEbJhPRJxbuBKVCBtVx26Ldd46dq5eyyzmNEyQGLrjQ4qd978VtG8TNT5rkn4ETJQEju5HfCBbjm3urGLFVqxhGVawecT4YM9Rry4EqXWkRJGTFQWQRnweUFbKNbVTC9NxcXEp6K5rSPEy9trb5UYLYhhMJ9fWSBMuenGRjNSJxeurMRCaxPpNppBLFnp8qW5ezfHgCBpEjkSNNzP4uXMZFAXmdUfJ8XQdPTWuYfdHYc5TZWnzrdq9wcfFQRDpDB2zX5Myu96krDt9vA7wNKfYwkSczA6qUQV66jA8nV4Cs38cDAKVBXnxz22ddAVrPv8ajpu7hgBtULMURjvLt94Nc5FDKw79CTTQxffWEj9BJCDCpQnTufmT8xenywwVJvtj49yv2MP2mGECrVDRmcGUAYBKR8G6ZnFAYDVC9UhY46FGWDcyVX3HKwgtHeb45Ww7dsW8JdMnZYctaEU585GZmqTJp2LcAWRcQPH25JewnPX8pjzVpJNcy7avfA2bcU86bfASvQBDUCrhjgRmK2ECR6vzPwTsYKRgFrDqb62FeMdrKgJ9vKs435T5ACN7MNtdRXHQ4fj5pNpUMDW26Wd7tt9bkBTqEGf" - saml_keycloak: - buttonName: 'Keycloak SAML' - enabled: true - type: 'saml' - issuer: 'ror-saml' - entryPoint: 'http://kc.localhost:8080/realms/ror/protocol/saml' - kibanaExternalHost: 'localhost:15601' - protocol: 'https' - usernameParameter: 'nameID' - groupsParameter: 'Role' - logoutUrl: 'http://kc.localhost:8080/realms/ror/protocol/saml' - YOU_SHOULD_READ_ME_IN_STRATEGY_CONFIGURATION_LOG: 'unknown conf params should be passed unmodified to the underlying passport-saml library' - cert: 'MIICrDCCAZQCCQDN5Tcc+Rn6rTANBgkqhkiG9w0BAQsFADAYMRYwFAYDVQQDDA1rZXljbG9hay1zYW1sMB4XDTI1MTExMDA0MjQyOFoXDTM1MTEwODA0MjQyOFowGDEWMBQGA1UEAwwNa2V5Y2xvYWstc2FtbDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAJv0EFcTjNkpn5kV/XedCJ/AQKsPafZ7u33u3zfYgZbTh0V+CJ4bUAZyTfvGrcxR/iZy3hkYQBL7DGM6c/fYJGq1IH+/jxAk/GBY9tVnxotRsyhIMJYtZCb+DBUVX+wLowv2kZPlf/6OibjVpy+I6klQxIU8aeMkd+M/Phl97l+pRUjTuQZvEaEtkVLcsG6gEUcNo2X0MpNFwT7UgpGZLGy8zSHGNu703tb8w0oCYTEj42WqLWYNm5NMqD/clbPRj8g+1qZHpSoIH+p/cqVfU+oAZd3R2Y8SGR2OhYwyu9wHRnuqgiEgCIYbzwyh3IzX+57R8MQxHnjBFnzJKFtBWHMCAwEAATANBgkqhkiG9w0BAQsFAAOCAQEAUGrj7hEEHJvm58RYTjqhcJ38MVkgEmAnGovObxvS9Xi6ZBct4irWdpyESowRJtY4nxFGs0uiArOpQDkteSeDzJs/IkSg3xTx3UwOpevf8IV5gU8Bwq68Fyh9An8NvczE5XhLZD1Tacphlz5OzoXthMsS1pcCumr5ZELwMtvLYAWkEV/cKmlis4JuMEZXc5v9KVybsmHv1hgM+fxg1neuWrPK1JebuVm4oaHUHYKCxqn9MXjnDvOq6MkYtKBfcYf6BKk29lapYuHNiRTi0hMPD1tWVaZg3H1/uMxlLXDxAZqkS4DNq/7MeIMUWemOqxQHZAi4rtplrVl/F3WES6pqWw==' - oidc_keycloak: - buttonName: "Keycloak OIDC" - type: "oidc" - protocol: "https" - issuer: 'http://kc.localhost:8080/realms/ror' - authorizationURL: 'http://kc.localhost:8080/realms/ror/protocol/openid-connect/auth' - tokenURL: 'http://kc.localhost:8080/realms/ror/protocol/openid-connect/token' - userInfoURL: 'http://kc.localhost:8080/realms/ror/protocol/openid-connect/userinfo' - jwksURL: 'http://kc.localhost:8080/realms/ror/protocol/openid-connect/certs' - clientID: 'ror-oidc' - clientSecret: 'kibanasecret123' - scope: 'openid profile email' - usernameParameter: 'preferred_username' - groupsParameter: 'groups' - kibanaExternalHost: 'localhost:15601' - logoutUrl: 'http://kc.localhost:8080/realms/ror/protocol/openid-connect/logout' - oidc_lemon_ldap: - buttonName: "LemonLDAP OpenID" - type: "oidc" - protocol: "https" - issuer: 'https://oidctest.wsweet.org/' - authorizationURL: 'https://oidctest.wsweet.org/oauth2/authorize' - tokenURL: 'https://oidctest.wsweet.org/oauth2/token' - userInfoURL: 'https://oidctest.wsweet.org/oauth2/userinfo' - clientID: 'private' - clientSecret: 'tardis' - scope: 'openid users roles' - usernameParameter: 'sub' - groupsParameter: 'roles' - kibanaExternalHost: 'localhost:15601' - logoutUrl: 'https://oidctest.wsweet.org/oauth2/logout' - jwksURL: 'https://oidctest.wsweet.org/oauth2/jwks' +xpack.encryptedSavedObjects: + encryptionKey: "min-32-byte-long-strong-encryption-key" + +logging: + root: + level: debug + loggers: + - name: elasticsearch + level: trace + +readonlyrest_kbn.logLevel: trace +readonlyrest_kbn.cookiePass: '12312313123213123213123abcdefghijklm' \ No newline at end of file diff --git a/ror-demo-cluster/docker-compose.yml b/ror-demo-cluster/docker-compose.yml index 4da7e38..4d65185 100644 --- a/ror-demo-cluster/docker-compose.yml +++ b/ror-demo-cluster/docker-compose.yml @@ -1,27 +1,4 @@ services: - # Enterprise-only service - keycloak: - image: quay.io/keycloak/keycloak:20.0.5 - profiles: ["ENT"] - environment: - - KEYCLOAK_ADMIN=admin - - KEYCLOAK_ADMIN_PASSWORD=admin - - KEYCLOAK_FRONTEND_URL=http://kc.localhost:8080 - volumes: - - ./conf/keycloak/ror-realm.json:/opt/keycloak/data/import/ror-realm.json:ro - command: ["start-dev", "--import-realm", "--hostname=kc.localhost", "--http-enabled=true", "--http-port=8080"] - ports: - - "8080:8080" - healthcheck: - test: ["CMD-SHELL", "curl -fksS --connect-timeout 3 --max-time 5 --retry 5 --retry-connrefused http://127.0.0.1:8080/realms/ror/.well-known/openid-configuration >/dev/null || exit 1"] - interval: 10s - timeout: 10s - retries: 30 - start_period: 40s - networks: - es-ror-network: - aliases: - - kc.localhost es-ror: build: @@ -55,6 +32,15 @@ services: soft: -1 hard: -1 + es-ror-proxy: + image: mitmproxy/mitmproxy + command: + mitmdump --set validate_inbound_headers=false --mode reverse:https://es-ror:9200 --ssl-insecure --verbose + depends_on: + - es-ror + networks: + - es-ror-network + kbn-ror: build: context: . @@ -67,13 +53,10 @@ services: depends_on: es-ror: condition: service_healthy - keycloak: - condition: service_healthy - required: false ports: - "15601:5601" environment: - ELASTICSEARCH_HOSTS: https://es-ror:9200 + ELASTICSEARCH_HOSTS: https://es-ror-proxy:8080 ROR_ACTIVATION_KEY: $ROR_ACTIVATION_KEY healthcheck: test: ["CMD-SHELL", "curl -fksS --connect-timeout 3 --max-time 5 --retry 2 --retry-connrefused -u admin:admin https://127.0.0.1:5601/api/features >/dev/null || exit 1"] diff --git a/ror-demo-cluster/readonlyrest-1.69.0-pre7_es8.19.0.zip b/ror-demo-cluster/readonlyrest-1.69.0-pre7_es8.19.0.zip new file mode 100644 index 0000000..1431451 Binary files /dev/null and b/ror-demo-cluster/readonlyrest-1.69.0-pre7_es8.19.0.zip differ diff --git a/ror-demo-cluster/readonlyrest_kbn_universal-1.69.0-pre6_es8.19.0.zip b/ror-demo-cluster/readonlyrest_kbn_universal-1.69.0-pre6_es8.19.0.zip new file mode 100644 index 0000000..5dd7fed Binary files /dev/null and b/ror-demo-cluster/readonlyrest_kbn_universal-1.69.0-pre6_es8.19.0.zip differ diff --git a/shared/init-scripts/init.sh b/shared/init-scripts/init.sh index 70164aa..116cd67 100755 --- a/shared/init-scripts/init.sh +++ b/shared/init-scripts/init.sh @@ -4,37 +4,10 @@ cd "$(dirname "$0")" source utils/lib.sh -function generate_log_documents() { - if [ "$#" -ne 1 ]; then - echo "ERROR: One required: 1) number of documents to generate" - return 1 - fi +#createDataStream "frontend_logs_ds" && generate_log_documents 100 | putDocument "frontend_logs_ds" +#createDataStream "business_logs_ds" && generate_log_documents 50 | putDocument "business_logs_ds" +#createDataStream "system_logs_ds" && generate_log_documents 60 | putDocument "system_logs_ds" - N=$1 - - for ((i = 1; i <= N; i++)); do - user_id=$((RANDOM % 10000 + 1)) - timestamp=$(date -u +"%Y-%m-%dT%H:%M:%SZ") - log_message="User $user_id login successful" - level="$(pick_randomly "INFO" "WARN" "ERROR" "DEBUG")" - - echo "{ \"message\": \"$log_message\", \"level\": \"$level\", \"timestamp\": \"$timestamp\", \"user_id\": \"$user_id\" }" - done -} - -function index_documents() { - if [ "$#" -ne 1 ]; then - echo "ERROR: One required: 1) index name" - return 1 - fi - - INDEX_NAME=$1 - - while IFS= read -r document; do - putDocument "$INDEX_NAME" "$document" - done -} - -generate_log_documents 100 | index_documents "frontend_logs" -generate_log_documents 50 | index_documents "business_logs" -generate_log_documents 60 | index_documents "system_logs" \ No newline at end of file +#createIndex "frontend_logs_index" && generate_log_documents 100 | putDocument "frontend_logs_index" +#createIndex "business_logs_index" && generate_log_documents 50 | putDocument "business_logs_index" +#createIndex "system_logs_index" && generate_log_documents 60 | putDocument "system_logs_index" diff --git a/shared/init-scripts/utils/lib.sh b/shared/init-scripts/utils/lib.sh index fd0e7b8..be08b88 100755 --- a/shared/init-scripts/utils/lib.sh +++ b/shared/init-scripts/utils/lib.sh @@ -7,9 +7,92 @@ function pick_randomly() { echo "${OPTIONS[$RANDOM_INDEX]}" } +function createIndex() { + if [ "$#" -ne 1 ]; then + echo "ERROR: One parameter required: 1) index name" + return 1 + fi + + if ! [ -v ELASTICSEARCH_ADDRESS ] || [ -z "$ELASTICSEARCH_ADDRESS" ]; then + echo "ERROR: required variable ELASTICSEARCH_ADDRESS not set or empty" + exit 2 + fi + + if ! [ -v ELASTICSEARCH_USER ] || [ -z "$ELASTICSEARCH_USER" ]; then + echo "ERROR: required variable ELASTICSEARCH_USER not set or empty" + exit 3 + fi + + if ! [ -v ELASTICSEARCH_PASSWORD ] || [ -z "$ELASTICSEARCH_PASSWORD" ]; then + echo "ERROR: required variable ELASTICSEARCH_PASSWORD not set or empty" + exit 4 + fi + + INDEX_NAME=$1 + + response=$(curl -k -s -L -w "\n%{http_code}" -u $ELASTICSEARCH_USER:$ELASTICSEARCH_PASSWORD \ + -X PUT "$ELASTICSEARCH_ADDRESS/$INDEX_NAME" \ + -H "Content-Type: application/json" + ) + + http_status=$(echo "$response" | tail -n 1) + response_body=$(echo "$response" | sed \$d) + + if [[ "$http_status" != 2* ]]; then + echo "ERROR: Cannot create index [$INDEX_NAME]. HTTP status: $http_status, response body: $response_body" + return 5 + fi + + return 0 +} + +function createDataStream() { + if [ "$#" -ne 1 ]; then + echo "ERROR: One parameter required: 1) data stream name" + return 1 + fi + + if ! [ -v ELASTICSEARCH_ADDRESS ] || [ -z "$ELASTICSEARCH_ADDRESS" ]; then + echo "ERROR: required variable ELASTICSEARCH_ADDRESS not set or empty" + exit 2 + fi + + if ! [ -v ELASTICSEARCH_USER ] || [ -z "$ELASTICSEARCH_USER" ]; then + echo "ERROR: required variable ELASTICSEARCH_USER not set or empty" + exit 3 + fi + + if ! [ -v ELASTICSEARCH_PASSWORD ] || [ -z "$ELASTICSEARCH_PASSWORD" ]; then + echo "ERROR: required variable ELASTICSEARCH_PASSWORD not set or empty" + exit 4 + fi + + STREAM_NAME=$1 + TEMPLATE_NAME="${STREAM_NAME}-template" + + response=$(curl -k -s -L -w "\n%{http_code}" -u $ELASTICSEARCH_USER:$ELASTICSEARCH_PASSWORD \ + -X PUT "$ELASTICSEARCH_ADDRESS/_index_template/$TEMPLATE_NAME" \ + -H "Content-Type: application/json" -d "{ + \"index_patterns\": [\"$STREAM_NAME\"], + \"data_stream\": {}, + \"priority\": 500 + }" + ) + + http_status=$(echo "$response" | tail -n 1) + response_body=$(echo "$response" | sed \$d) + + if [[ "$http_status" != 2* ]]; then + echo "ERROR: Cannot create index template for data stream [$STREAM_NAME]. HTTP status: $http_status, response body: $response_body" + return 5 + fi + + return 0 +} + function putDocument() { - if [ "$#" -ne 2 ]; then - echo "ERROR: Three parameters required: 1) index name, 2) document JSON string" + if [ "$#" -lt 1 ] || [ "$#" -gt 2 ]; then + echo "ERROR: Required: 1) index name, optionally 2) document JSON string (or via stdin)" return 1 fi @@ -29,22 +112,50 @@ function putDocument() { fi INDEX_NAME=$1 - DOCUMENT_CONTENT=$2 -set -x + if [ "$#" -eq 2 ]; then + putSingleDocument "$INDEX_NAME" "$2" + else + while IFS= read -r DOCUMENT_CONTENT; do + putSingleDocument "$INDEX_NAME" "$DOCUMENT_CONTENT" + done + fi +} - respone=$(curl -k -s -L -w "\n%{http_code}" -u $ELASTICSEARCH_USER:$ELASTICSEARCH_PASSWORD \ +function putSingleDocument() { + INDEX_NAME=$1 + DOCUMENT_CONTENT=$2 + + response=$(curl -k -s -L -w "\n%{http_code}" -u $ELASTICSEARCH_USER:$ELASTICSEARCH_PASSWORD \ -X POST "$ELASTICSEARCH_ADDRESS/$INDEX_NAME/_doc/" \ -H "Content-Type: application/json" -d "$DOCUMENT_CONTENT" ) - http_status=$(echo "$respone" | tail -n 1) - response_body=$(echo "$respone" | sed \$d) + http_status=$(echo "$response" | tail -n 1) + response_body=$(echo "$response" | sed \$d) if [[ "$http_status" != 2* ]] ; then - echo "ERROR: Cannot add document [$DOCUMENT_CONTENT] to index=[$INDEX_NAME].\nHTTP status: $HTTP_STATUS, response body: $RESPONSE_BODY" + echo "ERROR: Cannot add document [$DOCUMENT_CONTENT] to index=[$INDEX_NAME].\nHTTP status: $http_status, response body: $response_body" return 5 fi return 0 } + +function generate_log_documents() { + if [ "$#" -ne 1 ]; then + echo "ERROR: One required: 1) number of documents to generate" + return 1 + fi + + N=$1 + + for ((i = 1; i <= N; i++)); do + user_id=$((RANDOM % 10000 + 1)) + timestamp=$(date -u +"%Y-%m-%dT%H:%M:%SZ") + log_message="User $user_id login successful" + level="$(pick_randomly "INFO" "WARN" "ERROR" "DEBUG")" + + echo "{ \"message\": \"$log_message\", \"level\": \"$level\", \"@timestamp\": \"$timestamp\", \"user_id\": \"$user_id\" }" + done +} \ No newline at end of file diff --git a/xpack-docker-demo-cluster/docker-compose.yml b/xpack-docker-demo-cluster/docker-compose.yml index e89e548..66512f9 100644 --- a/xpack-docker-demo-cluster/docker-compose.yml +++ b/xpack-docker-demo-cluster/docker-compose.yml @@ -9,7 +9,7 @@ services: ports: - "29200:9200" - "29300:9300" - - "5005:5005" + - "5006:5005" environment: - cluster.name=xpack-es-cluster - node.name=es-xpack-single @@ -30,6 +30,15 @@ services: soft: -1 hard: -1 + es-xpack-proxy: + image: mitmproxy/mitmproxy + command: + mitmdump --set validate_inbound_headers=false --mode reverse:https://es-xpack:9200 --ssl-insecure --verbose + depends_on: + - es-xpack + networks: + - es-xpack-network + kbn-xpack: build: context: . @@ -42,7 +51,7 @@ services: ports: - "25601:5601" environment: - ES_API_URL: https://es-xpack:9200 + ES_API_URL: https://es-xpack-proxy:8080 networks: - es-xpack-network ulimits: