From 08a01529645d119c058f5c5fffa3b74093e68b94 Mon Sep 17 00:00:00 2001 From: Srid Banoor Date: Fri, 22 Sep 2017 16:31:45 -0700 Subject: [PATCH 1/6] DSE topology with Kerberos and non Kerberos support --- LICENSE => LICENSE.txt | 0 README.rst | 110 +++++++++++++ __init__.py | 12 ++ images/centos6.8/Dockerfile | 51 ++++++ images/centos6.8/dse.repo | 5 + start.py | 318 ++++++++++++++++++++++++++++++++++++ topology.yaml | 38 +++++ 7 files changed, 534 insertions(+) rename LICENSE => LICENSE.txt (100%) create mode 100644 README.rst create mode 100644 __init__.py create mode 100644 images/centos6.8/Dockerfile create mode 100644 images/centos6.8/dse.repo create mode 100644 start.py create mode 100644 topology.yaml diff --git a/LICENSE b/LICENSE.txt similarity index 100% rename from LICENSE rename to LICENSE.txt diff --git a/README.rst b/README.rst new file mode 100644 index 0000000..1839a07 --- /dev/null +++ b/README.rst @@ -0,0 +1,110 @@ +====================================== +DSE Cassandra topology for clusterdock +====================================== + +This repository houses the **DSE Cassandra** topology for `clusterdock`_. + +.. _clusterdock: https://github.com/clusterdock/clusterdock + +Usage +===== + +Assuming you've already installed **clusterdock** (if not, go `read the docs`_), +you use this topology by cloning it to a local folder and then running commands +with the ``clusterdock`` script: + +.. _read the docs: http://clusterdock.readthedocs.io/en/latest/ + +.. code-block:: console + + $ git clone https://github.com/clusterdock/topology_dse.git + $ clusterdock start topology_dse + 2017-09-20 10:17:38 PM clusterdock.models INFO Starting cluster on network (cluster) ... + 2017-09-20 10:17:38 PM clusterdock.models INFO Starting node node-1.cluster ... + 2017-09-20 10:17:39 PM clusterdock.models INFO Starting node node-2.cluster ... + 2017-09-20 10:17:40 PM clusterdock.topology_dse.start INFO Updating DSE configurations and starting DSE nodes ... + Restarting DSE daemon : dse + DSE daemon starting with only Cassandra enabled (edit /etc/default/dse to enable other features) + Restarting DSE daemon : dse + DSE daemon starting with only Cassandra enabled (edit /etc/default/dse to enable other features) + 2017-09-20 10:18:40 PM clusterdock.topology_dse.start INFO Validating DSE service health ... + 2017-09-20 10:18:53 PM clusterdock.topology_dse.start INFO DSE cluster is available and its contacts are: node-1.cluster,node-2.cluster + 2017-09-20 10:18:53 PM clusterdock.topology_dse.start INFO From its node, DSE can be accessed with: cqlsh -u cassandra -p cassandra + 2017-09-20 10:18:53 PM clusterdock.cli INFO Cluster started successfully (total time: 1m 14s). + +To start a Kerberos based DSE cluster: + +.. code-block:: console + + $ clusterdock start --kerberos topology_dse + 2017-09-20 10:21:54 PM clusterdock.models INFO Starting cluster on network (cluster) ... + 2017-09-20 10:21:54 PM clusterdock.models INFO Starting node kdc.cluster ... + 2017-09-20 10:21:56 PM clusterdock.models INFO Starting node node-1.cluster ... + 2017-09-20 10:21:57 PM clusterdock.models INFO Starting node node-2.cluster ... + 2017-09-20 10:21:57 PM clusterdock.topology_dse.start INFO Updating KDC configurations ... + 2017-09-20 10:21:59 PM clusterdock.topology_dse.start INFO Starting KDC ... + 2017-09-20 10:22:01 PM clusterdock.topology_dse.start INFO Validating KDC service health ... + 2017-09-20 10:22:01 PM clusterdock.topology_dse.start INFO Creating `dse` and `HTTP` Kerberos principals for DSE nodes ... + 2017-09-20 10:22:02 PM clusterdock.topology_dse.start INFO Updating DSE configurations and starting DSE nodes ... + Restarting DSE daemon : dse + DSE daemon starting with only Cassandra enabled (edit /etc/default/dse to enable other features) + Restarting DSE daemon : dse + DSE daemon starting with only Cassandra enabled (edit /etc/default/dse to enable other features) + 2017-09-20 10:23:02 PM clusterdock.topology_dse.start INFO Validating DSE service health ... + 2017-09-20 10:23:15 PM clusterdock.topology_dse.start INFO DSE cluster is available and its contacts are: node-1.cluster,node-2.cluster + 2017-09-20 10:23:15 PM clusterdock.topology_dse.start INFO From its node, DSE can be accessed with: cqlsh -u cassandra -p cassandra + 2017-09-20 10:23:15 PM clusterdock.cli INFO Cluster started successfully (total time: 1m 20s). + +To see full usage instructions for the ``start`` action, use ``-h``/``--help``: + +.. code-block:: console + + $ clusterdock start topology_dse -h + usage: clusterdock start [--always-pull] [--namespace ns] [--network nw] + [-o sys] [-r url] [-h] + [--dse-cluster-name DSE_CLUSTER_NAME] [--kerberos] + [--kerberos-config-directory path] + [--kerberos-principals princ1,princ2,...] + [--kdc-node node [node ...]] + [--nodes node [node ...]] + topology + + Start a DSE cluster + + positional arguments: + topology A clusterdock topology directory + + optional arguments: + --always-pull Pull latest images, even if they're available locally + (default: False) + --namespace ns Namespace to use when looking for images (default: + None) + --network nw Docker network to use (default: cluster) + -o sys, --operating-system sys + Operating system to use for cluster nodes (default: + None) + -r url, --registry url + Docker Registry from which to pull images (default: + docker.io) + -h, --help show this help message and exit + + DSE arguments: + --dse-cluster-name DSE_CLUSTER_NAME + DSE cluster name to use. (default: Test Cluster) + --kerberos If specified, sets up Kerberos based DSE cluster with + a KDC node. (default: False) + --kerberos-config-directory path + If specified, mounts this directory to KDC container + for Kerberos config files. (default: + ~/.clusterdock/kerberos) + --kerberos-principals princ1,princ2,... + If specified, a comma-separated list of Kerberos user + principals to create in KDC. (default: None) + + Node groups: + --kdc-node node [node ...] + Nodes of the kdc-node group (default: ['kdc']) + --nodes node [node ...] + Nodes of the nodes group (default: ['node-1', + 'node-2']) + 'node-2']) diff --git a/__init__.py b/__init__.py new file mode 100644 index 0000000..a48d6b9 --- /dev/null +++ b/__init__.py @@ -0,0 +1,12 @@ +# -*- coding: utf-8 -*- +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/images/centos6.8/Dockerfile b/images/centos6.8/Dockerfile new file mode 100644 index 0000000..82d3142 --- /dev/null +++ b/images/centos6.8/Dockerfile @@ -0,0 +1,51 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +FROM clusterdock/topology_nodebase:centos6.8 + +ADD dse.repo /etc/yum.repos.d/ + +ARG JDK_URL=http://download.oracle.com/otn-pub/java/jdk/8u131-b11/d54c1d3a095b4ff2b6607d096fa80163/jdk-8u131-linux-x64.tar.gz +ARG JCE_URL=http://download.oracle.com/otn-pub/java/jce/8/jce_policy-8.zip +ARG JDK_DIRECTORY=/usr/java/jdk1.8.0_131 + +RUN yum -y update && \ + yum -y install unzip + +# Java setup +RUN wget --no-check-certificate \ + --header "Cookie: oraclelicense=accept-securebackup-cookie" -O /tmp/jdk.tar.gz \ + "${JDK_URL}" && \ + mkdir -p /usr/java && \ + tar xf /tmp/jdk.tar.gz -C /usr/java && \ + rm -f /tmp/jdk.tar.gz + +RUN wget --no-check-certificate \ + --no-cookies --header "Cookie: oraclelicense=accept-securebackup-cookie" -O /tmp/unlimited_jce_policy.zip \ + "${JCE_URL}" && \ + unzip -jo -d ${JDK_DIRECTORY}/jre/lib/security /tmp/unlimited_jce_policy.zip && \ + rm -f /tmp/unlimited_jce_policy.zip + +RUN update-alternatives --install "/usr/bin/java" "java" "${JDK_DIRECTORY}/bin/java" 1 +RUN update-alternatives --set java ${JDK_DIRECTORY}/bin/java + +# DSE setup +RUN yum -y install dse-full + +# Add Python 2.7 as needed by DSE's cqlsh tool +RUN yum install -y scl-utils centos-release-scl-rh +RUN yum install -y python27 + +# Add Kerberos client specific packages +RUN yum install -y krb5-workstation krb5-devel gcc python26-kerberos +RUN . /opt/rh/python27/enable && pip install pure-sasl kerberos + +CMD ["/sbin/init"] diff --git a/images/centos6.8/dse.repo b/images/centos6.8/dse.repo new file mode 100644 index 0000000..95e9781 --- /dev/null +++ b/images/centos6.8/dse.repo @@ -0,0 +1,5 @@ +[datastax] +name = DataStax Repo for DataStax Enterprise +baseurl=https://srid%40streamsets.com:Streamsetsme3@rpm.datastax.com/enterprise +enabled=1 +gpgcheck=0 diff --git a/start.py b/start.py new file mode 100644 index 0000000..9be60c6 --- /dev/null +++ b/start.py @@ -0,0 +1,318 @@ +# -*- coding: utf-8 -*- +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import re +import textwrap +from os.path import expanduser + +import yaml + +from clusterdock.models import Cluster, Node +from clusterdock.utils import wait_for_condition + +logger = logging.getLogger('clusterdock.{}'.format(__name__)) + +DEFAULT_OPERATING_SYSTEM = 'centos6.8' + +KDC_ACL_FILEPATH = '/var/kerberos/krb5kdc/kadm5.acl' +KDC_CONF_FILEPATH = '/var/kerberos/krb5kdc/kdc.conf' +KERBEROS_VOLUME_DIR = '/etc/clusterdock/kerberos' +KDC_KEYTAB_FILENAME = 'clusterdock.keytab' +KDC_USER_KEYTAB_FILEPATH = '{}/{}'.format(KERBEROS_VOLUME_DIR, KDC_KEYTAB_FILENAME) +KDC_KRB5_CONF_FILEPATH = '/etc/krb5.conf' + +DSE_HOME_DIR = '/etc/dse' +DSE_CONF_FILENAME = 'dse.yaml' +DSE_CONF_FILEPATH = '{}/{}'.format(DSE_HOME_DIR, DSE_CONF_FILENAME) +DSE_CASSANDRA_CONF_FILENAME = 'cassandra.yaml' +DSE_CASSANDRA_CONF_FILEPATH = '{}/cassandra/{}'.format(DSE_HOME_DIR, DSE_CASSANDRA_CONF_FILENAME) +DSE_CQLSH_FILEPATH = '/usr/bin/cqlsh' +DSE_CQLSHRC_HOME_DIR = '/root/.cassandra' +DSE_CQLSHRC_FILEPATH = '{}/cqlshrc'.format(DSE_CQLSHRC_HOME_DIR) +DSE_KEYTAB_FILEPATH = '{}/dse.keytab'.format(DSE_HOME_DIR) +DSE_USER_KEYTAB_FILEPATH = '{}/{}'.format(DSE_HOME_DIR, KDC_KEYTAB_FILENAME) + + +def main(args): + global quiet_logging + quiet_logging = False if args.verbose else True + + dse_image = '{}/{}/topology_dse:{}'.format(args.registry, args.namespace, + args.operating_system or DEFAULT_OPERATING_SYSTEM) + if args.kerberos: + _setup_kerberos_nodes(args, dse_image) + else: + _setup_non_kerberos_nodes(args, dse_image) + + +def _setup_non_kerberos_nodes(args, dse_image): + nodes = [Node(hostname=hostname, group='nodes', image=dse_image) for hostname in args.nodes] + cluster = Cluster(*nodes) + cluster.start(args.network) + + # DSE node logic + logger.info('Updating DSE configurations and starting DSE nodes ...') + cluster_name = args.dse_cluster_name + cluster_seeds = ','.join(node.ip_address for node in nodes) + for node in nodes: + # DSE config specific commands + dse_config_commands = [ + 'cp {} {}.org'.format(DSE_CASSANDRA_CONF_FILEPATH, DSE_CASSANDRA_CONF_FILEPATH), + 'cp {} {}.org'.format(DSE_CONF_FILEPATH, DSE_CONF_FILEPATH), + 'mkdir -p {}'.format(DSE_CQLSHRC_HOME_DIR) + ] + node.execute(command="bash -c '{}'".format('; '.join(dse_config_commands)), quiet=quiet_logging) + # DSE cassandra.yaml mods + cassandra_config_data = yaml.load(node.get_file(DSE_CASSANDRA_CONF_FILEPATH)) + cassandra_config_data['cluster_name'] = cluster_name + cassandra_config_data['listen_address'] = node.ip_address + cassandra_config_data['rpc_address'] = node.ip_address + cassandra_config_data['seed_provider'][0]['parameters'][0]['seeds'] = cluster_seeds + node.put_file(DSE_CASSANDRA_CONF_FILEPATH, yaml.dump(cassandra_config_data)) + # DSE dse.yaml mods + dse_config_data = yaml.load(node.get_file(DSE_CONF_FILEPATH)) + dse_config_data['audit_logging_options']['enabled'] = True + dse_config_data['authentication_options'] = {'enabled': True, 'default_scheme': 'internal'} + node.put_file(DSE_CONF_FILEPATH, yaml.dump(dse_config_data)) + # DSE cqlsh specific commands + cqlsh_cmd_data = node.get_file(DSE_CQLSH_FILEPATH) + node.put_file(DSE_CQLSH_FILEPATH, re.sub(r'.*(bash code here).*', '. /opt/rh/python27/enable', cqlsh_cmd_data)) + node.execute(command='chmod +x {}'.format(DSE_CQLSH_FILEPATH), quiet=quiet_logging) + cqlshrc_data = """ + [connection] + hostname = {} + port = 9042 + """.format(node.fqdn) + node.put_file(DSE_CQLSHRC_FILEPATH, textwrap.dedent(cqlshrc_data)) + # start DSE on the node + node.execute('service dse restart') + + logger.info('Validating DSE service health ...') + cqlsh_cmd = "cqlsh -u cassandra -p cassandra {} --debug -e 'DESCRIBE KEYSPACES'".format(nodes[0].fqdn) + _validate_dse_health(nodes=nodes, node_cmd=cqlsh_cmd, node_cmd_expected='system_schema') + + logger.info('DSE cluster is available and its contacts are: {}'.format(','.join(node.fqdn for node in nodes))) + logger.info('From its node, DSE can be accessed with: cqlsh -u cassandra -p cassandra') + + +def _setup_kerberos_nodes(args, dse_image): + kerberos_volume_dir = args.kerberos_config_directory.replace('~', expanduser('~')) + + nodes = [Node(hostname=hostname, group='nodes', image=dse_image, + volumes=[{kerberos_volume_dir: KERBEROS_VOLUME_DIR}]) for hostname in args.nodes] + + kdc_image = '{}/{}/topology_nodebase_kerberos:{}'.format(args.registry, args.namespace, + args.operating_system or DEFAULT_OPERATING_SYSTEM) + kdc_hostname = args.kdc_node[0] + kdc_node = Node(hostname=kdc_hostname, group='kdc', image=kdc_image, + volumes=[{kerberos_volume_dir: KERBEROS_VOLUME_DIR}]) + cluster = Cluster(kdc_node, *nodes) + cluster.start(args.network) + + logger.info('Updating KDC configurations ...') + realm = cluster.network.upper() + # Update configurations + krb5_conf_data = kdc_node.get_file(KDC_KRB5_CONF_FILEPATH) + kdc_node.put_file(KDC_KRB5_CONF_FILEPATH, + re.sub(r'EXAMPLE.COM', realm, + re.sub(r'example.com', cluster.network, + re.sub(r'kerberos.example.com', r'{}.{}'.format(kdc_hostname, cluster.network), + krb5_conf_data)))) + kdc_conf_data = kdc_node.get_file(KDC_CONF_FILEPATH) + kdc_node.put_file(KDC_CONF_FILEPATH, + re.sub(r'EXAMPLE.COM', realm, + kdc_conf_data.replace(r'[kdcdefaults]', + '[kdcdefaults]\n max_renewablelife = 7d\n max_life = 1d'))) + acl_data = kdc_node.get_file(KDC_ACL_FILEPATH) + kdc_node.put_file(KDC_ACL_FILEPATH, re.sub(r'EXAMPLE.COM', realm, acl_data)) + + kdc_commands = [ + 'kdb5_util create -s -r {realm} -P kdcadmin'.format(realm=realm), + 'kadmin.local -q "addprinc -pw {adminpw} admin/admin@{realm}"'.format(adminpw='acladmin', realm=realm) + ] + + logger.info('Starting KDC ...') + # Add the following commands before starting kadmin daemon etc. + if args.kerberos_principals: + principal_list = ['{}@{}'.format(primary, realm) for primary in args.kerberos_principals.split(',')] + create_principals_cmds = ['kadmin.local -q "addprinc -randkey {}"'.format(principal) + for principal in principal_list] + kdc_commands.extend(create_principals_cmds) + + kdc_commands.append('rm -f {}'.format(KDC_USER_KEYTAB_FILEPATH)) + create_keytab_cmd = 'kadmin.local -q "xst -norandkey -k {} {}" '.format(KDC_USER_KEYTAB_FILEPATH, + ' '.join(principal_list)) + kdc_commands.append(create_keytab_cmd) + + kdc_commands.extend([ + 'krb5kdc', + 'kadmind', + 'authconfig --enablekrb5 --update', + 'service sshd start', + 'service krb5kdc start', + 'service kadmin start' + ]) + + # Gather keytab file and krb5.conf file in KERBEROS_VOLUME_DIR directory which is mounted on host. + kdc_commands.append('cp {} {}'.format(KDC_KRB5_CONF_FILEPATH, KERBEROS_VOLUME_DIR)) + if args.kerberos_principals: + kdc_commands.append('chmod 644 {}'.format(KDC_USER_KEYTAB_FILEPATH)) + + kdc_node.execute(command="bash -c '{}'".format('; '.join(kdc_commands)), quiet=quiet_logging) + + logger.info('Validating KDC service health ...') + _validate_kdc_health(node=kdc_node, services=['sshd', 'krb5kdc', 'kadmin']) + + # Add DSE specific logic on KDC host + logger.info('Creating `dse` and `HTTP` Kerberos principals for DSE nodes ...') + for node in nodes: + key_tab_filename = '{}/{}.keytab'.format(KERBEROS_VOLUME_DIR, node.fqdn) + kdc_dse_commands = [ + 'rm -f {}/{}.keytab'.format(KERBEROS_VOLUME_DIR, node.fqdn), + 'kadmin.local -q "addprinc -randkey dse/{}"'.format(node.fqdn), + 'kadmin.local -q "addprinc -randkey HTTP/{}"'.format(node.fqdn), + 'kadmin.local -q "ktadd -k {} dse/{}"'.format(key_tab_filename, node.fqdn), + 'kadmin.local -q "ktadd -k {} HTTP/{}"'.format(key_tab_filename, node.fqdn), + 'chmod 644 {}'.format(key_tab_filename) + ] + kdc_node.execute(command="bash -c '{}'".format('; '.join(kdc_dse_commands)), quiet=quiet_logging) + + # DSE node logic + logger.info('Updating DSE configurations and starting DSE nodes ...') + cluster_name = args.dse_cluster_name + cluster_seeds = ','.join(node.ip_address for node in nodes) + for node in nodes: + # kerberos specific commands + dse_kdc_commands = [ + 'cp {}/krb5.conf /etc/.'.format(KERBEROS_VOLUME_DIR), + 'cp {}/{}.keytab {}'.format(KERBEROS_VOLUME_DIR, node.fqdn, DSE_KEYTAB_FILEPATH), + 'chown cassandra:cassandra {}'.format(DSE_KEYTAB_FILEPATH), + 'chmod 600 {}'.format(DSE_KEYTAB_FILEPATH) + ] + if args.kerberos_principals: + dse_kdc_commands.extend([ + 'cp {} {}/.'.format(KDC_USER_KEYTAB_FILEPATH, DSE_HOME_DIR), + 'chown cassandra:cassandra {}'.format(DSE_USER_KEYTAB_FILEPATH), + 'chmod 600 {}'.format(DSE_USER_KEYTAB_FILEPATH) + ]) + node.execute(command="bash -c '{}'".format('; '.join(dse_kdc_commands)), quiet=quiet_logging) + # DSE config specific commands + dse_config_commands = [ + 'cp {} {}.org'.format(DSE_CASSANDRA_CONF_FILEPATH, DSE_CASSANDRA_CONF_FILEPATH), + 'cp {} {}.org'.format(DSE_CONF_FILEPATH, DSE_CONF_FILEPATH), + 'mkdir -p {}'.format(DSE_CQLSHRC_HOME_DIR) + ] + node.execute(command="bash -c '{}'".format('; '.join(dse_config_commands)), quiet=quiet_logging) + # DSE cassandra.yaml mods + cassandra_config_data = yaml.load(node.get_file(DSE_CASSANDRA_CONF_FILEPATH)) + cassandra_config_data['cluster_name'] = cluster_name + cassandra_config_data['listen_address'] = node.ip_address + cassandra_config_data['rpc_address'] = node.ip_address + cassandra_config_data['seed_provider'][0]['parameters'][0]['seeds'] = cluster_seeds + node.put_file(DSE_CASSANDRA_CONF_FILEPATH, yaml.dump(cassandra_config_data)) + # DSE dse.yaml mods + dse_config_data = yaml.load(node.get_file(DSE_CONF_FILEPATH)) + dse_config_data['audit_logging_options']['enabled'] = True + dse_config_data['authentication_options'] = {'enabled': True, 'default_scheme': 'internal', + 'allow_digest_with_kerberos': False, + 'plain_text_without_ssl': 'warn', 'transitional_mode': 'disabled', + 'other_schemes': ['internal', 'kerberos'], + 'scheme_permissions': False} + dse_config_data['role_management_options'] = {'mode': 'internal'} + dse_config_data['authorization_options'] = {'enabled': True, 'transitional_mode': 'disabled', + 'allow_row_level_security': False} + dse_config_data['kerberos_options'] = {'keytab': DSE_KEYTAB_FILEPATH, + 'service_principal': 'dse/_HOST@{}'.format(realm), + 'http_principal': 'HTTP/_HOST@{}'.format(realm), + 'qop': 'auth'} + node.put_file(DSE_CONF_FILEPATH, yaml.dump(dse_config_data)) + # DSE cqlsh specific commands + cqlsh_cmd_data = node.get_file(DSE_CQLSH_FILEPATH) + node.put_file(DSE_CQLSH_FILEPATH, re.sub(r'.*(bash code here).*', '. /opt/rh/python27/enable', cqlsh_cmd_data)) + node.execute(command='chmod +x {}'.format(DSE_CQLSH_FILEPATH), quiet=quiet_logging) + cqlshrc_data = """ + [connection] + hostname = {} + port = 9042 + + [kerberos] + service = dse + qops = auth + """.format(node.fqdn) + node.put_file(DSE_CQLSHRC_FILEPATH, textwrap.dedent(cqlshrc_data)) + # start DSE on the node + node.execute('service dse restart') + + logger.info('Validating DSE service health ...') + cqlsh_cmd = "cqlsh -u cassandra -p cassandra {} --debug -e 'DESCRIBE KEYSPACES'".format(nodes[0].fqdn) + _validate_dse_health(nodes=nodes, node_cmd=cqlsh_cmd, node_cmd_expected='system_schema') + + if args.kerberos_principals: + principal_list = ['{}@{}'.format(primary, realm) for primary in args.kerberos_principals.split(',')] + logger.info('Creating DSE Kerberos roles {} ...'.format(principal_list)) + logger.info('Kerberos DSE keytab file available on the node at {}'.format(DSE_USER_KEYTAB_FILEPATH)) + for principal in principal_list: + cqlsh_cmd = """cqlsh -u cassandra -p cassandra {} --debug """.format(nodes[0].fqdn) + cqlsh_cmd += """-e 'CREATE ROLE "{}" WITH LOGIN = true;""".format(principal) + cqlsh_cmd += """GRANT EXECUTE on KERBEROS SCHEME to "{}";""".format(principal) + cqlsh_cmd += """GRANT ALL on ALL KEYSPACES to "{}";'""".format(principal) + nodes[0].execute(command=cqlsh_cmd, quiet=quiet_logging) + + logger.info('DSE cluster is available and its contacts are: {}'.format(','.join(node.fqdn for node in nodes))) + logger.info('From its node, DSE can be accessed with: cqlsh -u cassandra -p cassandra') + + +def _validate_kdc_health(node, services): + def condition(node, services): + if all('is running' in (node.execute(command='service {} status'.format(service), quiet=quiet_logging).output) + for service in services): + return True + else: + logger.debug('Services with poor health: %s', + ', '.join(service + for service in services + if 'is running' not in node.execute(command='service {} status'.format(service), + quiet=quiet_logging).output)) + + def success(time): + logger.debug('Validated service health in %s seconds.', time) + + def failure(timeout): + raise TimeoutError('Timed out after {} seconds waiting ' + 'to validate service health.'.format(timeout)) + wait_for_condition(condition=condition, condition_args=[node, services], + time_between_checks=3, timeout=600, success=success, failure=failure) + + +def _validate_dse_health(nodes, node_cmd, node_cmd_expected): + def condition(nodes, node_cmd, node_cmd_expected): + if all('running' in (node.execute(command='nodetool statusgossip', quiet=quiet_logging).output) + for node in nodes) and node_cmd_expected in nodes[0].execute(command=node_cmd, + quiet=quiet_logging).output: + return True + else: + logger.debug('Node with poor health: %s', + ', '.join(node.fqdn + for node in nodes + if 'running' not in node.execute(command='nodetool statusgossip', + quiet=quiet_logging).output)) + + def success(time): + logger.debug('Validated DSE health in %s seconds.', time) + + def failure(timeout): + raise TimeoutError('Timed out after {} seconds waiting ' + 'to validate DSE health.'.format(timeout)) + wait_for_condition(condition=condition, condition_args=[nodes, node_cmd, node_cmd_expected], + time_between_checks=3, timeout=600, success=success, failure=failure) diff --git a/topology.yaml b/topology.yaml new file mode 100644 index 0000000..05e0c55 --- /dev/null +++ b/topology.yaml @@ -0,0 +1,38 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# yaml definition file for the nodebase DSE topology. + +name: DSE +description: A Datastax (DSE) cluster + +node groups: + kdc-node: + - kdc + nodes: + - node-1 + - node-2 + +start args: + --dse-cluster-name: + default: Test Cluster + help: DSE cluster name to use. + --kerberos: + help: If specified, sets up Kerberos based DSE cluster with a KDC node. + action: store_true + --kerberos-config-directory: + default: ~/.clusterdock/kerberos + help: If specified, mounts this directory to KDC container for Kerberos config files. + metavar: path + --kerberos-principals: + help: If specified, a comma-separated list of Kerberos user principals to create in KDC. + metavar: princ1,princ2,... From f59e86b5efe7300acda8ff21986ad37a6428b689 Mon Sep 17 00:00:00 2001 From: Srid Banoor Date: Fri, 22 Sep 2017 16:31:45 -0700 Subject: [PATCH 2/6] DSE topology with Kerberos and non Kerberos support --- images/centos6.8/Dockerfile | 3 ++- start.py | 7 ++++--- topology.yaml | 10 ++++++++++ 3 files changed, 16 insertions(+), 4 deletions(-) diff --git a/images/centos6.8/Dockerfile b/images/centos6.8/Dockerfile index 82d3142..f668c35 100644 --- a/images/centos6.8/Dockerfile +++ b/images/centos6.8/Dockerfile @@ -16,6 +16,7 @@ ADD dse.repo /etc/yum.repos.d/ ARG JDK_URL=http://download.oracle.com/otn-pub/java/jdk/8u131-b11/d54c1d3a095b4ff2b6607d096fa80163/jdk-8u131-linux-x64.tar.gz ARG JCE_URL=http://download.oracle.com/otn-pub/java/jce/8/jce_policy-8.zip ARG JDK_DIRECTORY=/usr/java/jdk1.8.0_131 +ARG DSE_VERSION=5.1.3-1 RUN yum -y update && \ yum -y install unzip @@ -38,7 +39,7 @@ RUN update-alternatives --install "/usr/bin/java" "java" "${JDK_DIRECTORY}/bin/j RUN update-alternatives --set java ${JDK_DIRECTORY}/bin/java # DSE setup -RUN yum -y install dse-full +RUN yum -y install dse-full-${DSE_VERSION} # Add Python 2.7 as needed by DSE's cqlsh tool RUN yum install -y scl-utils centos-release-scl-rh diff --git a/start.py b/start.py index 9be60c6..b81fb0f 100644 --- a/start.py +++ b/start.py @@ -23,6 +23,7 @@ logger = logging.getLogger('clusterdock.{}'.format(__name__)) +DEFAULT_NAMESPACE = 'clusterdock' DEFAULT_OPERATING_SYSTEM = 'centos6.8' KDC_ACL_FILEPATH = '/var/kerberos/krb5kdc/kadm5.acl' @@ -47,9 +48,8 @@ def main(args): global quiet_logging quiet_logging = False if args.verbose else True + dse_image = '{}/{}/clusterdock:dse{}'.format(args.registry, args.namespace, args.dse_version) - dse_image = '{}/{}/topology_dse:{}'.format(args.registry, args.namespace, - args.operating_system or DEFAULT_OPERATING_SYSTEM) if args.kerberos: _setup_kerberos_nodes(args, dse_image) else: @@ -112,7 +112,7 @@ def _setup_kerberos_nodes(args, dse_image): nodes = [Node(hostname=hostname, group='nodes', image=dse_image, volumes=[{kerberos_volume_dir: KERBEROS_VOLUME_DIR}]) for hostname in args.nodes] - kdc_image = '{}/{}/topology_nodebase_kerberos:{}'.format(args.registry, args.namespace, + kdc_image = '{}/{}/topology_nodebase_kerberos:{}'.format(args.registry, args.namespace or DEFAULT_NAMESPACE, args.operating_system or DEFAULT_OPERATING_SYSTEM) kdc_hostname = args.kdc_node[0] kdc_node = Node(hostname=kdc_hostname, group='kdc', image=kdc_image, @@ -150,6 +150,7 @@ def _setup_kerberos_nodes(args, dse_image): for principal in principal_list] kdc_commands.extend(create_principals_cmds) + kdc_commands.append('sleep 2') # sleep few seconds to have Docker volume available kdc_commands.append('rm -f {}'.format(KDC_USER_KEYTAB_FILEPATH)) create_keytab_cmd = 'kadmin.local -q "xst -norandkey -k {} {}" '.format(KDC_USER_KEYTAB_FILEPATH, ' '.join(principal_list)) diff --git a/topology.yaml b/topology.yaml index 05e0c55..5f55253 100644 --- a/topology.yaml +++ b/topology.yaml @@ -22,7 +22,17 @@ node groups: - node-1 - node-2 +build args: + --dse-version: + default: 5.1.3-1 + help: DSE version to use. + metavar: ver + start args: + --dse-version: + default: 5.1.3-1 + help: DSE version to use. + metavar: ver --dse-cluster-name: default: Test Cluster help: DSE cluster name to use. From 818cb499357701bb5cdabed9437bc0697b6cc43f Mon Sep 17 00:00:00 2001 From: Srid Banoor Date: Fri, 22 Sep 2017 16:31:45 -0700 Subject: [PATCH 3/6] DSE topology with Kerberos and non Kerberos support --- start.py | 150 ++++++++++++++++++++++++++++---------------------- topology.yaml | 12 ++-- 2 files changed, 90 insertions(+), 72 deletions(-) diff --git a/start.py b/start.py index b81fb0f..73e9b3d 100644 --- a/start.py +++ b/start.py @@ -14,7 +14,7 @@ import logging import re import textwrap -from os.path import expanduser +import os import yaml @@ -26,9 +26,10 @@ DEFAULT_NAMESPACE = 'clusterdock' DEFAULT_OPERATING_SYSTEM = 'centos6.8' +KERBEROS_VOLUME_DIR = '/etc/clusterdock/kerberos' + KDC_ACL_FILEPATH = '/var/kerberos/krb5kdc/kadm5.acl' KDC_CONF_FILEPATH = '/var/kerberos/krb5kdc/kdc.conf' -KERBEROS_VOLUME_DIR = '/etc/clusterdock/kerberos' KDC_KEYTAB_FILENAME = 'clusterdock.keytab' KDC_USER_KEYTAB_FILEPATH = '{}/{}'.format(KERBEROS_VOLUME_DIR, KDC_KEYTAB_FILENAME) KDC_KRB5_CONF_FILEPATH = '/etc/krb5.conf' @@ -46,8 +47,8 @@ def main(args): - global quiet_logging - quiet_logging = False if args.verbose else True + dse_image = '{}/{}/clusterdock:dse{}'.format(args.registry, args.namespace or DEFAULT_NAMESPACE, + args.dse_version) dse_image = '{}/{}/clusterdock:dse{}'.format(args.registry, args.namespace, args.dse_version) if args.kerberos: @@ -57,6 +58,8 @@ def main(args): def _setup_non_kerberos_nodes(args, dse_image): + quiet = not args.verbose + nodes = [Node(hostname=hostname, group='nodes', image=dse_image) for hostname in args.nodes] cluster = Cluster(*nodes) cluster.start(args.network) @@ -72,7 +75,7 @@ def _setup_non_kerberos_nodes(args, dse_image): 'cp {} {}.org'.format(DSE_CONF_FILEPATH, DSE_CONF_FILEPATH), 'mkdir -p {}'.format(DSE_CQLSHRC_HOME_DIR) ] - node.execute(command="bash -c '{}'".format('; '.join(dse_config_commands)), quiet=quiet_logging) + node.execute(command="bash -c '{}'".format('; '.join(dse_config_commands)), quiet=quiet) # DSE cassandra.yaml mods cassandra_config_data = yaml.load(node.get_file(DSE_CASSANDRA_CONF_FILEPATH)) cassandra_config_data['cluster_name'] = cluster_name @@ -87,8 +90,9 @@ def _setup_non_kerberos_nodes(args, dse_image): node.put_file(DSE_CONF_FILEPATH, yaml.dump(dse_config_data)) # DSE cqlsh specific commands cqlsh_cmd_data = node.get_file(DSE_CQLSH_FILEPATH) - node.put_file(DSE_CQLSH_FILEPATH, re.sub(r'.*(bash code here).*', '. /opt/rh/python27/enable', cqlsh_cmd_data)) - node.execute(command='chmod +x {}'.format(DSE_CQLSH_FILEPATH), quiet=quiet_logging) + node.put_file(DSE_CQLSH_FILEPATH, re.sub(r'.*(bash code here).*', + '. /opt/rh/python27/enable', cqlsh_cmd_data)) + node.execute(command='chmod +x {}'.format(DSE_CQLSH_FILEPATH), quiet=quiet) cqlshrc_data = """ [connection] hostname = {} @@ -99,21 +103,28 @@ def _setup_non_kerberos_nodes(args, dse_image): node.execute('service dse restart') logger.info('Validating DSE service health ...') - cqlsh_cmd = "cqlsh -u cassandra -p cassandra {} --debug -e 'DESCRIBE KEYSPACES'".format(nodes[0].fqdn) - _validate_dse_health(nodes=nodes, node_cmd=cqlsh_cmd, node_cmd_expected='system_schema') + cqlsh_cmd = "cqlsh -u cassandra -p cassandra {} --debug -e 'DESCRIBE KEYSPACES'".format( + nodes[0].fqdn) + _validate_dse_health(nodes=nodes, node_cmd=cqlsh_cmd, node_cmd_expected='system_schema', + quiet=quiet) - logger.info('DSE cluster is available and its contacts are: {}'.format(','.join(node.fqdn for node in nodes))) + logger.info('DSE cluster is available and its contacts are: {}'.format( + ','.join(node.fqdn for node in nodes))) logger.info('From its node, DSE can be accessed with: cqlsh -u cassandra -p cassandra') def _setup_kerberos_nodes(args, dse_image): - kerberos_volume_dir = args.kerberos_config_directory.replace('~', expanduser('~')) + quiet = not args.verbose + + kerberos_volume_dir = os.path.expanduser(args.kerberos_config_directory) nodes = [Node(hostname=hostname, group='nodes', image=dse_image, volumes=[{kerberos_volume_dir: KERBEROS_VOLUME_DIR}]) for hostname in args.nodes] - kdc_image = '{}/{}/topology_nodebase_kerberos:{}'.format(args.registry, args.namespace or DEFAULT_NAMESPACE, - args.operating_system or DEFAULT_OPERATING_SYSTEM) + kdc_image = '{}/{}/topology_nodebase_kerberos:{}'.format(args.registry, + args.namespace or DEFAULT_NAMESPACE, + args.operating_system + or DEFAULT_OPERATING_SYSTEM) kdc_hostname = args.kdc_node[0] kdc_node = Node(hostname=kdc_hostname, group='kdc', image=kdc_image, volumes=[{kerberos_volume_dir: KERBEROS_VOLUME_DIR}]) @@ -122,58 +133,57 @@ def _setup_kerberos_nodes(args, dse_image): logger.info('Updating KDC configurations ...') realm = cluster.network.upper() - # Update configurations krb5_conf_data = kdc_node.get_file(KDC_KRB5_CONF_FILEPATH) kdc_node.put_file(KDC_KRB5_CONF_FILEPATH, re.sub(r'EXAMPLE.COM', realm, re.sub(r'example.com', cluster.network, - re.sub(r'kerberos.example.com', r'{}.{}'.format(kdc_hostname, cluster.network), + re.sub(r'kerberos.example.com', + r'{}.{}'.format(kdc_hostname, cluster.network), krb5_conf_data)))) kdc_conf_data = kdc_node.get_file(KDC_CONF_FILEPATH) kdc_node.put_file(KDC_CONF_FILEPATH, re.sub(r'EXAMPLE.COM', realm, - kdc_conf_data.replace(r'[kdcdefaults]', - '[kdcdefaults]\n max_renewablelife = 7d\n max_life = 1d'))) + re.sub(r'\[kdcdefaults\]', + r'[kdcdefaults]\n max_renewablelife = 7d\n max_life = 1d', + kdc_conf_data))) acl_data = kdc_node.get_file(KDC_ACL_FILEPATH) kdc_node.put_file(KDC_ACL_FILEPATH, re.sub(r'EXAMPLE.COM', realm, acl_data)) + logger.info('Starting KDC ...') kdc_commands = [ 'kdb5_util create -s -r {realm} -P kdcadmin'.format(realm=realm), - 'kadmin.local -q "addprinc -pw {adminpw} admin/admin@{realm}"'.format(adminpw='acladmin', realm=realm) + 'kadmin.local -q "addprinc -pw {admin_pw} admin/admin@{realm}"'.format(admin_pw='acladmin', + realm=realm) ] - logger.info('Starting KDC ...') # Add the following commands before starting kadmin daemon etc. if args.kerberos_principals: - principal_list = ['{}@{}'.format(primary, realm) for primary in args.kerberos_principals.split(',')] + principal_list = ['{}@{}'.format(principal, realm) + for principal in args.kerberos_principals.split(',')] create_principals_cmds = ['kadmin.local -q "addprinc -randkey {}"'.format(principal) for principal in principal_list] kdc_commands.extend(create_principals_cmds) kdc_commands.append('sleep 2') # sleep few seconds to have Docker volume available kdc_commands.append('rm -f {}'.format(KDC_USER_KEYTAB_FILEPATH)) - create_keytab_cmd = 'kadmin.local -q "xst -norandkey -k {} {}" '.format(KDC_USER_KEYTAB_FILEPATH, - ' '.join(principal_list)) + create_keytab_cmd = 'kadmin.local -q "xst -norandkey -k {} {}" '.format( + KDC_USER_KEYTAB_FILEPATH, ' '.join(principal_list)) kdc_commands.append(create_keytab_cmd) kdc_commands.extend([ 'krb5kdc', 'kadmind', - 'authconfig --enablekrb5 --update', - 'service sshd start', - 'service krb5kdc start', - 'service kadmin start' + 'authconfig --enablekrb5 --update' ]) - # Gather keytab file and krb5.conf file in KERBEROS_VOLUME_DIR directory which is mounted on host. - kdc_commands.append('cp {} {}'.format(KDC_KRB5_CONF_FILEPATH, KERBEROS_VOLUME_DIR)) + kdc_commands.append('cp -f {} {}'.format(KDC_KRB5_CONF_FILEPATH, KERBEROS_VOLUME_DIR)) if args.kerberos_principals: kdc_commands.append('chmod 644 {}'.format(KDC_USER_KEYTAB_FILEPATH)) - kdc_node.execute(command="bash -c '{}'".format('; '.join(kdc_commands)), quiet=quiet_logging) + kdc_node.execute(command="bash -c '{}'".format('; '.join(kdc_commands)), quiet=quiet) - logger.info('Validating KDC service health ...') - _validate_kdc_health(node=kdc_node, services=['sshd', 'krb5kdc', 'kadmin']) + logger.info('Validating service health ...') + _validate_kdc_health(node=kdc_node, services=['krb5kdc', 'kadmin'], quiet=quiet) # Add DSE specific logic on KDC host logger.info('Creating `dse` and `HTTP` Kerberos principals for DSE nodes ...') @@ -187,7 +197,7 @@ def _setup_kerberos_nodes(args, dse_image): 'kadmin.local -q "ktadd -k {} HTTP/{}"'.format(key_tab_filename, node.fqdn), 'chmod 644 {}'.format(key_tab_filename) ] - kdc_node.execute(command="bash -c '{}'".format('; '.join(kdc_dse_commands)), quiet=quiet_logging) + kdc_node.execute(command="bash -c '{}'".format('; '.join(kdc_dse_commands)), quiet=quiet) # DSE node logic logger.info('Updating DSE configurations and starting DSE nodes ...') @@ -207,14 +217,14 @@ def _setup_kerberos_nodes(args, dse_image): 'chown cassandra:cassandra {}'.format(DSE_USER_KEYTAB_FILEPATH), 'chmod 600 {}'.format(DSE_USER_KEYTAB_FILEPATH) ]) - node.execute(command="bash -c '{}'".format('; '.join(dse_kdc_commands)), quiet=quiet_logging) + node.execute(command="bash -c '{}'".format('; '.join(dse_kdc_commands)), quiet=quiet) # DSE config specific commands dse_config_commands = [ 'cp {} {}.org'.format(DSE_CASSANDRA_CONF_FILEPATH, DSE_CASSANDRA_CONF_FILEPATH), 'cp {} {}.org'.format(DSE_CONF_FILEPATH, DSE_CONF_FILEPATH), 'mkdir -p {}'.format(DSE_CQLSHRC_HOME_DIR) ] - node.execute(command="bash -c '{}'".format('; '.join(dse_config_commands)), quiet=quiet_logging) + node.execute(command="bash -c '{}'".format('; '.join(dse_config_commands)), quiet=quiet) # DSE cassandra.yaml mods cassandra_config_data = yaml.load(node.get_file(DSE_CASSANDRA_CONF_FILEPATH)) cassandra_config_data['cluster_name'] = cluster_name @@ -227,11 +237,13 @@ def _setup_kerberos_nodes(args, dse_image): dse_config_data['audit_logging_options']['enabled'] = True dse_config_data['authentication_options'] = {'enabled': True, 'default_scheme': 'internal', 'allow_digest_with_kerberos': False, - 'plain_text_without_ssl': 'warn', 'transitional_mode': 'disabled', + 'plain_text_without_ssl': + 'warn', 'transitional_mode': 'disabled', 'other_schemes': ['internal', 'kerberos'], 'scheme_permissions': False} dse_config_data['role_management_options'] = {'mode': 'internal'} - dse_config_data['authorization_options'] = {'enabled': True, 'transitional_mode': 'disabled', + dse_config_data['authorization_options'] = {'enabled': True, 'transitional_mode': + 'disabled', 'allow_row_level_security': False} dse_config_data['kerberos_options'] = {'keytab': DSE_KEYTAB_FILEPATH, 'service_principal': 'dse/_HOST@{}'.format(realm), @@ -240,8 +252,9 @@ def _setup_kerberos_nodes(args, dse_image): node.put_file(DSE_CONF_FILEPATH, yaml.dump(dse_config_data)) # DSE cqlsh specific commands cqlsh_cmd_data = node.get_file(DSE_CQLSH_FILEPATH) - node.put_file(DSE_CQLSH_FILEPATH, re.sub(r'.*(bash code here).*', '. /opt/rh/python27/enable', cqlsh_cmd_data)) - node.execute(command='chmod +x {}'.format(DSE_CQLSH_FILEPATH), quiet=quiet_logging) + node.put_file(DSE_CQLSH_FILEPATH, re.sub(r'.*(bash code here).*', + '. /opt/rh/python27/enable', cqlsh_cmd_data)) + node.execute(command='chmod +x {}'.format(DSE_CQLSH_FILEPATH), quiet=quiet) cqlshrc_data = """ [connection] hostname = {} @@ -256,35 +269,38 @@ def _setup_kerberos_nodes(args, dse_image): node.execute('service dse restart') logger.info('Validating DSE service health ...') - cqlsh_cmd = "cqlsh -u cassandra -p cassandra {} --debug -e 'DESCRIBE KEYSPACES'".format(nodes[0].fqdn) + cqlsh_cmd = "cqlsh -u cassandra -p cassandra {} --debug -e 'DESCRIBE KEYSPACES'".format( + nodes[0].fqdn) _validate_dse_health(nodes=nodes, node_cmd=cqlsh_cmd, node_cmd_expected='system_schema') if args.kerberos_principals: - principal_list = ['{}@{}'.format(primary, realm) for primary in args.kerberos_principals.split(',')] + principal_list = ['{}@{}'.format(principal, realm) + for principal in args.kerberos_principals.split(',')] logger.info('Creating DSE Kerberos roles {} ...'.format(principal_list)) - logger.info('Kerberos DSE keytab file available on the node at {}'.format(DSE_USER_KEYTAB_FILEPATH)) + logger.info('Kerberos DSE keytab file available on the node at {}'.format( + DSE_USER_KEYTAB_FILEPATH)) for principal in principal_list: cqlsh_cmd = """cqlsh -u cassandra -p cassandra {} --debug """.format(nodes[0].fqdn) cqlsh_cmd += """-e 'CREATE ROLE "{}" WITH LOGIN = true;""".format(principal) cqlsh_cmd += """GRANT EXECUTE on KERBEROS SCHEME to "{}";""".format(principal) cqlsh_cmd += """GRANT ALL on ALL KEYSPACES to "{}";'""".format(principal) - nodes[0].execute(command=cqlsh_cmd, quiet=quiet_logging) + nodes[0].execute(command=cqlsh_cmd, quiet=quiet) - logger.info('DSE cluster is available and its contacts are: {}'.format(','.join(node.fqdn for node in nodes))) + logger.info('DSE cluster is available and its contacts are: {}'.format( + ','.join(node.fqdn for node in nodes))) logger.info('From its node, DSE can be accessed with: cqlsh -u cassandra -p cassandra') -def _validate_kdc_health(node, services): +def _validate_kdc_health(node, services, quiet=True): def condition(node, services): - if all('is running' in (node.execute(command='service {} status'.format(service), quiet=quiet_logging).output) - for service in services): - return True - else: - logger.debug('Services with poor health: %s', - ', '.join(service - for service in services - if 'is running' not in node.execute(command='service {} status'.format(service), - quiet=quiet_logging).output)) + services_with_poor_health = [service + for service in services + if node.execute(command='service {} status'.format(service), + quiet=quiet).exit_code != 0] + if services_with_poor_health: + logger.debug('Services with poor health: %s', ', '.join(services_with_poor_health)) + # Return True if the list of services with poor health is empty. + return not bool(services_with_poor_health) def success(time): logger.debug('Validated service health in %s seconds.', time) @@ -292,22 +308,23 @@ def success(time): def failure(timeout): raise TimeoutError('Timed out after {} seconds waiting ' 'to validate service health.'.format(timeout)) + wait_for_condition(condition=condition, condition_args=[node, services], - time_between_checks=3, timeout=600, success=success, failure=failure) + time_between_checks=3, timeout=30, success=success, failure=failure) -def _validate_dse_health(nodes, node_cmd, node_cmd_expected): +def _validate_dse_health(nodes, node_cmd, node_cmd_expected, quiet=True): def condition(nodes, node_cmd, node_cmd_expected): - if all('running' in (node.execute(command='nodetool statusgossip', quiet=quiet_logging).output) - for node in nodes) and node_cmd_expected in nodes[0].execute(command=node_cmd, - quiet=quiet_logging).output: - return True - else: - logger.debug('Node with poor health: %s', - ', '.join(node.fqdn - for node in nodes - if 'running' not in node.execute(command='nodetool statusgossip', - quiet=quiet_logging).output)) + nodes_with_poor_health = [node for node in nodes + if 'running' not in node.execute(command='nodetool statusgossip', + quiet=quiet).output + or node_cmd_expected not in node.execute(command=node_cmd, + quiet=quiet).output] + if nodes_with_poor_health: + logger.debug('Nodes with poor health: %s', + ', '.join(node.fqdn for node in nodes_with_poor_health)) + # Return True if the list of nodes with poor health is empty. + return not bool(nodes_with_poor_health) def success(time): logger.debug('Validated DSE health in %s seconds.', time) @@ -315,5 +332,6 @@ def success(time): def failure(timeout): raise TimeoutError('Timed out after {} seconds waiting ' 'to validate DSE health.'.format(timeout)) + wait_for_condition(condition=condition, condition_args=[nodes, node_cmd, node_cmd_expected], - time_between_checks=3, timeout=600, success=success, failure=failure) + time_between_checks=3, timeout=90, success=success, failure=failure) diff --git a/topology.yaml b/topology.yaml index 5f55253..94fb413 100644 --- a/topology.yaml +++ b/topology.yaml @@ -25,24 +25,24 @@ node groups: build args: --dse-version: default: 5.1.3-1 - help: DSE version to use. + help: DSE version to use metavar: ver start args: --dse-version: default: 5.1.3-1 - help: DSE version to use. + help: DSE version to use metavar: ver --dse-cluster-name: default: Test Cluster - help: DSE cluster name to use. + help: DSE cluster name to use --kerberos: - help: If specified, sets up Kerberos based DSE cluster with a KDC node. + help: If specified, sets up Kerberos based DSE cluster with a KDC node action: store_true --kerberos-config-directory: default: ~/.clusterdock/kerberos - help: If specified, mounts this directory to KDC container for Kerberos config files. + help: Mounts this directory to KDC container for Kerberos config files metavar: path --kerberos-principals: - help: If specified, a comma-separated list of Kerberos user principals to create in KDC. + help: If specified, a comma-separated list of Kerberos user principals to create in KDC metavar: princ1,princ2,... From ead2d474f741515557eb8e7dd94148c31be81ecb Mon Sep 17 00:00:00 2001 From: Srid Banoor Date: Fri, 22 Sep 2017 16:31:45 -0700 Subject: [PATCH 4/6] DSE topology with Kerberos and non Kerberos support --- .gitignore | 65 +++++++++++++++++++++ README.rst | 5 +- images/centos6.8/Dockerfile | 30 +++++----- images/centos6.8/dse.repo | 2 +- start.py | 109 ++++++++++++++++++------------------ topology.yaml | 6 +- 6 files changed, 141 insertions(+), 76 deletions(-) create mode 100644 .gitignore diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..9c4d6e3 --- /dev/null +++ b/.gitignore @@ -0,0 +1,65 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +env/ +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +*.egg-info/ +.installed.cfg +*.egg + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*,cover +.hypothesis/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# pyenv python configuration file +.python-version + +# Editor configuration files +.remote-sync.json diff --git a/README.rst b/README.rst index 1839a07..6af160d 100644 --- a/README.rst +++ b/README.rst @@ -32,7 +32,7 @@ with the ``clusterdock`` script: 2017-09-20 10:18:53 PM clusterdock.topology_dse.start INFO From its node, DSE can be accessed with: cqlsh -u cassandra -p cassandra 2017-09-20 10:18:53 PM clusterdock.cli INFO Cluster started successfully (total time: 1m 14s). -To start a Kerberos based DSE cluster: +To start a Kerberos-based DSE cluster: .. code-block:: console @@ -102,9 +102,6 @@ To see full usage instructions for the ``start`` action, use ``-h``/``--help``: principals to create in KDC. (default: None) Node groups: - --kdc-node node [node ...] - Nodes of the kdc-node group (default: ['kdc']) --nodes node [node ...] Nodes of the nodes group (default: ['node-1', 'node-2']) - 'node-2']) diff --git a/images/centos6.8/Dockerfile b/images/centos6.8/Dockerfile index f668c35..4346688 100644 --- a/images/centos6.8/Dockerfile +++ b/images/centos6.8/Dockerfile @@ -11,15 +11,23 @@ # limitations under the License. FROM clusterdock/topology_nodebase:centos6.8 -ADD dse.repo /etc/yum.repos.d/ - ARG JDK_URL=http://download.oracle.com/otn-pub/java/jdk/8u131-b11/d54c1d3a095b4ff2b6607d096fa80163/jdk-8u131-linux-x64.tar.gz ARG JCE_URL=http://download.oracle.com/otn-pub/java/jce/8/jce_policy-8.zip ARG JDK_DIRECTORY=/usr/java/jdk1.8.0_131 ARG DSE_VERSION=5.1.3-1 +ARG DSE_REPO_URL + +RUN echo ${DSE_REPO_URL} > /etc/yum/vars/dse_repo_url +ADD dse.repo /etc/yum.repos.d/ RUN yum -y update && \ - yum -y install unzip + yum -y install unzip \ + scl-utils \ + centos-release-scl-rh \ + krb5-workstation \ + krb5-devel \ + gcc \ + python26-kerberos # Java setup RUN wget --no-check-certificate \ @@ -35,18 +43,14 @@ RUN wget --no-check-certificate \ unzip -jo -d ${JDK_DIRECTORY}/jre/lib/security /tmp/unlimited_jce_policy.zip && \ rm -f /tmp/unlimited_jce_policy.zip +# Make the installed Java as default. RUN update-alternatives --install "/usr/bin/java" "java" "${JDK_DIRECTORY}/bin/java" 1 RUN update-alternatives --set java ${JDK_DIRECTORY}/bin/java -# DSE setup -RUN yum -y install dse-full-${DSE_VERSION} - -# Add Python 2.7 as needed by DSE's cqlsh tool -RUN yum install -y scl-utils centos-release-scl-rh -RUN yum install -y python27 +# Add Python 2.7 as needed by DSE's cqlsh tool - needs to follow after scl-utils install. +RUN yum -y install dse-full-${DSE_VERSION} \ + python27 && \ + yum clean all -# Add Kerberos client specific packages -RUN yum install -y krb5-workstation krb5-devel gcc python26-kerberos +# Install Kerberos Python package as required by DSE's cqlsh tool. RUN . /opt/rh/python27/enable && pip install pure-sasl kerberos - -CMD ["/sbin/init"] diff --git a/images/centos6.8/dse.repo b/images/centos6.8/dse.repo index 95e9781..52d54f7 100644 --- a/images/centos6.8/dse.repo +++ b/images/centos6.8/dse.repo @@ -1,5 +1,5 @@ [datastax] name = DataStax Repo for DataStax Enterprise -baseurl=https://srid%40streamsets.com:Streamsetsme3@rpm.datastax.com/enterprise +baseurl=$dse_repo_url enabled=1 gpgcheck=0 diff --git a/start.py b/start.py index 73e9b3d..74acdc7 100644 --- a/start.py +++ b/start.py @@ -12,10 +12,9 @@ # limitations under the License. import logging +import os import re import textwrap -import os - import yaml from clusterdock.models import Cluster, Node @@ -30,27 +29,28 @@ KDC_ACL_FILEPATH = '/var/kerberos/krb5kdc/kadm5.acl' KDC_CONF_FILEPATH = '/var/kerberos/krb5kdc/kdc.conf' +KDC_HOSTNAME = 'kdc' KDC_KEYTAB_FILENAME = 'clusterdock.keytab' -KDC_USER_KEYTAB_FILEPATH = '{}/{}'.format(KERBEROS_VOLUME_DIR, KDC_KEYTAB_FILENAME) KDC_KRB5_CONF_FILEPATH = '/etc/krb5.conf' +KDC_USER_KEYTAB_FILEPATH = '{}/{}'.format(KERBEROS_VOLUME_DIR, KDC_KEYTAB_FILENAME) +DSE_CQLSHRC_HOME_DIR = '/root/.cassandra' DSE_HOME_DIR = '/etc/dse' -DSE_CONF_FILENAME = 'dse.yaml' -DSE_CONF_FILEPATH = '{}/{}'.format(DSE_HOME_DIR, DSE_CONF_FILENAME) + DSE_CASSANDRA_CONF_FILENAME = 'cassandra.yaml' DSE_CASSANDRA_CONF_FILEPATH = '{}/cassandra/{}'.format(DSE_HOME_DIR, DSE_CASSANDRA_CONF_FILENAME) +DSE_CONF_FILENAME = 'dse.yaml' +DSE_CONF_FILEPATH = '{}/{}'.format(DSE_HOME_DIR, DSE_CONF_FILENAME) DSE_CQLSH_FILEPATH = '/usr/bin/cqlsh' -DSE_CQLSHRC_HOME_DIR = '/root/.cassandra' DSE_CQLSHRC_FILEPATH = '{}/cqlshrc'.format(DSE_CQLSHRC_HOME_DIR) DSE_KEYTAB_FILEPATH = '{}/dse.keytab'.format(DSE_HOME_DIR) DSE_USER_KEYTAB_FILEPATH = '{}/{}'.format(DSE_HOME_DIR, KDC_KEYTAB_FILENAME) def main(args): - dse_image = '{}/{}/clusterdock:dse{}'.format(args.registry, args.namespace or DEFAULT_NAMESPACE, + dse_image = '{}/{}/clusterdock:dse{}'.format(args.registry, + args.namespace or DEFAULT_NAMESPACE, args.dse_version) - dse_image = '{}/{}/clusterdock:dse{}'.format(args.registry, args.namespace, args.dse_version) - if args.kerberos: _setup_kerberos_nodes(args, dse_image) else: @@ -71,42 +71,34 @@ def _setup_non_kerberos_nodes(args, dse_image): for node in nodes: # DSE config specific commands dse_config_commands = [ - 'cp {} {}.org'.format(DSE_CASSANDRA_CONF_FILEPATH, DSE_CASSANDRA_CONF_FILEPATH), - 'cp {} {}.org'.format(DSE_CONF_FILEPATH, DSE_CONF_FILEPATH), + 'cp {} {}.orig'.format(DSE_CASSANDRA_CONF_FILEPATH, DSE_CASSANDRA_CONF_FILEPATH), + 'cp {} {}.orig'.format(DSE_CONF_FILEPATH, DSE_CONF_FILEPATH), 'mkdir -p {}'.format(DSE_CQLSHRC_HOME_DIR) ] - node.execute(command="bash -c '{}'".format('; '.join(dse_config_commands)), quiet=quiet) + node.execute('; '.join(dse_config_commands), quiet=quiet) # DSE cassandra.yaml mods - cassandra_config_data = yaml.load(node.get_file(DSE_CASSANDRA_CONF_FILEPATH)) - cassandra_config_data['cluster_name'] = cluster_name - cassandra_config_data['listen_address'] = node.ip_address - cassandra_config_data['rpc_address'] = node.ip_address - cassandra_config_data['seed_provider'][0]['parameters'][0]['seeds'] = cluster_seeds - node.put_file(DSE_CASSANDRA_CONF_FILEPATH, yaml.dump(cassandra_config_data)) + _configure_cassandra_yaml(cluster_name, cluster_seeds, node) # DSE dse.yaml mods dse_config_data = yaml.load(node.get_file(DSE_CONF_FILEPATH)) dse_config_data['audit_logging_options']['enabled'] = True dse_config_data['authentication_options'] = {'enabled': True, 'default_scheme': 'internal'} node.put_file(DSE_CONF_FILEPATH, yaml.dump(dse_config_data)) # DSE cqlsh specific commands - cqlsh_cmd_data = node.get_file(DSE_CQLSH_FILEPATH) - node.put_file(DSE_CQLSH_FILEPATH, re.sub(r'.*(bash code here).*', - '. /opt/rh/python27/enable', cqlsh_cmd_data)) node.execute(command='chmod +x {}'.format(DSE_CQLSH_FILEPATH), quiet=quiet) cqlshrc_data = """ [connection] hostname = {} port = 9042 """.format(node.fqdn) - node.put_file(DSE_CQLSHRC_FILEPATH, textwrap.dedent(cqlshrc_data)) + _configure_cqlsh(cqlshrc_data, node, quiet) # start DSE on the node node.execute('service dse restart') logger.info('Validating DSE service health ...') - cqlsh_cmd = "cqlsh -u cassandra -p cassandra {} --debug -e 'DESCRIBE KEYSPACES'".format( - nodes[0].fqdn) + cqlsh_cmd = ("cqlsh -u cassandra -p cassandra {} " + "--debug -e 'DESCRIBE KEYSPACES'").format(nodes[0].fqdn) _validate_dse_health(nodes=nodes, node_cmd=cqlsh_cmd, node_cmd_expected='system_schema', - quiet=quiet) + quiet=quiet) logger.info('DSE cluster is available and its contacts are: {}'.format( ','.join(node.fqdn for node in nodes))) @@ -125,8 +117,7 @@ def _setup_kerberos_nodes(args, dse_image): args.namespace or DEFAULT_NAMESPACE, args.operating_system or DEFAULT_OPERATING_SYSTEM) - kdc_hostname = args.kdc_node[0] - kdc_node = Node(hostname=kdc_hostname, group='kdc', image=kdc_image, + kdc_node = Node(hostname=KDC_HOSTNAME, group='kdc', image=kdc_image, volumes=[{kerberos_volume_dir: KERBEROS_VOLUME_DIR}]) cluster = Cluster(kdc_node, *nodes) cluster.start(args.network) @@ -138,7 +129,7 @@ def _setup_kerberos_nodes(args, dse_image): re.sub(r'EXAMPLE.COM', realm, re.sub(r'example.com', cluster.network, re.sub(r'kerberos.example.com', - r'{}.{}'.format(kdc_hostname, cluster.network), + r'{}.{}'.format(KDC_HOSTNAME, cluster.network), krb5_conf_data)))) kdc_conf_data = kdc_node.get_file(KDC_CONF_FILEPATH) kdc_node.put_file(KDC_CONF_FILEPATH, @@ -151,7 +142,7 @@ def _setup_kerberos_nodes(args, dse_image): logger.info('Starting KDC ...') kdc_commands = [ - 'kdb5_util create -s -r {realm} -P kdcadmin'.format(realm=realm), + 'kdb5_util create -s -r {} -P kdcadmin'.format(realm), 'kadmin.local -q "addprinc -pw {admin_pw} admin/admin@{realm}"'.format(admin_pw='acladmin', realm=realm) ] @@ -164,7 +155,6 @@ def _setup_kerberos_nodes(args, dse_image): for principal in principal_list] kdc_commands.extend(create_principals_cmds) - kdc_commands.append('sleep 2') # sleep few seconds to have Docker volume available kdc_commands.append('rm -f {}'.format(KDC_USER_KEYTAB_FILEPATH)) create_keytab_cmd = 'kadmin.local -q "xst -norandkey -k {} {}" '.format( KDC_USER_KEYTAB_FILEPATH, ' '.join(principal_list)) @@ -180,9 +170,9 @@ def _setup_kerberos_nodes(args, dse_image): if args.kerberos_principals: kdc_commands.append('chmod 644 {}'.format(KDC_USER_KEYTAB_FILEPATH)) - kdc_node.execute(command="bash -c '{}'".format('; '.join(kdc_commands)), quiet=quiet) + kdc_node.execute('; '.join(kdc_commands), quiet=quiet) - logger.info('Validating service health ...') + logger.info('Validating Kerberos service health ...') _validate_kdc_health(node=kdc_node, services=['krb5kdc', 'kadmin'], quiet=quiet) # Add DSE specific logic on KDC host @@ -197,7 +187,7 @@ def _setup_kerberos_nodes(args, dse_image): 'kadmin.local -q "ktadd -k {} HTTP/{}"'.format(key_tab_filename, node.fqdn), 'chmod 644 {}'.format(key_tab_filename) ] - kdc_node.execute(command="bash -c '{}'".format('; '.join(kdc_dse_commands)), quiet=quiet) + kdc_node.execute('; '.join(kdc_dse_commands), quiet=quiet) # DSE node logic logger.info('Updating DSE configurations and starting DSE nodes ...') @@ -206,32 +196,27 @@ def _setup_kerberos_nodes(args, dse_image): for node in nodes: # kerberos specific commands dse_kdc_commands = [ - 'cp {}/krb5.conf /etc/.'.format(KERBEROS_VOLUME_DIR), + 'cp {}/krb5.conf /etc'.format(KERBEROS_VOLUME_DIR), 'cp {}/{}.keytab {}'.format(KERBEROS_VOLUME_DIR, node.fqdn, DSE_KEYTAB_FILEPATH), 'chown cassandra:cassandra {}'.format(DSE_KEYTAB_FILEPATH), 'chmod 600 {}'.format(DSE_KEYTAB_FILEPATH) ] if args.kerberos_principals: dse_kdc_commands.extend([ - 'cp {} {}/.'.format(KDC_USER_KEYTAB_FILEPATH, DSE_HOME_DIR), + 'cp {} {}'.format(KDC_USER_KEYTAB_FILEPATH, DSE_HOME_DIR), 'chown cassandra:cassandra {}'.format(DSE_USER_KEYTAB_FILEPATH), 'chmod 600 {}'.format(DSE_USER_KEYTAB_FILEPATH) ]) - node.execute(command="bash -c '{}'".format('; '.join(dse_kdc_commands)), quiet=quiet) + node.execute('; '.join(dse_kdc_commands), quiet=quiet) # DSE config specific commands dse_config_commands = [ - 'cp {} {}.org'.format(DSE_CASSANDRA_CONF_FILEPATH, DSE_CASSANDRA_CONF_FILEPATH), - 'cp {} {}.org'.format(DSE_CONF_FILEPATH, DSE_CONF_FILEPATH), + 'cp {} {}.orig'.format(DSE_CASSANDRA_CONF_FILEPATH, DSE_CASSANDRA_CONF_FILEPATH), + 'cp {} {}.orig'.format(DSE_CONF_FILEPATH, DSE_CONF_FILEPATH), 'mkdir -p {}'.format(DSE_CQLSHRC_HOME_DIR) ] - node.execute(command="bash -c '{}'".format('; '.join(dse_config_commands)), quiet=quiet) + node.execute('; '.join(dse_config_commands), quiet=quiet) # DSE cassandra.yaml mods - cassandra_config_data = yaml.load(node.get_file(DSE_CASSANDRA_CONF_FILEPATH)) - cassandra_config_data['cluster_name'] = cluster_name - cassandra_config_data['listen_address'] = node.ip_address - cassandra_config_data['rpc_address'] = node.ip_address - cassandra_config_data['seed_provider'][0]['parameters'][0]['seeds'] = cluster_seeds - node.put_file(DSE_CASSANDRA_CONF_FILEPATH, yaml.dump(cassandra_config_data)) + _configure_cassandra_yaml(cluster_name, cluster_seeds, node) # DSE dse.yaml mods dse_config_data = yaml.load(node.get_file(DSE_CONF_FILEPATH)) dse_config_data['audit_logging_options']['enabled'] = True @@ -251,9 +236,6 @@ def _setup_kerberos_nodes(args, dse_image): 'qop': 'auth'} node.put_file(DSE_CONF_FILEPATH, yaml.dump(dse_config_data)) # DSE cqlsh specific commands - cqlsh_cmd_data = node.get_file(DSE_CQLSH_FILEPATH) - node.put_file(DSE_CQLSH_FILEPATH, re.sub(r'.*(bash code here).*', - '. /opt/rh/python27/enable', cqlsh_cmd_data)) node.execute(command='chmod +x {}'.format(DSE_CQLSH_FILEPATH), quiet=quiet) cqlshrc_data = """ [connection] @@ -264,14 +246,15 @@ def _setup_kerberos_nodes(args, dse_image): service = dse qops = auth """.format(node.fqdn) - node.put_file(DSE_CQLSHRC_FILEPATH, textwrap.dedent(cqlshrc_data)) + _configure_cqlsh(cqlshrc_data, node, quiet) # start DSE on the node node.execute('service dse restart') logger.info('Validating DSE service health ...') cqlsh_cmd = "cqlsh -u cassandra -p cassandra {} --debug -e 'DESCRIBE KEYSPACES'".format( nodes[0].fqdn) - _validate_dse_health(nodes=nodes, node_cmd=cqlsh_cmd, node_cmd_expected='system_schema') + _validate_dse_health(nodes=nodes, node_cmd=cqlsh_cmd, node_cmd_expected='system_schema', + quiet=quiet) if args.kerberos_principals: principal_list = ['{}@{}'.format(principal, realm) @@ -280,10 +263,11 @@ def _setup_kerberos_nodes(args, dse_image): logger.info('Kerberos DSE keytab file available on the node at {}'.format( DSE_USER_KEYTAB_FILEPATH)) for principal in principal_list: - cqlsh_cmd = """cqlsh -u cassandra -p cassandra {} --debug """.format(nodes[0].fqdn) - cqlsh_cmd += """-e 'CREATE ROLE "{}" WITH LOGIN = true;""".format(principal) - cqlsh_cmd += """GRANT EXECUTE on KERBEROS SCHEME to "{}";""".format(principal) - cqlsh_cmd += """GRANT ALL on ALL KEYSPACES to "{}";'""".format(principal) + cqlsh_cmd = ("""cqlsh -u cassandra -p cassandra {address} --debug """ + """-e 'CREATE ROLE "{principal}" WITH LOGIN = true;""" + """GRANT EXECUTE on KERBEROS SCHEME to "{principal}";""" + """GRANT ALL on ALL KEYSPACES to "{principal}";'""").format( + address=nodes[0].fqdn, principal=principal) nodes[0].execute(command=cqlsh_cmd, quiet=quiet) logger.info('DSE cluster is available and its contacts are: {}'.format( @@ -291,6 +275,23 @@ def _setup_kerberos_nodes(args, dse_image): logger.info('From its node, DSE can be accessed with: cqlsh -u cassandra -p cassandra') +def _configure_cassandra_yaml(cluster_name, cluster_seeds, node): + cassandra_config_data = yaml.load(node.get_file(DSE_CASSANDRA_CONF_FILEPATH)) + cassandra_config_data['cluster_name'] = cluster_name + cassandra_config_data['listen_address'] = node.ip_address + cassandra_config_data['rpc_address'] = node.ip_address + cassandra_config_data['seed_provider'][0]['parameters'][0]['seeds'] = cluster_seeds + node.put_file(DSE_CASSANDRA_CONF_FILEPATH, yaml.dump(cassandra_config_data)) + + +def _configure_cqlsh(cqlshrc_data, node, quiet=True): + cqlsh_cmd_data = node.get_file(DSE_CQLSH_FILEPATH) + node.put_file(DSE_CQLSH_FILEPATH, re.sub(r'.*(bash code here).*', + '. /opt/rh/python27/enable', cqlsh_cmd_data)) + node.execute(command='chmod +x {}'.format(DSE_CQLSH_FILEPATH), quiet=quiet) + node.put_file(DSE_CQLSHRC_FILEPATH, textwrap.dedent(cqlshrc_data)) + + def _validate_kdc_health(node, services, quiet=True): def condition(node, services): services_with_poor_health = [service diff --git a/topology.yaml b/topology.yaml index 94fb413..acadb54 100644 --- a/topology.yaml +++ b/topology.yaml @@ -13,11 +13,9 @@ # yaml definition file for the nodebase DSE topology. name: DSE -description: A Datastax (DSE) cluster +description: A DataStax Enterprise (DSE) cluster node groups: - kdc-node: - - kdc nodes: - node-1 - node-2 @@ -37,7 +35,7 @@ start args: default: Test Cluster help: DSE cluster name to use --kerberos: - help: If specified, sets up Kerberos based DSE cluster with a KDC node + help: If specified, sets up Kerberos-based DSE cluster with a KDC node action: store_true --kerberos-config-directory: default: ~/.clusterdock/kerberos From 55f08819880033bd8fc593eb79fec070cd2c32b0 Mon Sep 17 00:00:00 2001 From: Srid Banoor Date: Fri, 22 Sep 2017 16:31:45 -0700 Subject: [PATCH 5/6] DSE topology with Kerberos and non Kerberos support --- start.py | 28 ++++++++++++++++++++++------ 1 file changed, 22 insertions(+), 6 deletions(-) diff --git a/start.py b/start.py index 74acdc7..401e169 100644 --- a/start.py +++ b/start.py @@ -59,7 +59,6 @@ def main(args): def _setup_non_kerberos_nodes(args, dse_image): quiet = not args.verbose - nodes = [Node(hostname=hostname, group='nodes', image=dse_image) for hostname in args.nodes] cluster = Cluster(*nodes) cluster.start(args.network) @@ -115,8 +114,8 @@ def _setup_kerberos_nodes(args, dse_image): kdc_image = '{}/{}/topology_nodebase_kerberos:{}'.format(args.registry, args.namespace or DEFAULT_NAMESPACE, - args.operating_system - or DEFAULT_OPERATING_SYSTEM) + args.operating_system or + DEFAULT_OPERATING_SYSTEM) kdc_node = Node(hostname=KDC_HOSTNAME, group='kdc', image=kdc_image, volumes=[{kerberos_volume_dir: KERBEROS_VOLUME_DIR}]) cluster = Cluster(kdc_node, *nodes) @@ -292,6 +291,23 @@ def _configure_cqlsh(cqlshrc_data, node, quiet=True): node.put_file(DSE_CQLSHRC_FILEPATH, textwrap.dedent(cqlshrc_data)) +def _configure_cassandra_yaml(cluster_name, cluster_seeds, node): + cassandra_config_data = yaml.load(node.get_file(DSE_CASSANDRA_CONF_FILEPATH)) + cassandra_config_data['cluster_name'] = cluster_name + cassandra_config_data['listen_address'] = node.ip_address + cassandra_config_data['rpc_address'] = node.ip_address + cassandra_config_data['seed_provider'][0]['parameters'][0]['seeds'] = cluster_seeds + node.put_file(DSE_CASSANDRA_CONF_FILEPATH, yaml.dump(cassandra_config_data)) + + +def _configure_cqlsh(cqlshrc_data, node, quiet=True): + cqlsh_cmd_data = node.get_file(DSE_CQLSH_FILEPATH) + node.put_file(DSE_CQLSH_FILEPATH, re.sub(r'.*(bash code here).*', + '. /opt/rh/python27/enable', cqlsh_cmd_data)) + node.execute(command='chmod +x {}'.format(DSE_CQLSH_FILEPATH), quiet=quiet) + node.put_file(DSE_CQLSHRC_FILEPATH, textwrap.dedent(cqlshrc_data)) + + def _validate_kdc_health(node, services, quiet=True): def condition(node, services): services_with_poor_health = [service @@ -318,9 +334,9 @@ def _validate_dse_health(nodes, node_cmd, node_cmd_expected, quiet=True): def condition(nodes, node_cmd, node_cmd_expected): nodes_with_poor_health = [node for node in nodes if 'running' not in node.execute(command='nodetool statusgossip', - quiet=quiet).output - or node_cmd_expected not in node.execute(command=node_cmd, - quiet=quiet).output] + quiet=quiet).output or + node_cmd_expected not in node.execute(command=node_cmd, + quiet=quiet).output] if nodes_with_poor_health: logger.debug('Nodes with poor health: %s', ', '.join(node.fqdn for node in nodes_with_poor_health)) From c8936e1bfb05c4019ee25da9b631f80e157d3458 Mon Sep 17 00:00:00 2001 From: Srid Banoor Date: Fri, 22 Sep 2017 16:31:45 -0700 Subject: [PATCH 6/6] DSE topology with Kerberos and non Kerberos support --- start.py | 19 +------------------ 1 file changed, 1 insertion(+), 18 deletions(-) diff --git a/start.py b/start.py index 401e169..46a3fe1 100644 --- a/start.py +++ b/start.py @@ -97,7 +97,7 @@ def _setup_non_kerberos_nodes(args, dse_image): cqlsh_cmd = ("cqlsh -u cassandra -p cassandra {} " "--debug -e 'DESCRIBE KEYSPACES'").format(nodes[0].fqdn) _validate_dse_health(nodes=nodes, node_cmd=cqlsh_cmd, node_cmd_expected='system_schema', - quiet=quiet) + quiet=quiet) logger.info('DSE cluster is available and its contacts are: {}'.format( ','.join(node.fqdn for node in nodes))) @@ -291,23 +291,6 @@ def _configure_cqlsh(cqlshrc_data, node, quiet=True): node.put_file(DSE_CQLSHRC_FILEPATH, textwrap.dedent(cqlshrc_data)) -def _configure_cassandra_yaml(cluster_name, cluster_seeds, node): - cassandra_config_data = yaml.load(node.get_file(DSE_CASSANDRA_CONF_FILEPATH)) - cassandra_config_data['cluster_name'] = cluster_name - cassandra_config_data['listen_address'] = node.ip_address - cassandra_config_data['rpc_address'] = node.ip_address - cassandra_config_data['seed_provider'][0]['parameters'][0]['seeds'] = cluster_seeds - node.put_file(DSE_CASSANDRA_CONF_FILEPATH, yaml.dump(cassandra_config_data)) - - -def _configure_cqlsh(cqlshrc_data, node, quiet=True): - cqlsh_cmd_data = node.get_file(DSE_CQLSH_FILEPATH) - node.put_file(DSE_CQLSH_FILEPATH, re.sub(r'.*(bash code here).*', - '. /opt/rh/python27/enable', cqlsh_cmd_data)) - node.execute(command='chmod +x {}'.format(DSE_CQLSH_FILEPATH), quiet=quiet) - node.put_file(DSE_CQLSHRC_FILEPATH, textwrap.dedent(cqlshrc_data)) - - def _validate_kdc_health(node, services, quiet=True): def condition(node, services): services_with_poor_health = [service