diff --git a/testing/unittests/test_get_global_metrics.py b/testing/unittests/test_get_global_metrics.py new file mode 100644 index 00000000000..88730e545bd --- /dev/null +++ b/testing/unittests/test_get_global_metrics.py @@ -0,0 +1,232 @@ +#!/usr/bin/env python3 +""" +Unit tests for get_global_metrics.py script. + +This test module verifies the XML request generation and response parsing +functionality of the HPCCGlobalMetricsClient without requiring a live HPCC server. +""" + +import unittest +import sys +import os + +# Add the tools directory to the path to import the script +tools_path = os.path.join(os.path.dirname(__file__), '..', '..', 'tools') +sys.path.insert(0, tools_path) + +from get_global_metrics import HPCCGlobalMetricsClient, parse_dimensions, format_output + + +class TestHPCCGlobalMetricsClient(unittest.TestCase): + """Test cases for HPCCGlobalMetricsClient.""" + + def setUp(self): + """Set up test client.""" + self.client = HPCCGlobalMetricsClient("localhost", 8010) + + def test_create_request_xml_minimal(self): + """Test creating minimal XML request.""" + xml = self.client.create_request_xml() + expected = "" + self.assertEqual(xml.replace('>', ' />'), expected.replace('>', ' />')) + + def test_create_request_xml_with_category(self): + """Test creating XML request with category.""" + xml = self.client.create_request_xml(category="categoryOne") + self.assertIn("categoryOne", xml) + self.assertIn("", xml) + + def test_create_request_xml_with_all_category(self): + """Test creating XML request with 'All' category (should be omitted).""" + xml = self.client.create_request_xml(category="All") + self.assertNotIn("", xml) + self.assertIn("", xml) + self.assertIn("", xml) + self.assertIn("user", xml) + self.assertIn("alice", xml) + self.assertIn("cluster", xml) + self.assertIn("thor1", xml) + + def test_create_request_xml_with_datetime_range(self): + """Test creating XML request with date/time range.""" + xml = self.client.create_request_xml( + start_time="1999-01-01T00:00:00", + end_time="2099-01-01T00:00:00" + ) + + self.assertIn("", xml) + self.assertIn("1999-01-01T00:00:00", xml) + self.assertIn("2099-01-01T00:00:00", xml) + + def test_create_request_xml_complete(self): + """Test creating complete XML request with all parameters.""" + dimensions = {"user": "bob", "cluster": "thor1"} + xml = self.client.create_request_xml( + category="categoryOne", + dimensions=dimensions, + start_time="1999-01-01T00:00:00", + end_time="2099-01-01T00:00:00" + ) + + # Verify all elements are present + self.assertIn("", xml) + self.assertIn("categoryOne", xml) + self.assertIn("", xml) + self.assertIn("user", xml) + self.assertIn("bob", xml) + self.assertIn("", xml) + self.assertIn("1999-01-01T00:00:00", xml) + self.assertIn("2099-01-01T00:00:00", xml) + + def test_parse_response_based_on_cppunit_test(self): + """Test parsing response based on the expected format from cppunit tests.""" + # This is the expected XML from the cppunit test case + xml_response = """ + + categoryTwo + 19990701121999070112 + TimeLocalExecute444 + + + categoryOne + + useralice + + 19990701121999070112 + + TimeLocalExecute222 + CostExecute5 + + + """ + + metrics = self.client.parse_response(xml_response) + + # Verify we got 2 metrics + self.assertEqual(len(metrics), 2) + + # Verify first metric + metric1 = metrics[0] + self.assertEqual(metric1['category'], 'categoryTwo') + self.assertEqual(metric1['dimensions'], {}) + self.assertEqual(metric1['datetime_range']['start'], '1999070112') + self.assertEqual(metric1['datetime_range']['end'], '1999070112') + self.assertEqual(metric1['stats']['TimeLocalExecute'], 444) + + # Verify second metric + metric2 = metrics[1] + self.assertEqual(metric2['category'], 'categoryOne') + self.assertEqual(metric2['dimensions'], {'user': 'alice'}) + self.assertEqual(metric2['datetime_range']['start'], '1999070112') + self.assertEqual(metric2['datetime_range']['end'], '1999070112') + self.assertEqual(metric2['stats']['TimeLocalExecute'], 222) + self.assertEqual(metric2['stats']['CostExecute'], 5) + + def test_filter_stats(self): + """Test filtering statistics.""" + metrics = [ + { + 'category': 'test', + 'dimensions': {}, + 'stats': { + 'TimeLocalExecute': 100, + 'CostExecute': 5, + 'OtherStat': 200 + } + }, + { + 'category': 'test2', + 'dimensions': {}, + 'stats': { + 'TimeLocalExecute': 150, + 'UnwantedStat': 999 + } + } + ] + + filtered = self.client.filter_stats(metrics, ['TimeLocalExecute', 'CostExecute']) + + # Verify filtering + self.assertEqual(len(filtered), 2) + self.assertEqual(filtered[0]['stats'], {'TimeLocalExecute': 100, 'CostExecute': 5}) + self.assertEqual(filtered[1]['stats'], {'TimeLocalExecute': 150}) + + +class TestUtilityFunctions(unittest.TestCase): + """Test utility functions.""" + + def test_parse_dimensions(self): + """Test parsing dimension arguments.""" + dimensions = parse_dimensions(['user:alice', 'cluster:thor1', 'key:value with spaces']) + expected = { + 'user': 'alice', + 'cluster': 'thor1', + 'key': 'value with spaces' + } + self.assertEqual(dimensions, expected) + + def test_parse_dimensions_invalid_format(self): + """Test parsing dimensions with invalid format.""" + with self.assertRaises(ValueError): + parse_dimensions(['invalid_format']) + + def test_format_output_json(self): + """Test JSON output formatting.""" + metrics = [{'category': 'test', 'stats': {'count': 42}}] + output = format_output(metrics, 'json') + self.assertIn('"category": "test"', output) + self.assertIn('"count": 42', output) + + def test_format_output_table(self): + """Test table output formatting.""" + metrics = [ + { + 'category': 'test', + 'dimensions': {'user': 'alice'}, + 'datetime_range': {'start': '2023-01-01', 'end': '2023-12-31'}, + 'stats': {'count': 42} + } + ] + output = format_output(metrics, 'table') + self.assertIn('Global Metrics:', output) + self.assertIn('Category: test', output) + self.assertIn('user: alice', output) + self.assertIn('count: 42', output) + + def test_format_output_csv(self): + """Test CSV output formatting.""" + metrics = [ + { + 'category': 'test', + 'dimensions': {'user': 'alice'}, + 'datetime_range': {'start': '2023-01-01', 'end': '2023-12-31'}, + 'stats': {'count': 42, 'total': 100} + } + ] + output = format_output(metrics, 'csv') + lines = output.strip().split('\n') + + # Check header + self.assertEqual(lines[0], 'category,dimensions,start_time,end_time,stat_name,stat_value') + + # Check data lines + self.assertEqual(len(lines), 3) # Header + 2 stat lines + self.assertIn('test,user=alice,2023-01-01,2023-12-31,count,42', lines) + self.assertIn('test,user=alice,2023-01-01,2023-12-31,total,100', lines) + + def test_format_output_empty_metrics(self): + """Test output formatting with empty metrics.""" + self.assertEqual(format_output([], 'table'), "No metrics found.") + self.assertEqual(format_output([], 'json'), "[]") + self.assertIn('category,dimensions', format_output([], 'csv')) + + +if __name__ == '__main__': + unittest.main() \ No newline at end of file diff --git a/tools/README.md b/tools/README.md new file mode 100644 index 00000000000..49f25a41624 --- /dev/null +++ b/tools/README.md @@ -0,0 +1,221 @@ +# Global Metrics Python Client + +This directory contains a Python script to interact with the HPCC Platform GetGlobalMetrics service in ws_machines. + +## Files + +- `get_global_metrics.py` - Main Python script for querying global metrics +- `../testing/unittests/test_get_global_metrics.py` - Unit tests for the script + +## Features + +The `get_global_metrics.py` script provides: + +1. **Time Range Filtering**: Query metrics for a specific date/time range +2. **Category Filtering**: Filter by specific metric categories +3. **Dimension Filtering**: Filter by dimension name/value pairs (e.g., user, cluster) +4. **Statistics Selection**: Extract only specific statistics from the results +5. **Multiple Output Formats**: JSON, table, and CSV output formats +6. **Authentication Support**: Basic HTTP authentication +7. **HTTPS Support**: Secure connections to HPCC Platform + +## Usage + +### Basic Usage + +```bash +# Get all metrics +python3 get_global_metrics.py --host localhost --port 8010 + +# Get help +python3 get_global_metrics.py --help +``` + +### Time Range Filtering + +```bash +# Query metrics for a specific time range +python3 get_global_metrics.py --host localhost --port 8010 \ + --start "2023-01-01T00:00:00" \ + --end "2023-12-31T23:59:59" +``` + +### Category and Dimension Filtering + +```bash +# Filter by category +python3 get_global_metrics.py --host localhost --port 8010 \ + --category "categoryOne" + +# Filter by dimensions +python3 get_global_metrics.py --host localhost --port 8010 \ + --dimension user:alice \ + --dimension cluster:thor1 + +# Combine filters +python3 get_global_metrics.py --host localhost --port 8010 \ + --category "categoryOne" \ + --dimension user:bob \ + --start "2023-01-01T00:00:00" \ + --end "2023-12-31T23:59:59" +``` + +### Statistics Selection and Output Formatting + +```bash +# Select specific statistics and output as JSON +python3 get_global_metrics.py --host localhost --port 8010 \ + --stats TimeLocalExecute CostExecute \ + --format json + +# Output as CSV for further processing +python3 get_global_metrics.py --host localhost --port 8010 \ + --format csv > metrics.csv +``` + +### Authentication + +```bash +# With authentication +python3 get_global_metrics.py --host remote-host --port 8010 \ + --username myuser --password mypass \ + --https +``` + +## Output Formats + +### Table Format (Default) +Human-readable tabular output showing all metric details. + +### JSON Format +Structured JSON output suitable for programmatic processing: + +```json +[ + { + "category": "categoryOne", + "dimensions": { + "user": "alice", + "cluster": "thor1" + }, + "datetime_range": { + "start": "1999070112", + "end": "1999070112" + }, + "stats": { + "TimeLocalExecute": 222, + "CostExecute": 5 + } + } +] +``` + +### CSV Format +Comma-separated values for spreadsheet processing: + +```csv +category,dimensions,start_time,end_time,stat_name,stat_value +categoryOne,user=alice;cluster=thor1,1999070112,1999070112,TimeLocalExecute,222 +categoryOne,user=alice;cluster=thor1,1999070112,1999070112,CostExecute,5 +``` + +## Service Interface + +The script communicates with the HPCC Platform GetGlobalMetrics service: + +- **URL**: `http(s)://host:port/WsMachine/GetGlobalMetrics` +- **Method**: HTTP POST +- **Content-Type**: application/xml +- **Authentication**: Basic HTTP authentication (optional) + +### Request Format + +```xml + + categoryOne + + + user + alice + + + + 1999-01-01T00:00:00 + 2099-01-01T00:00:00 + + +``` + +### Response Format + +```xml + + + + categoryOne + + + user + alice + + + + 1999070112 + 1999070112 + + + + TimeLocalExecute + 222 + + + CostExecute + 5 + + + + + +``` + +## Future Extensions + +The script is designed to be easily extended with additional functionality: + +1. **Aggregation Functions**: Add support for aggregating statistics (sum, average, min, max) +2. **Filtering**: Advanced filtering capabilities (value ranges, regex patterns) +3. **Caching**: Optional response caching for repeated queries +4. **Batch Processing**: Process multiple queries from configuration files +5. **Visualization**: Integration with plotting libraries for chart generation + +## Testing + +Run the unit tests: + +```bash +cd /path/to/HPCC-Platform +python3 testing/unittests/test_get_global_metrics.py +``` + +The tests verify: +- XML request generation +- XML response parsing +- Statistics filtering +- Output formatting +- Utility functions + +## Dependencies + +- Python 3.6+ +- Standard library modules only (no external dependencies) + +## Error Handling + +The script provides informative error messages for common issues: +- Connection failures +- Authentication errors +- Invalid XML responses +- Malformed time formats +- Missing required parameters + +Use the `--verbose` flag for additional debugging information. \ No newline at end of file diff --git a/tools/example_aggregation.py b/tools/example_aggregation.py new file mode 100755 index 00000000000..a9a5276a95b --- /dev/null +++ b/tools/example_aggregation.py @@ -0,0 +1,272 @@ +#!/usr/bin/env python3 +""" +Example script showing how to use get_global_metrics.py for aggregation tasks. + +This demonstrates how the script can be extended to perform various aggregations +of metrics data, which could be built into the main script in future iterations. +""" + +import json +import subprocess +import sys +from collections import defaultdict +from typing import Dict, List, Any + + +def run_metrics_query(host: str, port: int, **kwargs) -> List[Dict]: + """ + Run get_global_metrics.py and return parsed results. + + Args: + host: HPCC hostname + port: ESP port + **kwargs: Additional arguments for the script + + Returns: + List of metric dictionaries + """ + cmd = ['python3', 'get_global_metrics.py', '--host', host, '--port', str(port), '--format', 'json'] + + # Add optional arguments + for key, value in kwargs.items(): + if key == 'category' and value: + cmd.extend(['--category', value]) + elif key == 'dimensions' and value: + for dim_name, dim_value in value.items(): + cmd.extend(['--dimension', f'{dim_name}:{dim_value}']) + elif key == 'start_time' and value: + cmd.extend(['--start', value]) + elif key == 'end_time' and value: + cmd.extend(['--end', value]) + elif key == 'stats' and value: + cmd.extend(['--stats'] + value) + + try: + result = subprocess.run(cmd, capture_output=True, text=True, check=True) + return json.loads(result.stdout) + except subprocess.CalledProcessError as e: + print(f"Error running metrics query: {e.stderr}", file=sys.stderr) + return [] + except json.JSONDecodeError as e: + print(f"Error parsing JSON response: {e}", file=sys.stderr) + return [] + + +def aggregate_by_category(metrics: List[Dict]) -> Dict[str, Dict[str, float]]: + """ + Aggregate statistics by category. + + Args: + metrics: List of metric dictionaries + + Returns: + Dictionary with categories as keys and aggregated stats as values + """ + category_stats = defaultdict(lambda: defaultdict(list)) + + for metric in metrics: + category = metric.get('category', 'Unknown') + stats = metric.get('stats', {}) + + for stat_name, stat_value in stats.items(): + if isinstance(stat_value, (int, float)): + category_stats[category][stat_name].append(stat_value) + + # Calculate aggregations + result = {} + for category, stats in category_stats.items(): + result[category] = {} + for stat_name, values in stats.items(): + if values: + result[category][stat_name] = { + 'sum': sum(values), + 'avg': sum(values) / len(values), + 'min': min(values), + 'max': max(values), + 'count': len(values) + } + + return result + + +def aggregate_by_dimension(metrics: List[Dict], dimension_name: str) -> Dict[str, Dict[str, float]]: + """ + Aggregate statistics by a specific dimension. + + Args: + metrics: List of metric dictionaries + dimension_name: Name of dimension to aggregate by + + Returns: + Dictionary with dimension values as keys and aggregated stats as values + """ + dimension_stats = defaultdict(lambda: defaultdict(list)) + + for metric in metrics: + dimensions = metric.get('dimensions', {}) + dimension_value = dimensions.get(dimension_name, 'Unknown') + stats = metric.get('stats', {}) + + for stat_name, stat_value in stats.items(): + if isinstance(stat_value, (int, float)): + dimension_stats[dimension_value][stat_name].append(stat_value) + + # Calculate aggregations + result = {} + for dim_value, stats in dimension_stats.items(): + result[dim_value] = {} + for stat_name, values in stats.items(): + if values: + result[dim_value][stat_name] = { + 'sum': sum(values), + 'avg': sum(values) / len(values), + 'min': min(values), + 'max': max(values), + 'count': len(values) + } + + return result + + +def find_top_performers(metrics: List[Dict], stat_name: str, top_n: int = 5) -> List[Dict]: + """ + Find top performing metrics by a specific statistic. + + Args: + metrics: List of metric dictionaries + stat_name: Name of statistic to rank by + top_n: Number of top performers to return + + Returns: + List of top performing metrics + """ + # Filter metrics that have the requested statistic + valid_metrics = [] + for metric in metrics: + stats = metric.get('stats', {}) + if stat_name in stats and isinstance(stats[stat_name], (int, float)): + valid_metrics.append((stats[stat_name], metric)) + + # Sort by statistic value (descending) + valid_metrics.sort(key=lambda x: x[0], reverse=True) + + # Return top N metrics + return [metric for _, metric in valid_metrics[:top_n]] + + +def time_series_analysis(metrics: List[Dict], stat_name: str) -> Dict[str, Any]: + """ + Analyze time series data for a specific statistic. + + Args: + metrics: List of metric dictionaries + stat_name: Name of statistic to analyze + + Returns: + Dictionary with time series analysis results + """ + time_points = [] + values = [] + + for metric in metrics: + stats = metric.get('stats', {}) + datetime_range = metric.get('datetime_range', {}) + + if stat_name in stats and isinstance(stats[stat_name], (int, float)): + start_time = datetime_range.get('start') + if start_time: + time_points.append(start_time) + values.append(stats[stat_name]) + + if not values: + return {'error': f'No data found for statistic: {stat_name}'} + + # Simple time series analysis + return { + 'stat_name': stat_name, + 'total_points': len(values), + 'time_range': { + 'start': min(time_points) if time_points else None, + 'end': max(time_points) if time_points else None + }, + 'value_stats': { + 'sum': sum(values), + 'avg': sum(values) / len(values), + 'min': min(values), + 'max': max(values), + 'variance': sum((x - sum(values) / len(values)) ** 2 for x in values) / len(values) + } + } + + +def main(): + """Example usage of aggregation functions.""" + if len(sys.argv) < 3: + print("Usage: python3 example_aggregation.py ") + sys.exit(1) + + host = sys.argv[1] + port = int(sys.argv[2]) + + print("=== Example: Basic Metrics Query ===") + # Get all metrics for demonstration + all_metrics = run_metrics_query( + host, port, + start_time="1999-01-01T00:00:00", + end_time="2099-01-01T00:00:00" + ) + + if not all_metrics: + print("No metrics found or query failed.") + return + + print(f"Found {len(all_metrics)} total metrics") + + print("\n=== Example: Aggregation by Category ===") + category_agg = aggregate_by_category(all_metrics) + for category, stats in category_agg.items(): + print(f"\nCategory: {category}") + for stat_name, stat_data in stats.items(): + print(f" {stat_name}: avg={stat_data['avg']:.2f}, sum={stat_data['sum']}, count={stat_data['count']}") + + print("\n=== Example: Aggregation by User Dimension ===") + user_agg = aggregate_by_dimension(all_metrics, 'user') + for user, stats in user_agg.items(): + print(f"\nUser: {user}") + for stat_name, stat_data in stats.items(): + print(f" {stat_name}: avg={stat_data['avg']:.2f}, sum={stat_data['sum']}, count={stat_data['count']}") + + print("\n=== Example: Top Performers by TimeLocalExecute ===") + top_performers = find_top_performers(all_metrics, 'TimeLocalExecute', top_n=3) + for i, metric in enumerate(top_performers, 1): + print(f"{i}. Category: {metric.get('category')}, " + f"Dimensions: {metric.get('dimensions')}, " + f"TimeLocalExecute: {metric.get('stats', {}).get('TimeLocalExecute')}") + + print("\n=== Example: Time Series Analysis for TimeLocalExecute ===") + time_analysis = time_series_analysis(all_metrics, 'TimeLocalExecute') + if 'error' not in time_analysis: + print(f"Analyzed {time_analysis['total_points']} data points") + print(f"Value range: {time_analysis['value_stats']['min']} - {time_analysis['value_stats']['max']}") + print(f"Average: {time_analysis['value_stats']['avg']:.2f}") + print(f"Variance: {time_analysis['value_stats']['variance']:.2f}") + else: + print(time_analysis['error']) + + print("\n=== Example: Filtered Query with Specific Statistics ===") + filtered_metrics = run_metrics_query( + host, port, + category="categoryOne", + dimensions={'user': 'alice'}, + stats=['TimeLocalExecute', 'CostExecute'], + start_time="1999-01-01T00:00:00", + end_time="2099-01-01T00:00:00" + ) + + print(f"Filtered query returned {len(filtered_metrics)} metrics") + for metric in filtered_metrics: + print(f" Category: {metric.get('category')}, Stats: {metric.get('stats')}") + + +if __name__ == '__main__': + main() \ No newline at end of file diff --git a/tools/get_global_metrics.py b/tools/get_global_metrics.py new file mode 100755 index 00000000000..f269bd0d92b --- /dev/null +++ b/tools/get_global_metrics.py @@ -0,0 +1,438 @@ +#!/usr/bin/env python3 +""" +Python script to interact with HPCC Platform GetGlobalMetrics service. + +This script provides functionality to query global metrics from the HPCC Platform +ws_machine service, with support for filtering by time range, category, and dimensions. +It can be extended to perform aggregates of certain filtered statistics. + +Usage: + python get_global_metrics.py --host --port [options] + +Example: + python get_global_metrics.py --host localhost --port 8010 \ + --start "2023-01-01T00:00:00" --end "2023-12-31T23:59:59" \ + --category "categoryOne" --dimension user:alice +""" + +import argparse +import json +import sys +import urllib.parse +import urllib.request +import xml.etree.ElementTree as ET +from datetime import datetime +from typing import Dict, List, Optional, Tuple + + +class HPCCGlobalMetricsClient: + """Client for HPCC Platform GetGlobalMetrics service.""" + + def __init__(self, host: str, port: int, username: Optional[str] = None, + password: Optional[str] = None, use_https: bool = False): + """ + Initialize the client. + + Args: + host: HPCC Platform hostname + port: ESP service port + username: Optional username for authentication + password: Optional password for authentication + use_https: Whether to use HTTPS instead of HTTP + """ + self.host = host + self.port = port + self.username = username + self.password = password + self.use_https = use_https + self.base_url = f"{'https' if use_https else 'http'}://{host}:{port}" + + def create_request_xml(self, category: Optional[str] = None, + dimensions: Optional[Dict[str, str]] = None, + start_time: Optional[str] = None, + end_time: Optional[str] = None) -> str: + """ + Create XML request for GetGlobalMetrics service. + + Args: + category: Optional category filter (use 'All' or None for all categories) + dimensions: Optional dictionary of dimension name/value pairs + start_time: Start time in format 'yyyy-mm-ddThh:mm:ss' + end_time: End time in format 'yyyy-mm-ddThh:mm:ss' + + Returns: + XML request string + """ + root = ET.Element("GetGlobalMetricsRequest") + + # Add category if specified + if category and category != "All": + category_elem = ET.SubElement(root, "Category") + category_elem.text = category + + # Add dimensions if specified + if dimensions: + dimensions_elem = ET.SubElement(root, "Dimensions") + for name, value in dimensions.items(): + dim_elem = ET.SubElement(dimensions_elem, "Dimension") + name_elem = ET.SubElement(dim_elem, "Name") + name_elem.text = name + value_elem = ET.SubElement(dim_elem, "Value") + value_elem.text = value + + # Add date/time range if specified + if start_time or end_time: + datetime_range = ET.SubElement(root, "DateTimeRange") + if start_time: + start_elem = ET.SubElement(datetime_range, "Start") + start_elem.text = start_time + if end_time: + end_elem = ET.SubElement(datetime_range, "End") + end_elem.text = end_time + + return ET.tostring(root, encoding='unicode') + + def send_request(self, xml_request: str) -> str: + """ + Send HTTP request to GetGlobalMetrics service. + + Args: + xml_request: XML request payload + + Returns: + XML response string + + Raises: + Exception: If HTTP request fails + """ + url = f"{self.base_url}/WsMachine/GetGlobalMetrics" + + # Prepare request + data = xml_request.encode('utf-8') + req = urllib.request.Request(url, data=data) + req.add_header('Content-Type', 'application/xml') + req.add_header('SOAPAction', '""') + + # Add authentication if provided + if self.username and self.password: + import base64 + credentials = base64.b64encode(f"{self.username}:{self.password}".encode()).decode() + req.add_header('Authorization', f'Basic {credentials}') + + try: + with urllib.request.urlopen(req) as response: + return response.read().decode('utf-8') + except urllib.error.HTTPError as e: + error_msg = f"HTTP Error {e.code}: {e.reason}" + if hasattr(e, 'read'): + error_msg += f"\nResponse: {e.read().decode('utf-8')}" + raise Exception(error_msg) + except Exception as e: + raise Exception(f"Request failed: {str(e)}") + + def parse_response(self, xml_response: str) -> List[Dict]: + """ + Parse XML response into structured data. + + Args: + xml_response: XML response from service + + Returns: + List of metric dictionaries + """ + try: + root = ET.fromstring(xml_response) + metrics = [] + + # Find all GlobalMetric elements + for metric_elem in root.findall('.//GlobalMetric'): + metric = {} + + # Extract category + category_elem = metric_elem.find('Category') + if category_elem is not None: + metric['category'] = category_elem.text + + # Extract dimensions + dimensions = {} + dimensions_elem = metric_elem.find('Dimensions') + if dimensions_elem is not None: + for dim_elem in dimensions_elem.findall('Dimension'): + name_elem = dim_elem.find('Name') + value_elem = dim_elem.find('Value') + if name_elem is not None and value_elem is not None: + dimensions[name_elem.text] = value_elem.text + metric['dimensions'] = dimensions + + # Extract date/time range + datetime_range_elem = metric_elem.find('DateTimeRange') + if datetime_range_elem is not None: + start_elem = datetime_range_elem.find('Start') + end_elem = datetime_range_elem.find('End') + metric['datetime_range'] = { + 'start': start_elem.text if start_elem is not None else None, + 'end': end_elem.text if end_elem is not None else None + } + + # Extract stats + stats = {} + stats_elem = metric_elem.find('Stats') + if stats_elem is not None: + for stat_elem in stats_elem.findall('Stat'): + name_elem = stat_elem.find('Name') + value_elem = stat_elem.find('Value') + if name_elem is not None and value_elem is not None: + try: + # Try to convert to numeric value + stats[name_elem.text] = int(value_elem.text) + except ValueError: + # Keep as string if not numeric + stats[name_elem.text] = value_elem.text + metric['stats'] = stats + + metrics.append(metric) + + return metrics + except ET.ParseError as e: + raise Exception(f"Failed to parse XML response: {str(e)}") + + def get_global_metrics(self, category: Optional[str] = None, + dimensions: Optional[Dict[str, str]] = None, + start_time: Optional[str] = None, + end_time: Optional[str] = None) -> List[Dict]: + """ + Get global metrics from HPCC Platform. + + Args: + category: Optional category filter + dimensions: Optional dimension filters + start_time: Optional start time filter + end_time: Optional end time filter + + Returns: + List of metric dictionaries + """ + xml_request = self.create_request_xml(category, dimensions, start_time, end_time) + xml_response = self.send_request(xml_request) + return self.parse_response(xml_response) + + def filter_stats(self, metrics: List[Dict], stat_names: List[str]) -> List[Dict]: + """ + Filter metrics to include only specified statistics. + + Args: + metrics: List of metric dictionaries + stat_names: List of statistic names to include + + Returns: + Filtered list of metric dictionaries + """ + filtered_metrics = [] + for metric in metrics: + filtered_stats = {name: value for name, value in metric.get('stats', {}).items() + if name in stat_names} + if filtered_stats: # Only include metrics that have at least one of the requested stats + filtered_metric = metric.copy() + filtered_metric['stats'] = filtered_stats + filtered_metrics.append(filtered_metric) + return filtered_metrics + + +def parse_dimensions(dimension_args: List[str]) -> Dict[str, str]: + """ + Parse dimension arguments in format 'name:value'. + + Args: + dimension_args: List of dimension strings + + Returns: + Dictionary of dimension name/value pairs + """ + dimensions = {} + for dim_arg in dimension_args: + if ':' not in dim_arg: + raise ValueError(f"Invalid dimension format: {dim_arg}. Expected 'name:value'") + name, value = dim_arg.split(':', 1) + dimensions[name.strip()] = value.strip() + return dimensions + + +def format_output(metrics: List[Dict], output_format: str) -> str: + """ + Format metrics for output. + + Args: + metrics: List of metric dictionaries + output_format: Output format ('json', 'table', 'csv') + + Returns: + Formatted output string + """ + if output_format == 'json': + return json.dumps(metrics, indent=2) + + elif output_format == 'table': + if not metrics: + return "No metrics found." + + output = [] + output.append("Global Metrics:") + output.append("=" * 50) + + for i, metric in enumerate(metrics): + output.append(f"\nMetric {i + 1}:") + output.append(f" Category: {metric.get('category', 'N/A')}") + + dimensions = metric.get('dimensions', {}) + if dimensions: + output.append(" Dimensions:") + for name, value in dimensions.items(): + output.append(f" {name}: {value}") + + datetime_range = metric.get('datetime_range', {}) + if datetime_range.get('start') or datetime_range.get('end'): + output.append(" Time Range:") + if datetime_range.get('start'): + output.append(f" Start: {datetime_range['start']}") + if datetime_range.get('end'): + output.append(f" End: {datetime_range['end']}") + + stats = metric.get('stats', {}) + if stats: + output.append(" Statistics:") + for name, value in stats.items(): + output.append(f" {name}: {value}") + + return '\n'.join(output) + + elif output_format == 'csv': + if not metrics: + return "category,dimensions,start_time,end_time,stat_name,stat_value" + + lines = ["category,dimensions,start_time,end_time,stat_name,stat_value"] + + for metric in metrics: + category = metric.get('category', '') + dimensions_str = ';'.join([f"{k}={v}" for k, v in metric.get('dimensions', {}).items()]) + datetime_range = metric.get('datetime_range', {}) + start_time = datetime_range.get('start', '') + end_time = datetime_range.get('end', '') + + for stat_name, stat_value in metric.get('stats', {}).items(): + lines.append(f"{category},{dimensions_str},{start_time},{end_time},{stat_name},{stat_value}") + + return '\n'.join(lines) + + else: + raise ValueError(f"Unsupported output format: {output_format}") + + +def main(): + """Main entry point.""" + parser = argparse.ArgumentParser( + description="Query HPCC Platform global metrics", + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=""" +Examples: + # Get all metrics for all categories + %(prog)s --host localhost --port 8010 + + # Get metrics for specific category and time range + %(prog)s --host localhost --port 8010 \\ + --category "categoryOne" \\ + --start "2023-01-01T00:00:00" \\ + --end "2023-12-31T23:59:59" + + # Get metrics with dimension filter + %(prog)s --host localhost --port 8010 \\ + --dimension user:alice \\ + --dimension cluster:thor1 + + # Filter specific statistics and output as JSON + %(prog)s --host localhost --port 8010 \\ + --stats TimeLocalExecute CostExecute \\ + --format json + """) + + parser.add_argument('--host', required=True, + help='HPCC Platform hostname') + parser.add_argument('--port', type=int, required=True, + help='ESP service port') + parser.add_argument('--username', + help='Username for authentication') + parser.add_argument('--password', + help='Password for authentication') + parser.add_argument('--https', action='store_true', + help='Use HTTPS instead of HTTP') + + parser.add_argument('--category', + help='Category filter (use "All" or omit for all categories)') + parser.add_argument('--dimension', action='append', dest='dimensions', + help='Dimension filter in format "name:value" (can be specified multiple times)') + parser.add_argument('--start', + help='Start time in format "yyyy-mm-ddThh:mm:ss"') + parser.add_argument('--end', + help='End time in format "yyyy-mm-ddThh:mm:ss"') + + parser.add_argument('--stats', nargs='*', + help='Filter to include only specified statistics') + parser.add_argument('--format', choices=['json', 'table', 'csv'], default='table', + help='Output format (default: table)') + + parser.add_argument('--verbose', '-v', action='store_true', + help='Enable verbose output') + + args = parser.parse_args() + + try: + # Parse dimensions + dimensions = None + if args.dimensions: + dimensions = parse_dimensions(args.dimensions) + + # Create client + client = HPCCGlobalMetricsClient( + host=args.host, + port=args.port, + username=args.username, + password=args.password, + use_https=args.https + ) + + if args.verbose: + print(f"Connecting to {client.base_url}", file=sys.stderr) + if args.category: + print(f"Category filter: {args.category}", file=sys.stderr) + if dimensions: + print(f"Dimension filters: {dimensions}", file=sys.stderr) + if args.start or args.end: + print(f"Time range: {args.start} to {args.end}", file=sys.stderr) + + # Get metrics + metrics = client.get_global_metrics( + category=args.category, + dimensions=dimensions, + start_time=args.start, + end_time=args.end + ) + + # Filter statistics if requested + if args.stats: + metrics = client.filter_stats(metrics, args.stats) + if args.verbose: + print(f"Filtered to statistics: {args.stats}", file=sys.stderr) + + # Output results + output = format_output(metrics, args.format) + print(output) + + if args.verbose: + print(f"Found {len(metrics)} metric(s)", file=sys.stderr) + + except Exception as e: + print(f"Error: {str(e)}", file=sys.stderr) + sys.exit(1) + + +if __name__ == '__main__': + main() \ No newline at end of file