From f33b5d2efeedf502d2c1bb35f24f1402ecc39978 Mon Sep 17 00:00:00 2001
From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com>
Date: Mon, 22 Sep 2025 11:43:53 +0000
Subject: [PATCH 1/2] Initial plan
From b224c92748c25a5f22d9612b16228cc1fd8269b1 Mon Sep 17 00:00:00 2001
From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com>
Date: Mon, 22 Sep 2025 11:51:19 +0000
Subject: [PATCH 2/2] Add Python script to interact with GetGlobalMetrics
service
Co-authored-by: jakesmith <902700+jakesmith@users.noreply.github.com>
---
testing/unittests/test_get_global_metrics.py | 232 ++++++++++
tools/README.md | 221 ++++++++++
tools/example_aggregation.py | 272 ++++++++++++
tools/get_global_metrics.py | 438 +++++++++++++++++++
4 files changed, 1163 insertions(+)
create mode 100644 testing/unittests/test_get_global_metrics.py
create mode 100644 tools/README.md
create mode 100755 tools/example_aggregation.py
create mode 100755 tools/get_global_metrics.py
diff --git a/testing/unittests/test_get_global_metrics.py b/testing/unittests/test_get_global_metrics.py
new file mode 100644
index 00000000000..88730e545bd
--- /dev/null
+++ b/testing/unittests/test_get_global_metrics.py
@@ -0,0 +1,232 @@
+#!/usr/bin/env python3
+"""
+Unit tests for get_global_metrics.py script.
+
+This test module verifies the XML request generation and response parsing
+functionality of the HPCCGlobalMetricsClient without requiring a live HPCC server.
+"""
+
+import unittest
+import sys
+import os
+
+# Add the tools directory to the path to import the script
+tools_path = os.path.join(os.path.dirname(__file__), '..', '..', 'tools')
+sys.path.insert(0, tools_path)
+
+from get_global_metrics import HPCCGlobalMetricsClient, parse_dimensions, format_output
+
+
+class TestHPCCGlobalMetricsClient(unittest.TestCase):
+ """Test cases for HPCCGlobalMetricsClient."""
+
+ def setUp(self):
+ """Set up test client."""
+ self.client = HPCCGlobalMetricsClient("localhost", 8010)
+
+ def test_create_request_xml_minimal(self):
+ """Test creating minimal XML request."""
+ xml = self.client.create_request_xml()
+ expected = ""
+ self.assertEqual(xml.replace('>', ' />'), expected.replace('>', ' />'))
+
+ def test_create_request_xml_with_category(self):
+ """Test creating XML request with category."""
+ xml = self.client.create_request_xml(category="categoryOne")
+ self.assertIn("categoryOne", xml)
+ self.assertIn("", xml)
+
+ def test_create_request_xml_with_all_category(self):
+ """Test creating XML request with 'All' category (should be omitted)."""
+ xml = self.client.create_request_xml(category="All")
+ self.assertNotIn("", xml)
+ self.assertIn("", xml)
+ self.assertIn("", xml)
+ self.assertIn("user", xml)
+ self.assertIn("alice", xml)
+ self.assertIn("cluster", xml)
+ self.assertIn("thor1", xml)
+
+ def test_create_request_xml_with_datetime_range(self):
+ """Test creating XML request with date/time range."""
+ xml = self.client.create_request_xml(
+ start_time="1999-01-01T00:00:00",
+ end_time="2099-01-01T00:00:00"
+ )
+
+ self.assertIn("", xml)
+ self.assertIn("1999-01-01T00:00:00", xml)
+ self.assertIn("2099-01-01T00:00:00", xml)
+
+ def test_create_request_xml_complete(self):
+ """Test creating complete XML request with all parameters."""
+ dimensions = {"user": "bob", "cluster": "thor1"}
+ xml = self.client.create_request_xml(
+ category="categoryOne",
+ dimensions=dimensions,
+ start_time="1999-01-01T00:00:00",
+ end_time="2099-01-01T00:00:00"
+ )
+
+ # Verify all elements are present
+ self.assertIn("", xml)
+ self.assertIn("categoryOne", xml)
+ self.assertIn("", xml)
+ self.assertIn("user", xml)
+ self.assertIn("bob", xml)
+ self.assertIn("", xml)
+ self.assertIn("1999-01-01T00:00:00", xml)
+ self.assertIn("2099-01-01T00:00:00", xml)
+
+ def test_parse_response_based_on_cppunit_test(self):
+ """Test parsing response based on the expected format from cppunit tests."""
+ # This is the expected XML from the cppunit test case
+ xml_response = """
+
+ categoryTwo
+ 19990701121999070112
+ TimeLocalExecute444
+
+
+ categoryOne
+
+ useralice
+
+ 19990701121999070112
+
+ TimeLocalExecute222
+ CostExecute5
+
+
+ """
+
+ metrics = self.client.parse_response(xml_response)
+
+ # Verify we got 2 metrics
+ self.assertEqual(len(metrics), 2)
+
+ # Verify first metric
+ metric1 = metrics[0]
+ self.assertEqual(metric1['category'], 'categoryTwo')
+ self.assertEqual(metric1['dimensions'], {})
+ self.assertEqual(metric1['datetime_range']['start'], '1999070112')
+ self.assertEqual(metric1['datetime_range']['end'], '1999070112')
+ self.assertEqual(metric1['stats']['TimeLocalExecute'], 444)
+
+ # Verify second metric
+ metric2 = metrics[1]
+ self.assertEqual(metric2['category'], 'categoryOne')
+ self.assertEqual(metric2['dimensions'], {'user': 'alice'})
+ self.assertEqual(metric2['datetime_range']['start'], '1999070112')
+ self.assertEqual(metric2['datetime_range']['end'], '1999070112')
+ self.assertEqual(metric2['stats']['TimeLocalExecute'], 222)
+ self.assertEqual(metric2['stats']['CostExecute'], 5)
+
+ def test_filter_stats(self):
+ """Test filtering statistics."""
+ metrics = [
+ {
+ 'category': 'test',
+ 'dimensions': {},
+ 'stats': {
+ 'TimeLocalExecute': 100,
+ 'CostExecute': 5,
+ 'OtherStat': 200
+ }
+ },
+ {
+ 'category': 'test2',
+ 'dimensions': {},
+ 'stats': {
+ 'TimeLocalExecute': 150,
+ 'UnwantedStat': 999
+ }
+ }
+ ]
+
+ filtered = self.client.filter_stats(metrics, ['TimeLocalExecute', 'CostExecute'])
+
+ # Verify filtering
+ self.assertEqual(len(filtered), 2)
+ self.assertEqual(filtered[0]['stats'], {'TimeLocalExecute': 100, 'CostExecute': 5})
+ self.assertEqual(filtered[1]['stats'], {'TimeLocalExecute': 150})
+
+
+class TestUtilityFunctions(unittest.TestCase):
+ """Test utility functions."""
+
+ def test_parse_dimensions(self):
+ """Test parsing dimension arguments."""
+ dimensions = parse_dimensions(['user:alice', 'cluster:thor1', 'key:value with spaces'])
+ expected = {
+ 'user': 'alice',
+ 'cluster': 'thor1',
+ 'key': 'value with spaces'
+ }
+ self.assertEqual(dimensions, expected)
+
+ def test_parse_dimensions_invalid_format(self):
+ """Test parsing dimensions with invalid format."""
+ with self.assertRaises(ValueError):
+ parse_dimensions(['invalid_format'])
+
+ def test_format_output_json(self):
+ """Test JSON output formatting."""
+ metrics = [{'category': 'test', 'stats': {'count': 42}}]
+ output = format_output(metrics, 'json')
+ self.assertIn('"category": "test"', output)
+ self.assertIn('"count": 42', output)
+
+ def test_format_output_table(self):
+ """Test table output formatting."""
+ metrics = [
+ {
+ 'category': 'test',
+ 'dimensions': {'user': 'alice'},
+ 'datetime_range': {'start': '2023-01-01', 'end': '2023-12-31'},
+ 'stats': {'count': 42}
+ }
+ ]
+ output = format_output(metrics, 'table')
+ self.assertIn('Global Metrics:', output)
+ self.assertIn('Category: test', output)
+ self.assertIn('user: alice', output)
+ self.assertIn('count: 42', output)
+
+ def test_format_output_csv(self):
+ """Test CSV output formatting."""
+ metrics = [
+ {
+ 'category': 'test',
+ 'dimensions': {'user': 'alice'},
+ 'datetime_range': {'start': '2023-01-01', 'end': '2023-12-31'},
+ 'stats': {'count': 42, 'total': 100}
+ }
+ ]
+ output = format_output(metrics, 'csv')
+ lines = output.strip().split('\n')
+
+ # Check header
+ self.assertEqual(lines[0], 'category,dimensions,start_time,end_time,stat_name,stat_value')
+
+ # Check data lines
+ self.assertEqual(len(lines), 3) # Header + 2 stat lines
+ self.assertIn('test,user=alice,2023-01-01,2023-12-31,count,42', lines)
+ self.assertIn('test,user=alice,2023-01-01,2023-12-31,total,100', lines)
+
+ def test_format_output_empty_metrics(self):
+ """Test output formatting with empty metrics."""
+ self.assertEqual(format_output([], 'table'), "No metrics found.")
+ self.assertEqual(format_output([], 'json'), "[]")
+ self.assertIn('category,dimensions', format_output([], 'csv'))
+
+
+if __name__ == '__main__':
+ unittest.main()
\ No newline at end of file
diff --git a/tools/README.md b/tools/README.md
new file mode 100644
index 00000000000..49f25a41624
--- /dev/null
+++ b/tools/README.md
@@ -0,0 +1,221 @@
+# Global Metrics Python Client
+
+This directory contains a Python script to interact with the HPCC Platform GetGlobalMetrics service in ws_machines.
+
+## Files
+
+- `get_global_metrics.py` - Main Python script for querying global metrics
+- `../testing/unittests/test_get_global_metrics.py` - Unit tests for the script
+
+## Features
+
+The `get_global_metrics.py` script provides:
+
+1. **Time Range Filtering**: Query metrics for a specific date/time range
+2. **Category Filtering**: Filter by specific metric categories
+3. **Dimension Filtering**: Filter by dimension name/value pairs (e.g., user, cluster)
+4. **Statistics Selection**: Extract only specific statistics from the results
+5. **Multiple Output Formats**: JSON, table, and CSV output formats
+6. **Authentication Support**: Basic HTTP authentication
+7. **HTTPS Support**: Secure connections to HPCC Platform
+
+## Usage
+
+### Basic Usage
+
+```bash
+# Get all metrics
+python3 get_global_metrics.py --host localhost --port 8010
+
+# Get help
+python3 get_global_metrics.py --help
+```
+
+### Time Range Filtering
+
+```bash
+# Query metrics for a specific time range
+python3 get_global_metrics.py --host localhost --port 8010 \
+ --start "2023-01-01T00:00:00" \
+ --end "2023-12-31T23:59:59"
+```
+
+### Category and Dimension Filtering
+
+```bash
+# Filter by category
+python3 get_global_metrics.py --host localhost --port 8010 \
+ --category "categoryOne"
+
+# Filter by dimensions
+python3 get_global_metrics.py --host localhost --port 8010 \
+ --dimension user:alice \
+ --dimension cluster:thor1
+
+# Combine filters
+python3 get_global_metrics.py --host localhost --port 8010 \
+ --category "categoryOne" \
+ --dimension user:bob \
+ --start "2023-01-01T00:00:00" \
+ --end "2023-12-31T23:59:59"
+```
+
+### Statistics Selection and Output Formatting
+
+```bash
+# Select specific statistics and output as JSON
+python3 get_global_metrics.py --host localhost --port 8010 \
+ --stats TimeLocalExecute CostExecute \
+ --format json
+
+# Output as CSV for further processing
+python3 get_global_metrics.py --host localhost --port 8010 \
+ --format csv > metrics.csv
+```
+
+### Authentication
+
+```bash
+# With authentication
+python3 get_global_metrics.py --host remote-host --port 8010 \
+ --username myuser --password mypass \
+ --https
+```
+
+## Output Formats
+
+### Table Format (Default)
+Human-readable tabular output showing all metric details.
+
+### JSON Format
+Structured JSON output suitable for programmatic processing:
+
+```json
+[
+ {
+ "category": "categoryOne",
+ "dimensions": {
+ "user": "alice",
+ "cluster": "thor1"
+ },
+ "datetime_range": {
+ "start": "1999070112",
+ "end": "1999070112"
+ },
+ "stats": {
+ "TimeLocalExecute": 222,
+ "CostExecute": 5
+ }
+ }
+]
+```
+
+### CSV Format
+Comma-separated values for spreadsheet processing:
+
+```csv
+category,dimensions,start_time,end_time,stat_name,stat_value
+categoryOne,user=alice;cluster=thor1,1999070112,1999070112,TimeLocalExecute,222
+categoryOne,user=alice;cluster=thor1,1999070112,1999070112,CostExecute,5
+```
+
+## Service Interface
+
+The script communicates with the HPCC Platform GetGlobalMetrics service:
+
+- **URL**: `http(s)://host:port/WsMachine/GetGlobalMetrics`
+- **Method**: HTTP POST
+- **Content-Type**: application/xml
+- **Authentication**: Basic HTTP authentication (optional)
+
+### Request Format
+
+```xml
+
+ categoryOne
+
+
+ user
+ alice
+
+
+
+ 1999-01-01T00:00:00
+ 2099-01-01T00:00:00
+
+
+```
+
+### Response Format
+
+```xml
+
+
+
+ categoryOne
+
+
+ user
+ alice
+
+
+
+ 1999070112
+ 1999070112
+
+
+
+ TimeLocalExecute
+ 222
+
+
+ CostExecute
+ 5
+
+
+
+
+
+```
+
+## Future Extensions
+
+The script is designed to be easily extended with additional functionality:
+
+1. **Aggregation Functions**: Add support for aggregating statistics (sum, average, min, max)
+2. **Filtering**: Advanced filtering capabilities (value ranges, regex patterns)
+3. **Caching**: Optional response caching for repeated queries
+4. **Batch Processing**: Process multiple queries from configuration files
+5. **Visualization**: Integration with plotting libraries for chart generation
+
+## Testing
+
+Run the unit tests:
+
+```bash
+cd /path/to/HPCC-Platform
+python3 testing/unittests/test_get_global_metrics.py
+```
+
+The tests verify:
+- XML request generation
+- XML response parsing
+- Statistics filtering
+- Output formatting
+- Utility functions
+
+## Dependencies
+
+- Python 3.6+
+- Standard library modules only (no external dependencies)
+
+## Error Handling
+
+The script provides informative error messages for common issues:
+- Connection failures
+- Authentication errors
+- Invalid XML responses
+- Malformed time formats
+- Missing required parameters
+
+Use the `--verbose` flag for additional debugging information.
\ No newline at end of file
diff --git a/tools/example_aggregation.py b/tools/example_aggregation.py
new file mode 100755
index 00000000000..a9a5276a95b
--- /dev/null
+++ b/tools/example_aggregation.py
@@ -0,0 +1,272 @@
+#!/usr/bin/env python3
+"""
+Example script showing how to use get_global_metrics.py for aggregation tasks.
+
+This demonstrates how the script can be extended to perform various aggregations
+of metrics data, which could be built into the main script in future iterations.
+"""
+
+import json
+import subprocess
+import sys
+from collections import defaultdict
+from typing import Dict, List, Any
+
+
+def run_metrics_query(host: str, port: int, **kwargs) -> List[Dict]:
+ """
+ Run get_global_metrics.py and return parsed results.
+
+ Args:
+ host: HPCC hostname
+ port: ESP port
+ **kwargs: Additional arguments for the script
+
+ Returns:
+ List of metric dictionaries
+ """
+ cmd = ['python3', 'get_global_metrics.py', '--host', host, '--port', str(port), '--format', 'json']
+
+ # Add optional arguments
+ for key, value in kwargs.items():
+ if key == 'category' and value:
+ cmd.extend(['--category', value])
+ elif key == 'dimensions' and value:
+ for dim_name, dim_value in value.items():
+ cmd.extend(['--dimension', f'{dim_name}:{dim_value}'])
+ elif key == 'start_time' and value:
+ cmd.extend(['--start', value])
+ elif key == 'end_time' and value:
+ cmd.extend(['--end', value])
+ elif key == 'stats' and value:
+ cmd.extend(['--stats'] + value)
+
+ try:
+ result = subprocess.run(cmd, capture_output=True, text=True, check=True)
+ return json.loads(result.stdout)
+ except subprocess.CalledProcessError as e:
+ print(f"Error running metrics query: {e.stderr}", file=sys.stderr)
+ return []
+ except json.JSONDecodeError as e:
+ print(f"Error parsing JSON response: {e}", file=sys.stderr)
+ return []
+
+
+def aggregate_by_category(metrics: List[Dict]) -> Dict[str, Dict[str, float]]:
+ """
+ Aggregate statistics by category.
+
+ Args:
+ metrics: List of metric dictionaries
+
+ Returns:
+ Dictionary with categories as keys and aggregated stats as values
+ """
+ category_stats = defaultdict(lambda: defaultdict(list))
+
+ for metric in metrics:
+ category = metric.get('category', 'Unknown')
+ stats = metric.get('stats', {})
+
+ for stat_name, stat_value in stats.items():
+ if isinstance(stat_value, (int, float)):
+ category_stats[category][stat_name].append(stat_value)
+
+ # Calculate aggregations
+ result = {}
+ for category, stats in category_stats.items():
+ result[category] = {}
+ for stat_name, values in stats.items():
+ if values:
+ result[category][stat_name] = {
+ 'sum': sum(values),
+ 'avg': sum(values) / len(values),
+ 'min': min(values),
+ 'max': max(values),
+ 'count': len(values)
+ }
+
+ return result
+
+
+def aggregate_by_dimension(metrics: List[Dict], dimension_name: str) -> Dict[str, Dict[str, float]]:
+ """
+ Aggregate statistics by a specific dimension.
+
+ Args:
+ metrics: List of metric dictionaries
+ dimension_name: Name of dimension to aggregate by
+
+ Returns:
+ Dictionary with dimension values as keys and aggregated stats as values
+ """
+ dimension_stats = defaultdict(lambda: defaultdict(list))
+
+ for metric in metrics:
+ dimensions = metric.get('dimensions', {})
+ dimension_value = dimensions.get(dimension_name, 'Unknown')
+ stats = metric.get('stats', {})
+
+ for stat_name, stat_value in stats.items():
+ if isinstance(stat_value, (int, float)):
+ dimension_stats[dimension_value][stat_name].append(stat_value)
+
+ # Calculate aggregations
+ result = {}
+ for dim_value, stats in dimension_stats.items():
+ result[dim_value] = {}
+ for stat_name, values in stats.items():
+ if values:
+ result[dim_value][stat_name] = {
+ 'sum': sum(values),
+ 'avg': sum(values) / len(values),
+ 'min': min(values),
+ 'max': max(values),
+ 'count': len(values)
+ }
+
+ return result
+
+
+def find_top_performers(metrics: List[Dict], stat_name: str, top_n: int = 5) -> List[Dict]:
+ """
+ Find top performing metrics by a specific statistic.
+
+ Args:
+ metrics: List of metric dictionaries
+ stat_name: Name of statistic to rank by
+ top_n: Number of top performers to return
+
+ Returns:
+ List of top performing metrics
+ """
+ # Filter metrics that have the requested statistic
+ valid_metrics = []
+ for metric in metrics:
+ stats = metric.get('stats', {})
+ if stat_name in stats and isinstance(stats[stat_name], (int, float)):
+ valid_metrics.append((stats[stat_name], metric))
+
+ # Sort by statistic value (descending)
+ valid_metrics.sort(key=lambda x: x[0], reverse=True)
+
+ # Return top N metrics
+ return [metric for _, metric in valid_metrics[:top_n]]
+
+
+def time_series_analysis(metrics: List[Dict], stat_name: str) -> Dict[str, Any]:
+ """
+ Analyze time series data for a specific statistic.
+
+ Args:
+ metrics: List of metric dictionaries
+ stat_name: Name of statistic to analyze
+
+ Returns:
+ Dictionary with time series analysis results
+ """
+ time_points = []
+ values = []
+
+ for metric in metrics:
+ stats = metric.get('stats', {})
+ datetime_range = metric.get('datetime_range', {})
+
+ if stat_name in stats and isinstance(stats[stat_name], (int, float)):
+ start_time = datetime_range.get('start')
+ if start_time:
+ time_points.append(start_time)
+ values.append(stats[stat_name])
+
+ if not values:
+ return {'error': f'No data found for statistic: {stat_name}'}
+
+ # Simple time series analysis
+ return {
+ 'stat_name': stat_name,
+ 'total_points': len(values),
+ 'time_range': {
+ 'start': min(time_points) if time_points else None,
+ 'end': max(time_points) if time_points else None
+ },
+ 'value_stats': {
+ 'sum': sum(values),
+ 'avg': sum(values) / len(values),
+ 'min': min(values),
+ 'max': max(values),
+ 'variance': sum((x - sum(values) / len(values)) ** 2 for x in values) / len(values)
+ }
+ }
+
+
+def main():
+ """Example usage of aggregation functions."""
+ if len(sys.argv) < 3:
+ print("Usage: python3 example_aggregation.py ")
+ sys.exit(1)
+
+ host = sys.argv[1]
+ port = int(sys.argv[2])
+
+ print("=== Example: Basic Metrics Query ===")
+ # Get all metrics for demonstration
+ all_metrics = run_metrics_query(
+ host, port,
+ start_time="1999-01-01T00:00:00",
+ end_time="2099-01-01T00:00:00"
+ )
+
+ if not all_metrics:
+ print("No metrics found or query failed.")
+ return
+
+ print(f"Found {len(all_metrics)} total metrics")
+
+ print("\n=== Example: Aggregation by Category ===")
+ category_agg = aggregate_by_category(all_metrics)
+ for category, stats in category_agg.items():
+ print(f"\nCategory: {category}")
+ for stat_name, stat_data in stats.items():
+ print(f" {stat_name}: avg={stat_data['avg']:.2f}, sum={stat_data['sum']}, count={stat_data['count']}")
+
+ print("\n=== Example: Aggregation by User Dimension ===")
+ user_agg = aggregate_by_dimension(all_metrics, 'user')
+ for user, stats in user_agg.items():
+ print(f"\nUser: {user}")
+ for stat_name, stat_data in stats.items():
+ print(f" {stat_name}: avg={stat_data['avg']:.2f}, sum={stat_data['sum']}, count={stat_data['count']}")
+
+ print("\n=== Example: Top Performers by TimeLocalExecute ===")
+ top_performers = find_top_performers(all_metrics, 'TimeLocalExecute', top_n=3)
+ for i, metric in enumerate(top_performers, 1):
+ print(f"{i}. Category: {metric.get('category')}, "
+ f"Dimensions: {metric.get('dimensions')}, "
+ f"TimeLocalExecute: {metric.get('stats', {}).get('TimeLocalExecute')}")
+
+ print("\n=== Example: Time Series Analysis for TimeLocalExecute ===")
+ time_analysis = time_series_analysis(all_metrics, 'TimeLocalExecute')
+ if 'error' not in time_analysis:
+ print(f"Analyzed {time_analysis['total_points']} data points")
+ print(f"Value range: {time_analysis['value_stats']['min']} - {time_analysis['value_stats']['max']}")
+ print(f"Average: {time_analysis['value_stats']['avg']:.2f}")
+ print(f"Variance: {time_analysis['value_stats']['variance']:.2f}")
+ else:
+ print(time_analysis['error'])
+
+ print("\n=== Example: Filtered Query with Specific Statistics ===")
+ filtered_metrics = run_metrics_query(
+ host, port,
+ category="categoryOne",
+ dimensions={'user': 'alice'},
+ stats=['TimeLocalExecute', 'CostExecute'],
+ start_time="1999-01-01T00:00:00",
+ end_time="2099-01-01T00:00:00"
+ )
+
+ print(f"Filtered query returned {len(filtered_metrics)} metrics")
+ for metric in filtered_metrics:
+ print(f" Category: {metric.get('category')}, Stats: {metric.get('stats')}")
+
+
+if __name__ == '__main__':
+ main()
\ No newline at end of file
diff --git a/tools/get_global_metrics.py b/tools/get_global_metrics.py
new file mode 100755
index 00000000000..f269bd0d92b
--- /dev/null
+++ b/tools/get_global_metrics.py
@@ -0,0 +1,438 @@
+#!/usr/bin/env python3
+"""
+Python script to interact with HPCC Platform GetGlobalMetrics service.
+
+This script provides functionality to query global metrics from the HPCC Platform
+ws_machine service, with support for filtering by time range, category, and dimensions.
+It can be extended to perform aggregates of certain filtered statistics.
+
+Usage:
+ python get_global_metrics.py --host --port [options]
+
+Example:
+ python get_global_metrics.py --host localhost --port 8010 \
+ --start "2023-01-01T00:00:00" --end "2023-12-31T23:59:59" \
+ --category "categoryOne" --dimension user:alice
+"""
+
+import argparse
+import json
+import sys
+import urllib.parse
+import urllib.request
+import xml.etree.ElementTree as ET
+from datetime import datetime
+from typing import Dict, List, Optional, Tuple
+
+
+class HPCCGlobalMetricsClient:
+ """Client for HPCC Platform GetGlobalMetrics service."""
+
+ def __init__(self, host: str, port: int, username: Optional[str] = None,
+ password: Optional[str] = None, use_https: bool = False):
+ """
+ Initialize the client.
+
+ Args:
+ host: HPCC Platform hostname
+ port: ESP service port
+ username: Optional username for authentication
+ password: Optional password for authentication
+ use_https: Whether to use HTTPS instead of HTTP
+ """
+ self.host = host
+ self.port = port
+ self.username = username
+ self.password = password
+ self.use_https = use_https
+ self.base_url = f"{'https' if use_https else 'http'}://{host}:{port}"
+
+ def create_request_xml(self, category: Optional[str] = None,
+ dimensions: Optional[Dict[str, str]] = None,
+ start_time: Optional[str] = None,
+ end_time: Optional[str] = None) -> str:
+ """
+ Create XML request for GetGlobalMetrics service.
+
+ Args:
+ category: Optional category filter (use 'All' or None for all categories)
+ dimensions: Optional dictionary of dimension name/value pairs
+ start_time: Start time in format 'yyyy-mm-ddThh:mm:ss'
+ end_time: End time in format 'yyyy-mm-ddThh:mm:ss'
+
+ Returns:
+ XML request string
+ """
+ root = ET.Element("GetGlobalMetricsRequest")
+
+ # Add category if specified
+ if category and category != "All":
+ category_elem = ET.SubElement(root, "Category")
+ category_elem.text = category
+
+ # Add dimensions if specified
+ if dimensions:
+ dimensions_elem = ET.SubElement(root, "Dimensions")
+ for name, value in dimensions.items():
+ dim_elem = ET.SubElement(dimensions_elem, "Dimension")
+ name_elem = ET.SubElement(dim_elem, "Name")
+ name_elem.text = name
+ value_elem = ET.SubElement(dim_elem, "Value")
+ value_elem.text = value
+
+ # Add date/time range if specified
+ if start_time or end_time:
+ datetime_range = ET.SubElement(root, "DateTimeRange")
+ if start_time:
+ start_elem = ET.SubElement(datetime_range, "Start")
+ start_elem.text = start_time
+ if end_time:
+ end_elem = ET.SubElement(datetime_range, "End")
+ end_elem.text = end_time
+
+ return ET.tostring(root, encoding='unicode')
+
+ def send_request(self, xml_request: str) -> str:
+ """
+ Send HTTP request to GetGlobalMetrics service.
+
+ Args:
+ xml_request: XML request payload
+
+ Returns:
+ XML response string
+
+ Raises:
+ Exception: If HTTP request fails
+ """
+ url = f"{self.base_url}/WsMachine/GetGlobalMetrics"
+
+ # Prepare request
+ data = xml_request.encode('utf-8')
+ req = urllib.request.Request(url, data=data)
+ req.add_header('Content-Type', 'application/xml')
+ req.add_header('SOAPAction', '""')
+
+ # Add authentication if provided
+ if self.username and self.password:
+ import base64
+ credentials = base64.b64encode(f"{self.username}:{self.password}".encode()).decode()
+ req.add_header('Authorization', f'Basic {credentials}')
+
+ try:
+ with urllib.request.urlopen(req) as response:
+ return response.read().decode('utf-8')
+ except urllib.error.HTTPError as e:
+ error_msg = f"HTTP Error {e.code}: {e.reason}"
+ if hasattr(e, 'read'):
+ error_msg += f"\nResponse: {e.read().decode('utf-8')}"
+ raise Exception(error_msg)
+ except Exception as e:
+ raise Exception(f"Request failed: {str(e)}")
+
+ def parse_response(self, xml_response: str) -> List[Dict]:
+ """
+ Parse XML response into structured data.
+
+ Args:
+ xml_response: XML response from service
+
+ Returns:
+ List of metric dictionaries
+ """
+ try:
+ root = ET.fromstring(xml_response)
+ metrics = []
+
+ # Find all GlobalMetric elements
+ for metric_elem in root.findall('.//GlobalMetric'):
+ metric = {}
+
+ # Extract category
+ category_elem = metric_elem.find('Category')
+ if category_elem is not None:
+ metric['category'] = category_elem.text
+
+ # Extract dimensions
+ dimensions = {}
+ dimensions_elem = metric_elem.find('Dimensions')
+ if dimensions_elem is not None:
+ for dim_elem in dimensions_elem.findall('Dimension'):
+ name_elem = dim_elem.find('Name')
+ value_elem = dim_elem.find('Value')
+ if name_elem is not None and value_elem is not None:
+ dimensions[name_elem.text] = value_elem.text
+ metric['dimensions'] = dimensions
+
+ # Extract date/time range
+ datetime_range_elem = metric_elem.find('DateTimeRange')
+ if datetime_range_elem is not None:
+ start_elem = datetime_range_elem.find('Start')
+ end_elem = datetime_range_elem.find('End')
+ metric['datetime_range'] = {
+ 'start': start_elem.text if start_elem is not None else None,
+ 'end': end_elem.text if end_elem is not None else None
+ }
+
+ # Extract stats
+ stats = {}
+ stats_elem = metric_elem.find('Stats')
+ if stats_elem is not None:
+ for stat_elem in stats_elem.findall('Stat'):
+ name_elem = stat_elem.find('Name')
+ value_elem = stat_elem.find('Value')
+ if name_elem is not None and value_elem is not None:
+ try:
+ # Try to convert to numeric value
+ stats[name_elem.text] = int(value_elem.text)
+ except ValueError:
+ # Keep as string if not numeric
+ stats[name_elem.text] = value_elem.text
+ metric['stats'] = stats
+
+ metrics.append(metric)
+
+ return metrics
+ except ET.ParseError as e:
+ raise Exception(f"Failed to parse XML response: {str(e)}")
+
+ def get_global_metrics(self, category: Optional[str] = None,
+ dimensions: Optional[Dict[str, str]] = None,
+ start_time: Optional[str] = None,
+ end_time: Optional[str] = None) -> List[Dict]:
+ """
+ Get global metrics from HPCC Platform.
+
+ Args:
+ category: Optional category filter
+ dimensions: Optional dimension filters
+ start_time: Optional start time filter
+ end_time: Optional end time filter
+
+ Returns:
+ List of metric dictionaries
+ """
+ xml_request = self.create_request_xml(category, dimensions, start_time, end_time)
+ xml_response = self.send_request(xml_request)
+ return self.parse_response(xml_response)
+
+ def filter_stats(self, metrics: List[Dict], stat_names: List[str]) -> List[Dict]:
+ """
+ Filter metrics to include only specified statistics.
+
+ Args:
+ metrics: List of metric dictionaries
+ stat_names: List of statistic names to include
+
+ Returns:
+ Filtered list of metric dictionaries
+ """
+ filtered_metrics = []
+ for metric in metrics:
+ filtered_stats = {name: value for name, value in metric.get('stats', {}).items()
+ if name in stat_names}
+ if filtered_stats: # Only include metrics that have at least one of the requested stats
+ filtered_metric = metric.copy()
+ filtered_metric['stats'] = filtered_stats
+ filtered_metrics.append(filtered_metric)
+ return filtered_metrics
+
+
+def parse_dimensions(dimension_args: List[str]) -> Dict[str, str]:
+ """
+ Parse dimension arguments in format 'name:value'.
+
+ Args:
+ dimension_args: List of dimension strings
+
+ Returns:
+ Dictionary of dimension name/value pairs
+ """
+ dimensions = {}
+ for dim_arg in dimension_args:
+ if ':' not in dim_arg:
+ raise ValueError(f"Invalid dimension format: {dim_arg}. Expected 'name:value'")
+ name, value = dim_arg.split(':', 1)
+ dimensions[name.strip()] = value.strip()
+ return dimensions
+
+
+def format_output(metrics: List[Dict], output_format: str) -> str:
+ """
+ Format metrics for output.
+
+ Args:
+ metrics: List of metric dictionaries
+ output_format: Output format ('json', 'table', 'csv')
+
+ Returns:
+ Formatted output string
+ """
+ if output_format == 'json':
+ return json.dumps(metrics, indent=2)
+
+ elif output_format == 'table':
+ if not metrics:
+ return "No metrics found."
+
+ output = []
+ output.append("Global Metrics:")
+ output.append("=" * 50)
+
+ for i, metric in enumerate(metrics):
+ output.append(f"\nMetric {i + 1}:")
+ output.append(f" Category: {metric.get('category', 'N/A')}")
+
+ dimensions = metric.get('dimensions', {})
+ if dimensions:
+ output.append(" Dimensions:")
+ for name, value in dimensions.items():
+ output.append(f" {name}: {value}")
+
+ datetime_range = metric.get('datetime_range', {})
+ if datetime_range.get('start') or datetime_range.get('end'):
+ output.append(" Time Range:")
+ if datetime_range.get('start'):
+ output.append(f" Start: {datetime_range['start']}")
+ if datetime_range.get('end'):
+ output.append(f" End: {datetime_range['end']}")
+
+ stats = metric.get('stats', {})
+ if stats:
+ output.append(" Statistics:")
+ for name, value in stats.items():
+ output.append(f" {name}: {value}")
+
+ return '\n'.join(output)
+
+ elif output_format == 'csv':
+ if not metrics:
+ return "category,dimensions,start_time,end_time,stat_name,stat_value"
+
+ lines = ["category,dimensions,start_time,end_time,stat_name,stat_value"]
+
+ for metric in metrics:
+ category = metric.get('category', '')
+ dimensions_str = ';'.join([f"{k}={v}" for k, v in metric.get('dimensions', {}).items()])
+ datetime_range = metric.get('datetime_range', {})
+ start_time = datetime_range.get('start', '')
+ end_time = datetime_range.get('end', '')
+
+ for stat_name, stat_value in metric.get('stats', {}).items():
+ lines.append(f"{category},{dimensions_str},{start_time},{end_time},{stat_name},{stat_value}")
+
+ return '\n'.join(lines)
+
+ else:
+ raise ValueError(f"Unsupported output format: {output_format}")
+
+
+def main():
+ """Main entry point."""
+ parser = argparse.ArgumentParser(
+ description="Query HPCC Platform global metrics",
+ formatter_class=argparse.RawDescriptionHelpFormatter,
+ epilog="""
+Examples:
+ # Get all metrics for all categories
+ %(prog)s --host localhost --port 8010
+
+ # Get metrics for specific category and time range
+ %(prog)s --host localhost --port 8010 \\
+ --category "categoryOne" \\
+ --start "2023-01-01T00:00:00" \\
+ --end "2023-12-31T23:59:59"
+
+ # Get metrics with dimension filter
+ %(prog)s --host localhost --port 8010 \\
+ --dimension user:alice \\
+ --dimension cluster:thor1
+
+ # Filter specific statistics and output as JSON
+ %(prog)s --host localhost --port 8010 \\
+ --stats TimeLocalExecute CostExecute \\
+ --format json
+ """)
+
+ parser.add_argument('--host', required=True,
+ help='HPCC Platform hostname')
+ parser.add_argument('--port', type=int, required=True,
+ help='ESP service port')
+ parser.add_argument('--username',
+ help='Username for authentication')
+ parser.add_argument('--password',
+ help='Password for authentication')
+ parser.add_argument('--https', action='store_true',
+ help='Use HTTPS instead of HTTP')
+
+ parser.add_argument('--category',
+ help='Category filter (use "All" or omit for all categories)')
+ parser.add_argument('--dimension', action='append', dest='dimensions',
+ help='Dimension filter in format "name:value" (can be specified multiple times)')
+ parser.add_argument('--start',
+ help='Start time in format "yyyy-mm-ddThh:mm:ss"')
+ parser.add_argument('--end',
+ help='End time in format "yyyy-mm-ddThh:mm:ss"')
+
+ parser.add_argument('--stats', nargs='*',
+ help='Filter to include only specified statistics')
+ parser.add_argument('--format', choices=['json', 'table', 'csv'], default='table',
+ help='Output format (default: table)')
+
+ parser.add_argument('--verbose', '-v', action='store_true',
+ help='Enable verbose output')
+
+ args = parser.parse_args()
+
+ try:
+ # Parse dimensions
+ dimensions = None
+ if args.dimensions:
+ dimensions = parse_dimensions(args.dimensions)
+
+ # Create client
+ client = HPCCGlobalMetricsClient(
+ host=args.host,
+ port=args.port,
+ username=args.username,
+ password=args.password,
+ use_https=args.https
+ )
+
+ if args.verbose:
+ print(f"Connecting to {client.base_url}", file=sys.stderr)
+ if args.category:
+ print(f"Category filter: {args.category}", file=sys.stderr)
+ if dimensions:
+ print(f"Dimension filters: {dimensions}", file=sys.stderr)
+ if args.start or args.end:
+ print(f"Time range: {args.start} to {args.end}", file=sys.stderr)
+
+ # Get metrics
+ metrics = client.get_global_metrics(
+ category=args.category,
+ dimensions=dimensions,
+ start_time=args.start,
+ end_time=args.end
+ )
+
+ # Filter statistics if requested
+ if args.stats:
+ metrics = client.filter_stats(metrics, args.stats)
+ if args.verbose:
+ print(f"Filtered to statistics: {args.stats}", file=sys.stderr)
+
+ # Output results
+ output = format_output(metrics, args.format)
+ print(output)
+
+ if args.verbose:
+ print(f"Found {len(metrics)} metric(s)", file=sys.stderr)
+
+ except Exception as e:
+ print(f"Error: {str(e)}", file=sys.stderr)
+ sys.exit(1)
+
+
+if __name__ == '__main__':
+ main()
\ No newline at end of file