diff --git a/fastops/__init__.py b/fastops/__init__.py index 3e5abf6..c9d60ce 100644 --- a/fastops/__init__.py +++ b/fastops/__init__.py @@ -11,4 +11,5 @@ from .compliance import * from .secrets import * from .resources import * -from .ship import * \ No newline at end of file +from .ship import * +from .ci import * \ No newline at end of file diff --git a/fastops/ci.py b/fastops/ci.py new file mode 100644 index 0000000..617c856 --- /dev/null +++ b/fastops/ci.py @@ -0,0 +1,592 @@ +"""CI/CD pipeline generation: GitHub Actions, GitLab CI, and deploy-on-push workflows.""" + +# AUTOGENERATED! DO NOT EDIT! File to edit: ../nbs/14_ci.ipynb. + +# %% auto #0 +__all__ = ['github_actions', 'gitlab_ci', 'deploy_workflow', 'test_workflow', 'multi_env_workflow'] + +# %% ../nbs/14_ci.ipynb +import os, json +from pathlib import Path + +# %% ../nbs/14_ci.ipynb +def _yaml_dump(data, path=None): + 'Try PyYAML, fallback to simple YAML-like string generation' + try: + import yaml + result = yaml.dump(data, default_flow_style=False, sort_keys=False, allow_unicode=True) + if path: + Path(path).parent.mkdir(parents=True, exist_ok=True) + Path(path).write_text(result) + return result + except ImportError: + # Fallback: simple YAML-like string generation + def _format(obj, indent=0): + prefix = ' ' * indent + if isinstance(obj, dict): + lines = [] + for k, v in obj.items(): + if v is None: + continue + elif isinstance(v, (dict, list)): + lines.append(f'{prefix}{k}:') + lines.append(_format(v, indent + 1)) + elif isinstance(v, bool): + lines.append(f'{prefix}{k}: {str(v).lower()}') + elif isinstance(v, str) and ('\n' in v or ':' in v): + lines.append(f'{prefix}{k}: |') + for line in v.split('\n'): + lines.append(f'{prefix} {line}') + else: + lines.append(f'{prefix}{k}: {v}') + return '\n'.join(lines) + elif isinstance(obj, list): + lines = [] + for item in obj: + if isinstance(item, dict): + first_key = list(item.keys())[0] if item else '' + lines.append(f'{prefix}- {first_key}: {item[first_key]}') + for k, v in list(item.items())[1:]: + if isinstance(v, (dict, list)): + lines.append(f'{prefix} {k}:') + lines.append(_format(v, indent + 2)) + else: + lines.append(f'{prefix} {k}: {v}') + else: + lines.append(f'{prefix}- {item}') + return '\n'.join(lines) + return str(obj) + + result = _format(data) + if path: + Path(path).parent.mkdir(parents=True, exist_ok=True) + Path(path).write_text(result) + return result + +# %% ../nbs/14_ci.ipynb +def github_actions(name='deploy', app_name='app', **kw): + 'Generate GitHub Actions workflow YAML and optionally save to .github/workflows/' + + # Extract parameters with defaults + trigger = kw.get('trigger', {'push': {'branches': ['main']}}) + python_version = kw.get('python_version', '3.12') + node_version = kw.get('node_version', None) + test_cmd = kw.get('test_cmd', 'python -m pytest') + build = kw.get('build', True) + registry = kw.get('registry', 'ghcr') + deploy_target = kw.get('deploy_target', None) + deploy_host = kw.get('deploy_host', None) + deploy_user = kw.get('deploy_user', 'deploy') + domain = kw.get('domain', None) + env_vars = kw.get('env_vars', {}) + services = kw.get('services', None) + cache = kw.get('cache', True) + lint = kw.get('lint', False) + save = kw.get('save', True) + + # Build workflow structure + workflow = { + 'name': name, + 'on': trigger, + 'jobs': {} + } + + # Job: test + if test_cmd is not None: + test_job = { + 'runs-on': 'ubuntu-latest', + 'steps': [ + {'name': 'Checkout code', 'uses': 'actions/checkout@v4'}, + { + 'name': 'Set up Python', + 'uses': 'actions/setup-python@v5', + 'with': {'python-version': python_version} + } + ] + } + + # Add Node setup if needed + if node_version: + test_job['steps'].append({ + 'name': 'Set up Node.js', + 'uses': 'actions/setup-node@v4', + 'with': {'node-version': node_version} + }) + + # Add cache + if cache: + test_job['steps'].append({ + 'name': 'Cache dependencies', + 'uses': 'actions/cache@v4', + 'with': { + 'path': '~/.cache/pip', + 'key': "${{ runner.os }}-pip-${{ hashFiles('**/requirements*.txt', '**/pyproject.toml') }}", + 'restore-keys': '${{ runner.os }}-pip-' + } + }) + + # Install dependencies + test_job['steps'].append({ + 'name': 'Install dependencies', + 'run': "pip install -e '.[dev]'" + }) + + # Add lint step + if lint: + test_job['steps'].append({ + 'name': 'Lint code', + 'run': 'ruff check . || true' + }) + + # Run tests + test_job['steps'].append({ + 'name': 'Run tests', + 'run': test_cmd + }) + + # Add services if needed + if services: + test_job['services'] = {} + for svc in services: + for svc_name, svc_config in svc.items(): + test_job['services'][svc_name] = svc_config + + workflow['jobs']['test'] = test_job + + # Job: build + if build: + build_job = { + 'runs-on': 'ubuntu-latest', + 'steps': [ + {'name': 'Checkout code', 'uses': 'actions/checkout@v4'} + ] + } + + # Add test dependency + if test_cmd is not None: + build_job['needs'] = 'test' + + # Registry login + if registry == 'ghcr': + build_job['steps'].append({ + 'name': 'Login to GitHub Container Registry', + 'uses': 'docker/login-action@v3', + 'with': { + 'registry': 'ghcr.io', + 'username': '${{ github.actor }}', + 'password': '${{ secrets.GITHUB_TOKEN }}' + } + }) + image_prefix = f'ghcr.io/${{ github.repository_owner }}' + elif registry == 'dockerhub': + build_job['steps'].append({ + 'name': 'Login to Docker Hub', + 'uses': 'docker/login-action@v3', + 'with': { + 'username': '${{ secrets.DOCKERHUB_USERNAME }}', + 'password': '${{ secrets.DOCKERHUB_TOKEN }}' + } + }) + image_prefix = '${{ secrets.DOCKERHUB_USERNAME }}' + elif registry == 'ecr': + build_job['steps'].extend([ + { + 'name': 'Configure AWS credentials', + 'uses': 'aws-actions/configure-aws-credentials@v4', + 'with': { + 'aws-access-key-id': '${{ secrets.AWS_ACCESS_KEY_ID }}', + 'aws-secret-access-key': '${{ secrets.AWS_SECRET_ACCESS_KEY }}', + 'aws-region': '${{ secrets.AWS_REGION }}' + } + }, + { + 'name': 'Login to Amazon ECR', + 'uses': 'aws-actions/amazon-ecr-login@v2' + } + ]) + image_prefix = '${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.${{ secrets.AWS_REGION }}.amazonaws.com' + elif registry == 'acr': + build_job['steps'].extend([ + { + 'name': 'Azure Login', + 'uses': 'azure/login@v2', + 'with': { + 'creds': '${{ secrets.AZURE_CREDENTIALS }}' + } + }, + { + 'name': 'Login to Azure Container Registry', + 'uses': 'azure/docker-login@v2', + 'with': { + 'login-server': '${{ secrets.ACR_LOGIN_SERVER }}', + 'username': '${{ secrets.ACR_USERNAME }}', + 'password': '${{ secrets.ACR_PASSWORD }}' + } + } + ]) + image_prefix = '${{ secrets.ACR_LOGIN_SERVER }}' + + # Build and push + build_job['steps'].append({ + 'name': 'Build and push Docker image', + 'run': f''' +docker build -t {image_prefix}/{app_name}:${{{{ github.sha }}}} . +docker tag {image_prefix}/{app_name}:${{{{ github.sha }}}} {image_prefix}/{app_name}:latest +docker push {image_prefix}/{app_name}:${{{{ github.sha }}}} +docker push {image_prefix}/{app_name}:latest + '''.strip() + }) + + workflow['jobs']['build'] = build_job + + # Job: deploy + if deploy_target is not None: + deploy_job = { + 'runs-on': 'ubuntu-latest', + 'needs': 'build' if build else ('test' if test_cmd is not None else None), + 'steps': [ + {'name': 'Checkout code', 'uses': 'actions/checkout@v4'} + ] + } + + if deploy_target in ('docker', 'vps', 'hetzner'): + # SSH-based deployment + deploy_job['steps'].extend([ + { + 'name': 'Setup SSH', + 'run': f''' +mkdir -p ~/.ssh +echo "${{{{ secrets.SSH_PRIVATE_KEY }}}}" > ~/.ssh/id_rsa +chmod 600 ~/.ssh/id_rsa +ssh-keyscan -H {deploy_host} >> ~/.ssh/known_hosts + '''.strip() + }, + { + 'name': 'Deploy to server', + 'run': f''' +ssh {deploy_user}@{deploy_host} "cd /srv/app && docker compose pull && docker compose up -d" + '''.strip() + } + ]) + elif deploy_target == 'azure': + # Azure deployment + deploy_job['steps'].extend([ + { + 'name': 'Azure Login', + 'uses': 'azure/login@v2', + 'with': { + 'creds': '${{ secrets.AZURE_CREDENTIALS }}' + } + }, + { + 'name': 'Deploy to Azure Web App', + 'uses': 'azure/webapps-deploy@v3', + 'with': { + 'app-name': app_name, + 'images': f'{image_prefix}/{app_name}:${{{{ github.sha }}}}' + } + } + ]) + elif deploy_target == 'aws': + # AWS ECS deployment + deploy_job['steps'].extend([ + { + 'name': 'Configure AWS credentials', + 'uses': 'aws-actions/configure-aws-credentials@v4', + 'with': { + 'aws-access-key-id': '${{ secrets.AWS_ACCESS_KEY_ID }}', + 'aws-secret-access-key': '${{ secrets.AWS_SECRET_ACCESS_KEY }}', + 'aws-region': '${{ secrets.AWS_REGION }}' + } + }, + { + 'name': 'Deploy to ECS', + 'run': f''' +aws ecs update-service --cluster {app_name}-cluster --service {app_name}-service --force-new-deployment + '''.strip() + } + ]) + + # Add environment variables + if env_vars: + deploy_job['env'] = {} + for var_name, secret_name in env_vars.items(): + deploy_job['env'][var_name] = f'${{{{ secrets.{secret_name} }}}}' + + if deploy_job['needs'] is not None: + workflow['jobs']['deploy'] = deploy_job + + # Save to file + if save: + output_path = f'.github/workflows/{name}.yml' + _yaml_dump(workflow, output_path) + + return workflow + +# %% ../nbs/14_ci.ipynb +def gitlab_ci(name='deploy', app_name='app', **kw): + 'Generate GitLab CI configuration and optionally save to .gitlab-ci.yml' + + # Extract parameters with defaults + trigger = kw.get('trigger', {'branches': ['main']}) + python_version = kw.get('python_version', '3.12') + test_cmd = kw.get('test_cmd', 'python -m pytest') + build = kw.get('build', True) + registry = kw.get('registry', 'gitlab') + deploy_target = kw.get('deploy_target', None) + deploy_host = kw.get('deploy_host', None) + deploy_user = kw.get('deploy_user', 'deploy') + services = kw.get('services', None) + lint = kw.get('lint', False) + save = kw.get('save', True) + + # Build GitLab CI structure + config = { + 'stages': [] + } + + # Test stage + if test_cmd is not None: + config['stages'].append('test') + + test_job = { + 'stage': 'test', + 'image': f'python:{python_version}', + 'script': [ + "pip install -e '.[dev]'" + ] + } + + if lint: + test_job['script'].append('ruff check . || true') + + test_job['script'].append(test_cmd) + + # Add services + if services: + test_job['services'] = [] + for svc in services: + for svc_name, svc_config in svc.items(): + test_job['services'].append(svc_config['image']) + + # Add only rule + if 'branches' in trigger: + test_job['only'] = trigger['branches'] + + config['test'] = test_job + + # Build stage + if build: + config['stages'].append('build') + + image_name = f'$CI_REGISTRY_IMAGE/{app_name}' if registry == 'gitlab' else f'{app_name}' + + build_job = { + 'stage': 'build', + 'image': 'docker:latest', + 'services': ['docker:dind'], + 'script': [ + 'docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY', + f'docker build -t {image_name}:$CI_COMMIT_SHA .', + f'docker tag {image_name}:$CI_COMMIT_SHA {image_name}:latest', + f'docker push {image_name}:$CI_COMMIT_SHA', + f'docker push {image_name}:latest' + ] + } + + if 'branches' in trigger: + build_job['only'] = trigger['branches'] + + config['build'] = build_job + + # Deploy stage + if deploy_target is not None: + config['stages'].append('deploy') + + deploy_job = { + 'stage': 'deploy', + 'image': 'alpine:latest', + 'script': [] + } + + if deploy_target in ('docker', 'vps', 'hetzner'): + deploy_job['script'] = [ + 'apk add --no-cache openssh-client', + 'mkdir -p ~/.ssh', + 'echo "$SSH_PRIVATE_KEY" > ~/.ssh/id_rsa', + 'chmod 600 ~/.ssh/id_rsa', + f'ssh-keyscan -H {deploy_host} >> ~/.ssh/known_hosts', + f'ssh {deploy_user}@{deploy_host} "cd /srv/app && docker compose pull && docker compose up -d"' + ] + + if 'branches' in trigger: + deploy_job['only'] = trigger['branches'] + + config['deploy'] = deploy_job + + # Save to file + if save: + output_path = '.gitlab-ci.yml' + _yaml_dump(config, output_path) + + return config + +# %% ../nbs/14_ci.ipynb +def deploy_workflow(app_name='app', target='docker', **kw): + 'Convenience wrapper for deploy-focused GitHub Actions workflow' + return github_actions( + name='deploy', + app_name=app_name, + build=True, + deploy_target=target, + trigger={'push': {'branches': ['main']}, 'workflow_dispatch': {}}, + **kw + ) + +# %% ../nbs/14_ci.ipynb +def test_workflow(app_name='app', **kw): + 'Convenience wrapper for test-only GitHub Actions workflow' + return github_actions( + name='test', + app_name=app_name, + build=False, + deploy_target=None, + trigger={'push': {'branches': ['main', 'develop']}, 'pull_request': {'branches': ['main']}}, + lint=True, + **kw + ) + +# %% ../nbs/14_ci.ipynb +def multi_env_workflow(app_name='app', environments=None, **kw): + 'Generate a workflow with staging → production promotion' + + # Default environments + if environments is None: + environments = { + 'staging': {'branch': 'develop', 'domain': f'staging.{app_name}.com'}, + 'production': {'branch': 'main', 'domain': f'{app_name}.com', 'approval': True} + } + + # Extract common parameters + python_version = kw.get('python_version', '3.12') + test_cmd = kw.get('test_cmd', 'python -m pytest') + registry = kw.get('registry', 'ghcr') + deploy_target = kw.get('deploy_target', 'docker') + save = kw.get('save', True) + + # Build workflow + workflow = { + 'name': 'Multi-Environment Deploy', + 'on': { + 'push': { + 'branches': [env['branch'] for env in environments.values()] + }, + 'workflow_dispatch': {} + }, + 'jobs': {} + } + + # Test job (runs on all branches) + workflow['jobs']['test'] = { + 'runs-on': 'ubuntu-latest', + 'steps': [ + {'name': 'Checkout code', 'uses': 'actions/checkout@v4'}, + { + 'name': 'Set up Python', + 'uses': 'actions/setup-python@v5', + 'with': {'python-version': python_version} + }, + { + 'name': 'Install dependencies', + 'run': "pip install -e '.[dev]'" + }, + { + 'name': 'Run tests', + 'run': test_cmd + } + ] + } + + # Build and deploy jobs per environment + for env_name, env_config in environments.items(): + branch = env_config['branch'] + domain = env_config.get('domain') + needs_approval = env_config.get('approval', False) + + # Build job + image_prefix = f'ghcr.io/${{ github.repository_owner }}' if registry == 'ghcr' else registry + + build_job_name = f'build-{env_name}' + workflow['jobs'][build_job_name] = { + 'runs-on': 'ubuntu-latest', + 'needs': 'test', + 'if': f"github.ref == 'refs/heads/{branch}'", + 'steps': [ + {'name': 'Checkout code', 'uses': 'actions/checkout@v4'}, + { + 'name': 'Login to GitHub Container Registry', + 'uses': 'docker/login-action@v3', + 'with': { + 'registry': 'ghcr.io', + 'username': '${{ github.actor }}', + 'password': '${{ secrets.GITHUB_TOKEN }}' + } + }, + { + 'name': 'Build and push Docker image', + 'run': f''' +docker build -t {image_prefix}/{app_name}:{env_name}-${{{{ github.sha }}}} . +docker tag {image_prefix}/{app_name}:{env_name}-${{{{ github.sha }}}} {image_prefix}/{app_name}:{env_name} +docker push {image_prefix}/{app_name}:{env_name}-${{{{ github.sha }}}} +docker push {image_prefix}/{app_name}:{env_name} + '''.strip() + } + ] + } + + # Deploy job + deploy_job_name = f'deploy-{env_name}' + deploy_job = { + 'runs-on': 'ubuntu-latest', + 'needs': build_job_name, + 'if': f"github.ref == 'refs/heads/{branch}'", + 'steps': [ + {'name': 'Checkout code', 'uses': 'actions/checkout@v4'} + ] + } + + # Add environment with approval if needed + if needs_approval: + deploy_job['environment'] = { + 'name': env_name, + 'url': f'https://{domain}' if domain else None + } + + # Add deploy steps based on target + if deploy_target in ('docker', 'vps', 'hetzner'): + deploy_job['steps'].extend([ + { + 'name': 'Setup SSH', + 'run': f''' +mkdir -p ~/.ssh +echo "${{{{ secrets.SSH_PRIVATE_KEY_{env_name.upper()} }}}}" > ~/.ssh/id_rsa +chmod 600 ~/.ssh/id_rsa +ssh-keyscan -H ${{{{ secrets.DEPLOY_HOST_{env_name.upper()} }}}} >> ~/.ssh/known_hosts + '''.strip() + }, + { + 'name': f'Deploy to {env_name}', + 'run': f''' +ssh ${{{{ secrets.DEPLOY_USER_{env_name.upper()} }}}}@${{{{ secrets.DEPLOY_HOST_{env_name.upper()} }}}} "cd /srv/app && docker compose pull && docker compose up -d" + '''.strip() + } + ]) + + workflow['jobs'][deploy_job_name] = deploy_job + + # Save to file + if save: + output_path = '.github/workflows/deploy.yml' + _yaml_dump(workflow, output_path) + + return workflow