diff --git a/.gitignore b/.gitignore index a00ea14..9813619 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,7 @@ docci dist/ .goreleaser/ build/ + +# used in the readme.md for testing +example.html +styles.css diff --git a/README.md b/README.md index 4ff1750..69ac18a 100644 --- a/README.md +++ b/README.md @@ -4,6 +4,14 @@ Your documentation is now your test suite! 🎯 *(pronounced "doc-ee", short for A CI tool that brings your markdown docs to life by executing code blocks in sequence. Run processes in the background, handle environment variables, add delays, verify outputs, and modify files - all through simple markdown tags. Perfect for ensuring your docs stay accurate and your examples actually work! 📚 +## 🌟 Projects Using Docci + +Several projects have adopted Docci to ensure their documentation stays accurate and executable: + +- **[Spawn](https://github.com/rollchains/spawn)** - Rapid Cosmos blockchain development framework +- **[WAVS](https://github.com/Lay3rLabs/wavs-middleware)** - Eigenlayer AVS WebAssembly development +- **[OndoChain](https://ondo.finance/ondo-chain/)** - Institutional RWA Cosmos-EVM blockchain + ## 🏃‍♂️ Quick Start Find sample workspaces in the [`examples/` directory](./examples/). @@ -28,7 +36,7 @@ task install # go install ./*.go # docci_Linux_x86_64, docci_Linux_arm64, docci_Darwin_x86_64, docci_Darwin_arm64 - name: Install Docci Readme Test Tool run: | - VERSION=v0.9.0 + VERSION=v0.9.2 BINARY=docci_Linux_x86_64.tar.gz curl -fsSL "https://github.com/Reecepbcups/docci/releases/download/${VERSION}/${BINARY}" | sudo tar -xzC /usr/local/bin sudo chmod +x /usr/local/bin/docci @@ -65,6 +73,14 @@ docci version * 🖥️ `docci-os=mac|linux`: Run the command only on it's the specified OS * 🔄 `docci-replace-text="old;new"`: Replace text in the code block before execution (including env variables!) +### 📄 File Tags + * 📝 `docci-file`: The file name to operate on + * 🔄 `docci-reset-file`: Reset the file to its original content + * 🚫 `docci-if-file-not-exists`: Only run if a file does not exist + * ➕ `docci-line-insert=N`: Insert content at line N + * ✏️ `docci-line-replace=N`: Replace content at line N + * 📋 `docci-line-replace=N-M`: Replace content from line N to M + ### 💡 Code Block Tag Examples (Operations) @@ -139,3 +155,47 @@ Cleanup demo server if running in the background: curl http://localhost:3000/kill ``` ```` + +### 💡 Files Code Block Tag Examples + +Create a new file from content: 📝 + + +````html +```html docci-file=example.html docci-reset-file + + + My Titlee + + +``` +```` + +Replace the typo'ed line: + +````html +```html docci-file=example.html docci-line-replace=3 + My Title +``` +```` + +Add new content + +````html +```html docci-file=example.html docci-line-insert=4 + +

My Header

+

1 paragraph

+

2 paragraph

+ +``` +```` + +Replace multiple lines + +````html +```html docci-file=example.html docci-line-replace=7-9 +

First paragraph

+

Second paragraph

+``` +```` diff --git a/examples/docci_config.json b/examples/docci_config.json new file mode 100644 index 0000000..9095f73 --- /dev/null +++ b/examples/docci_config.json @@ -0,0 +1,7 @@ +{ + "files": [ + "base.md", + "file-operations.md", + "if-not-installed-test.md" + ] +} \ No newline at end of file diff --git a/examples/file-operations.md b/examples/file-operations.md new file mode 100644 index 0000000..94e29b7 --- /dev/null +++ b/examples/file-operations.md @@ -0,0 +1,97 @@ +# File Operations Test + +This example demonstrates the new file operation tags in docci. + +## Create a new HTML file + +```html docci-file="example.html" docci-reset-file + + + + My Titlee + + +

Welcome

+ + +``` + +## Verify the file was created + +```bash docci-output-contains="" +cat example.html +``` + +## Fix the typo in the title (line 4) + +```html docci-file="example.html" docci-line-replace="4" + My Title +``` + +## Verify the typo was fixed + +```bash docci-output-contains="My Title" +grep "title" example.html +``` + +## Insert content after the h1 tag (line 7) + +```html docci-file="example.html" docci-line-insert="7" +

This is a paragraph

+

This is another paragraph

+``` + +## Verify the paragraphs were added + +```bash docci-output-contains="This is a paragraph" +cat example.html +``` + +## Create a CSS file + +```css docci-file="styles.css" +body { + font-family: Arial, sans-serif; + margin: 0; + padding: 20px; +} + +h1 { + color: #333; +} +``` + +## Add more styles at the end + +```css docci-file="styles.css" docci-line-insert="10" + +p { + line-height: 1.6; + color: #666; +} +``` + +## Replace the h1 color (line 8) + +```css docci-file="styles.css" docci-line-replace="8" + color: #0066cc; +``` + +## Verify the final CSS + +```bash docci-output-contains="color: #0066cc" +cat styles.css +``` + +## Test conditional file creation + +```bash docci-if-file-not-exists="example.html" +echo "This should not run because example.html exists" +``` + +## Clean up + +```bash +rm -f example.html styles.css +echo "Test files cleaned up" +``` diff --git a/main.go b/main.go index a2e1613..3028253 100644 --- a/main.go +++ b/main.go @@ -1,6 +1,7 @@ package main import ( + "encoding/json" "fmt" "os" "os/exec" @@ -23,6 +24,11 @@ var ( keepRunning bool ) +// DocciConfig represents the JSON configuration file format +type DocciConfig struct { + Files []string `json:"files"` +} + var rootCmd = &cobra.Command{ Use: "docci", Short: "Execute and validate code blocks in markdown files", @@ -33,11 +39,26 @@ It helps ensure your documentation examples are always accurate and working.`, } var runCmd = &cobra.Command{ - Use: "run ", + Use: "run ", Short: "Execute code blocks in markdown file(s)", Long: `Execute all code blocks marked with 'exec' in markdown file(s). The command will run the blocks in sequence and validate any expected outputs. -Multiple files can be specified separated by commas.`, + +You can specify files in three ways: +1. Single file: docci run file.md +2. Multiple files (comma-separated): docci run file1.md,file2.md,file3.md +3. JSON config file: docci run config.json + +When using a JSON config file, create a file with this format: +{ + "files": [ + "file1.md", + "subdir/file2.md", + "file3.md" + ] +} + +File paths in the JSON config are resolved relative to the config file's location.`, Args: cobra.ExactArgs(1), RunE: func(cmd *cobra.Command, args []string) error { // Initialize logging based on flags @@ -340,9 +361,60 @@ func runCleanupCommands(commands []string) { log.Info("Cleanup complete") } -// parseFileList parses comma separated file paths +// parseFileList parses comma separated file paths or JSON config file func parseFileList(input string) []string { - // Split by comma + // Check if input is a JSON file + if strings.HasSuffix(strings.ToLower(input), ".json") { + // Try to read and parse as JSON config + configData, err := os.ReadFile(input) + if err == nil { + var config DocciConfig + if jsonErr := json.Unmarshal(configData, &config); jsonErr == nil && len(config.Files) > 0 { + // Successfully parsed JSON config + // Get the directory containing the JSON file + configDir := filepath.Dir(input) + + // Get git root for fallback + var gitRoot string + cmd := exec.Command("git", "rev-parse", "--show-toplevel") + if output, err := cmd.Output(); err == nil { + gitRoot = strings.TrimSpace(string(output)) + } + + // Resolve relative file paths + var resolvedFiles []string + for _, file := range config.Files { + // If the file path is absolute, use it as-is + if filepath.IsAbs(file) { + resolvedFiles = append(resolvedFiles, file) + } else { + // First try relative to the JSON config directory + resolvedPath := filepath.Join(configDir, file) + if _, err := os.Stat(resolvedPath); err == nil { + resolvedFiles = append(resolvedFiles, resolvedPath) + } else if gitRoot != "" { + // If not found, try relative to git root + rootPath := filepath.Join(gitRoot, file) + if _, err := os.Stat(rootPath); err == nil { + resolvedFiles = append(resolvedFiles, rootPath) + } else { + // If still not found, use the original resolved path + // (will fail later with proper error message) + resolvedFiles = append(resolvedFiles, resolvedPath) + } + } else { + // No git root available, use original resolved path + resolvedFiles = append(resolvedFiles, resolvedPath) + } + } + } + return resolvedFiles + } + } + // If we can't parse as JSON config, treat it as a regular file + } + + // Original comma-separated logic if !strings.Contains(input, ",") { // Single file return []string{strings.TrimSpace(input)} diff --git a/parser/codeblocks.go b/parser/codeblocks.go index c825267..337939b 100644 --- a/parser/codeblocks.go +++ b/parser/codeblocks.go @@ -35,7 +35,14 @@ type CodeBlock struct { LineNumber int FileName string // Added for debugging multiple files ReplaceText string - content strings.Builder // Used during parsing to build content + + // File operation fields + File string // docci-file: The file name to operate on + ResetFile bool // docci-reset-file: Reset the file to its original content + LineInsert int // docci-line-insert: Insert content at line N (1-based) + LineReplace string // docci-line-replace: Replace content at line N or N-M + + content strings.Builder // Used during parsing to build content } // given a markdown file, parse out all the code blocks within it. @@ -67,6 +74,10 @@ func (c *CodeBlock) applyTags(tags MetaTag, lineNumber int, fileName string) { c.IfFileNotExists = tags.IfFileNotExists c.IfNotInstalled = tags.IfNotInstalled c.ReplaceText = tags.ReplaceText + c.File = tags.File + c.ResetFile = tags.ResetFile + c.LineInsert = tags.LineInsert + c.LineReplace = tags.LineReplace c.LineNumber = lineNumber c.FileName = fileName c.content.Reset() @@ -150,22 +161,11 @@ func ParseCodeBlocksWithFileName(markdown string, fileName string) ([]CodeBlock, lang = langParts[0] } - if contains(ValidLangs, lang) { - // Validate tag combinations - if tags.OutputContains != "" && tags.Background { - return nil, fmt.Errorf("line %d: Cannot use both docci-output-contains and docci-background on the same code block", lineNumber) - } - if tags.AssertFailure && tags.Background { - return nil, fmt.Errorf("line %d: Cannot use both docci-assert-failure and docci-background on the same code block", lineNumber) - } - if tags.AssertFailure && tags.OutputContains != "" { - return nil, fmt.Errorf("line %d: Cannot use both docci-assert-failure and docci-output-contains on the same code block", lineNumber) - } - if tags.WaitForEndpoint != "" && tags.Background { - return nil, fmt.Errorf("line %d: Cannot use both docci-wait-for-endpoint and docci-background on the same code block", lineNumber) - } - if tags.RetryCount > 0 && tags.Background { - return nil, fmt.Errorf("line %d: Cannot use both docci-retry and docci-background on the same code block", lineNumber) + // Allow block if it's a valid language OR if it has file operation tags + if contains(ValidLangs, lang) || tags.File != "" { + // Validate tag combinations using the centralized validation + if err := tags.Validate(lineNumber); err != nil { + return nil, err } startParsing = true @@ -340,28 +340,77 @@ func BuildExecutableScriptWithOptions(blocks []CodeBlock, opts types.DocciOpts) } } - // Prepare the code content with per-command delay and command display - delaySeconds := block.DelayPerCmdSecs - codeContent := replaceTemplateVars(codeExecutionTemplate, map[string]string{ - "DELAY": strconv.FormatFloat(delaySeconds, 'g', -1, 64), - "BASH_FLAGS": formatBashFlags(block.AssertFailure), - "CONTENT": blockContent, - }) - - // Add the actual code with retry logic if needed - if block.RetryCount > 0 { - retryDelay := GetRetryDelay() - script.WriteString(replaceTemplateVars(retryWrapperStartTemplate, map[string]string{ - "INDEX": strconv.Itoa(block.Index), - "MAX_RETRIES": strconv.Itoa(block.RetryCount), - "RETRY_DELAY": strconv.Itoa(retryDelay), - })) - script.WriteString(codeContent) - script.WriteString(replaceTemplateVars(retryWrapperEndTemplate, map[string]string{ - "INDEX": strconv.Itoa(block.Index), - })) + // Check if this is a file operation block + if block.File != "" { + // Handle file operations + if block.ResetFile || block.LineInsert == 0 && block.LineReplace == "" { + // Create or reset file + operation := "create" + if block.ResetFile { + operation = "reset" + } + script.WriteString(replaceTemplateVars(fileCreateOrResetTemplate, map[string]string{ + "OPERATION": operation, + "FILE": block.File, + "FILE_INFO": formatFileInfo(block.FileName), + "CONTENT": blockContent, + })) + } else if block.LineInsert > 0 { + // Insert at line + script.WriteString(replaceTemplateVars(fileLineInsertTemplate, map[string]string{ + "FILE": block.File, + "LINE": strconv.Itoa(block.LineInsert), + "FILE_INFO": formatFileInfo(block.FileName), + "CONTENT": blockContent, + })) + } else if block.LineReplace != "" { + // Replace line(s) + startLine := "" + endLine := "" + + if strings.Contains(block.LineReplace, "-") { + parts := strings.Split(block.LineReplace, "-") + startLine = strings.TrimSpace(parts[0]) + endLine = strings.TrimSpace(parts[1]) + } else { + startLine = block.LineReplace + endLine = block.LineReplace + } + + script.WriteString(replaceTemplateVars(fileLineReplaceTemplate, map[string]string{ + "FILE": block.File, + "LINES": block.LineReplace, + "START_LINE": startLine, + "END_LINE": endLine, + "FILE_INFO": formatFileInfo(block.FileName), + "CONTENT": blockContent, + })) + } } else { - script.WriteString(codeContent) + // Regular code execution (not a file operation) + // Prepare the code content with per-command delay and command display + delaySeconds := block.DelayPerCmdSecs + codeContent := replaceTemplateVars(codeExecutionTemplate, map[string]string{ + "DELAY": strconv.FormatFloat(delaySeconds, 'g', -1, 64), + "BASH_FLAGS": formatBashFlags(block.AssertFailure), + "CONTENT": blockContent, + }) + + // Add the actual code with retry logic if needed + if block.RetryCount > 0 { + retryDelay := GetRetryDelay() + script.WriteString(replaceTemplateVars(retryWrapperStartTemplate, map[string]string{ + "INDEX": strconv.Itoa(block.Index), + "MAX_RETRIES": strconv.Itoa(block.RetryCount), + "RETRY_DELAY": strconv.Itoa(retryDelay), + })) + script.WriteString(codeContent) + script.WriteString(replaceTemplateVars(retryWrapperEndTemplate, map[string]string{ + "INDEX": strconv.Itoa(block.Index), + })) + } else { + script.WriteString(codeContent) + } } // Close the guard clause if needed diff --git a/parser/script_templates.go b/parser/script_templates.go index d9dbe5a..cf108fa 100644 --- a/parser/script_templates.go +++ b/parser/script_templates.go @@ -167,5 +167,78 @@ cleanup_on_interrupt() { trap cleanup_on_interrupt INT TERM sleep infinity +` + + // File operation templates + fileCreateOrResetTemplate = `# File operation: {{OPERATION}} {{FILE}}{{FILE_INFO}} +cat > "{{FILE}}" << 'DOCCI_EOF' +{{CONTENT}}DOCCI_EOF +` + + fileLineInsertTemplate = `# File operation: insert at line {{LINE}} in {{FILE}}{{FILE_INFO}} +if [ -f "{{FILE}}" ]; then + # Create a temporary file + temp_file=$(mktemp) + + # Read existing content and insert at specified line + line_count=0 + inserted=false + while IFS= read -r line || [ -n "$line" ]; do + line_count=$((line_count + 1)) + if [ $line_count -eq {{LINE}} ] && [ "$inserted" = "false" ]; then + cat << 'DOCCI_EOF' >> "$temp_file" +{{CONTENT}}DOCCI_EOF + inserted=true + fi + printf '%s\n' "$line" >> "$temp_file" + done < "{{FILE}}" + + # If insert line is beyond EOF, append at the end + total_lines=$line_count + if [ {{LINE}} -gt $total_lines ] && [ "$inserted" = "false" ]; then + cat << 'DOCCI_EOF' >> "$temp_file" +{{CONTENT}}DOCCI_EOF + fi + + # Replace original file + mv "$temp_file" "{{FILE}}" +else + echo "Error: File {{FILE}} does not exist for line insert operation" + exit 1 +fi +` + + fileLineReplaceTemplate = `# File operation: replace line(s) {{LINES}} in {{FILE}}{{FILE_INFO}} +if [ -f "{{FILE}}" ]; then + # Create a temporary file + temp_file=$(mktemp) + + # Parse line range + start_line={{START_LINE}} + end_line={{END_LINE}} + + # Read and replace lines + line_count=0 + replaced=false + while IFS= read -r line || [ -n "$line" ]; do + line_count=$((line_count + 1)) + if [ $line_count -ge $start_line ] && [ $line_count -le $end_line ]; then + if [ "$replaced" = "false" ]; then + cat << 'DOCCI_EOF' >> "$temp_file" +{{CONTENT}}DOCCI_EOF + replaced=true + fi + # Skip the lines being replaced + else + printf '%s\n' "$line" >> "$temp_file" + fi + done < "{{FILE}}" + + # Replace original file + mv "$temp_file" "{{FILE}}" +else + echo "Error: File {{FILE}} does not exist for line replace operation" + exit 1 +fi ` ) diff --git a/parser/tags.go b/parser/tags.go index cdaf5b7..d6154d9 100644 --- a/parser/tags.go +++ b/parser/tags.go @@ -29,6 +29,12 @@ type MetaTag struct { IfFileNotExists string IfNotInstalled string ReplaceText string + + // File operation tags + File string // docci-file: The file name to operate on + ResetFile bool // docci-reset-file: Reset the file to its original content + LineInsert int // docci-line-insert: Insert content at line N (1-based) + LineReplace string // docci-line-replace: Replace content at line N or N-M (e.g., "3" or "7-9") } var tags string @@ -48,6 +54,10 @@ const ( TagIfFileNotExists = "docci-if-file-not-exists" TagIfNotInstalled = "docci-if-not-installed" TagReplaceText = "docci-replace-text" + TagFile = "docci-file" + TagResetFile = "docci-reset-file" + TagLineInsert = "docci-line-insert" + TagLineReplace = "docci-line-replace" ) // TagInfo holds information about a tag and its aliases @@ -144,6 +154,30 @@ var tagDefinitions = []TagInfo{ Description: "Replace text in the code block before execution (format: 'old;new')", Example: "```bash docci-replace-text=\"bbbbbb;$SOME_ENV_VAR\"", }, + { + Name: TagFile, + Aliases: []string{}, + Description: "Specify the file name to operate on", + Example: "```html docci-file=\"example.html\"", + }, + { + Name: TagResetFile, + Aliases: []string{}, + Description: "Reset the file to its original content (creates or overwrites)", + Example: "```html docci-file=\"example.html\" docci-reset-file", + }, + { + Name: TagLineInsert, + Aliases: []string{}, + Description: "Insert content at line N (1-based)", + Example: "```html docci-file=\"example.html\" docci-line-insert=\"4\"", + }, + { + Name: TagLineReplace, + Aliases: []string{}, + Description: "Replace content at line N or lines N-M (1-based)", + Example: "```html docci-file=\"example.html\" docci-line-replace=\"3\" or docci-line-replace=\"7-9\"", + }, } // tagAliasMap is built from tagDefinitions for fast lookup @@ -171,24 +205,6 @@ func TagAlias(tag string) (string, error) { return "", fmt.Errorf("unknown tag / alias: %s", tag) } -// example input: -// ```bash docci-output-contains="Persist: test" -// ```bash docci-ignore -// func ParseTags(line string) MetaTags { -// // given some codeblock, parse out any tags that are present to be used -// // ```bash docci-output-contains="Persist: test" -// // ```bash docci-ignore -// var mt MetaTags - -// lang := strings.TrimPrefix(line, "```") -// lang = strings.TrimSpace(lang) - -// mt.Language = lang -// mt.Ignore = strings.Contains(line, "docci-ignore") - -// return mt -// } - // given a line, find any docci- tags that are present and parse them out func ParseTags(line string) (MetaTag, error) { // Use regex to find all docci-* tags with optional quoted or unquoted values @@ -370,6 +386,61 @@ func parseTagsFromPotential(potential []string) (MetaTag, error) { } mt.ReplaceText = content logger.GetLogger().Debugf("Replace text tag found: %s", content) + case TagFile: + if content == "" { + return MetaTag{}, fmt.Errorf("docci-file requires a file name") + } + mt.File = content + logger.GetLogger().Debugf("File tag found with name: %s", content) + case TagResetFile: + mt.ResetFile = true + logger.GetLogger().Debugf("Reset file tag found") + case TagLineInsert: + if content == "" { + return MetaTag{}, fmt.Errorf("docci-line-insert requires a line number") + } + lineNum, err := strconv.Atoi(content) + if err != nil { + return MetaTag{}, fmt.Errorf("invalid line number in docci-line-insert: %s", content) + } + if lineNum <= 0 { + return MetaTag{}, fmt.Errorf("line number must be positive (1-based) in docci-line-insert, got: %d", lineNum) + } + mt.LineInsert = lineNum + logger.GetLogger().Debugf("Line insert tag found at line: %d", lineNum) + case TagLineReplace: + if content == "" { + return MetaTag{}, fmt.Errorf("docci-line-replace requires a line number or range (e.g., '3' or '7-9')") + } + // Validate format: either a single number or N-M + if strings.Contains(content, "-") { + parts := strings.Split(content, "-") + if len(parts) != 2 { + return MetaTag{}, fmt.Errorf("invalid line range format in docci-line-replace: %s (expected 'N-M')", content) + } + startLine, err1 := strconv.Atoi(strings.TrimSpace(parts[0])) + endLine, err2 := strconv.Atoi(strings.TrimSpace(parts[1])) + if err1 != nil || err2 != nil { + return MetaTag{}, fmt.Errorf("invalid line numbers in docci-line-replace: %s", content) + } + if startLine <= 0 || endLine <= 0 { + return MetaTag{}, fmt.Errorf("line numbers must be positive (1-based) in docci-line-replace") + } + if startLine > endLine { + return MetaTag{}, fmt.Errorf("start line must be <= end line in docci-line-replace: %s", content) + } + } else { + // Single line number + lineNum, err := strconv.Atoi(content) + if err != nil { + return MetaTag{}, fmt.Errorf("invalid line number in docci-line-replace: %s", content) + } + if lineNum <= 0 { + return MetaTag{}, fmt.Errorf("line number must be positive (1-based) in docci-line-replace, got: %d", lineNum) + } + } + mt.LineReplace = content + logger.GetLogger().Debugf("Line replace tag found: %s", content) default: return MetaTag{}, fmt.Errorf("unknown tag: %s", normalizedTag) } @@ -440,3 +511,38 @@ func ShouldRunBasedOnCommandInstallation(ifNotInstalledCommand string) bool { func GetAllTagsInfo() []TagInfo { return tagDefinitions } + +// Validate checks if the tag combinations are valid +func (mt *MetaTag) Validate(lineNumber int) error { + // Validate tag combinations + if mt.OutputContains != "" && mt.Background { + return fmt.Errorf("line %d: Cannot use both docci-output-contains and docci-background on the same code block", lineNumber) + } + if mt.AssertFailure && mt.Background { + return fmt.Errorf("line %d: Cannot use both docci-assert-failure and docci-background on the same code block", lineNumber) + } + // TODO: it is possible we can allow this in the future, but need to think more about it & test (do we output contains stderr or stdout or both or?) + if mt.AssertFailure && mt.OutputContains != "" { + return fmt.Errorf("line %d: Cannot use both docci-assert-failure and docci-output-contains on the same code block", lineNumber) + } + if mt.WaitForEndpoint != "" && mt.Background { + return fmt.Errorf("line %d: Cannot use both docci-wait-for-endpoint and docci-background on the same code block", lineNumber) + } + if mt.RetryCount > 0 && mt.Background { + return fmt.Errorf("line %d: Cannot use both docci-retry and docci-background on the same code block", lineNumber) + } + + // Validate file operations + if mt.File != "" { + // Can't use file operations with background blocks + if mt.Background { + return fmt.Errorf("line %d: Cannot use file operations with docci-background", lineNumber) + } + // Can't have both line-insert and line-replace + if mt.LineInsert > 0 && mt.LineReplace != "" { + return fmt.Errorf("line %d: Cannot use both docci-line-insert and docci-line-replace on the same code block", lineNumber) + } + } + + return nil +}