diff --git a/cmd/run.go b/cmd/run.go index 20c47b5..d57d1c9 100644 --- a/cmd/run.go +++ b/cmd/run.go @@ -1,11 +1,7 @@ package cmd import ( - "archive/tar" - "io" - "os" - "path/filepath" - + "github.com/Snider/Borg/pkg/matrix" "github.com/spf13/cobra" ) @@ -17,57 +13,7 @@ func NewRunCmd() *cobra.Command { Short: "Run a Terminal Isolation Matrix.", Args: cobra.ExactArgs(1), RunE: func(cmd *cobra.Command, args []string) error { - matrixFile := args[0] - - // Create a temporary directory to unpack the matrix file. - tempDir, err := os.MkdirTemp("", "borg-run-*") - if err != nil { - return err - } - defer os.RemoveAll(tempDir) - - // Unpack the matrix file. - file, err := os.Open(matrixFile) - if err != nil { - return err - } - defer file.Close() - - tr := tar.NewReader(file) - for { - header, err := tr.Next() - if err == io.EOF { - break - } - if err != nil { - return err - } - - path := filepath.Join(tempDir, header.Name) - if header.Typeflag == tar.TypeDir { - if err := os.MkdirAll(path, 0755); err != nil { - return err - } - continue - } - - outFile, err := os.Create(path) - if err != nil { - return err - } - defer outFile.Close() - if _, err := io.Copy(outFile, tr); err != nil { - return err - } - } - - // Run the matrix. - runc := execCommand("runc", "run", "borg-container") - runc.Dir = tempDir - runc.Stdout = os.Stdout - runc.Stderr = os.Stderr - runc.Stdin = os.Stdin - return runc.Run() + return matrix.Run(args[0]) }, } } diff --git a/cmd/run_test.go b/cmd/run_test.go index 4811ceb..051e561 100644 --- a/cmd/run_test.go +++ b/cmd/run_test.go @@ -10,30 +10,21 @@ import ( "github.com/Snider/Borg/pkg/matrix" ) -func helperProcess(command string, args ...string) *exec.Cmd { - cs := []string{"-test.run=TestHelperProcess", "--", command} - cs = append(cs, args...) - cmd := exec.Command(os.Args[0], cs...) - cmd.Env = []string{"GO_WANT_HELPER_PROCESS=1"} - return cmd -} - -func TestHelperProcess(t *testing.T) { - if os.Getenv("GO_WANT_HELPER_PROCESS") != "1" { - return - } - os.Exit(0) -} - func TestRunCmd_Good(t *testing.T) { // Create a dummy matrix file. matrixPath := createDummyMatrix(t) - // Mock the exec.Command function. - origExecCommand := execCommand - execCommand = helperProcess + // Mock the exec.Command function in the matrix package. + origExecCommand := matrix.ExecCommand + matrix.ExecCommand = func(command string, args ...string) *exec.Cmd { + cs := []string{"-test.run=TestHelperProcess", "--", command} + cs = append(cs, args...) + cmd := exec.Command(os.Args[0], cs...) + cmd.Env = []string{"GO_WANT_HELPER_PROCESS=1"} + return cmd + } t.Cleanup(func() { - execCommand = origExecCommand + matrix.ExecCommand = origExecCommand }) // Run the run command. @@ -90,8 +81,8 @@ func createDummyMatrix(t *testing.T) string { tw := tar.NewWriter(matrixFile) - // Add a dummy config.json. - configContent := []byte(matrix.DefaultConfigJSON) + // Add a dummy config.json. This is not a valid config, but it's enough to test the run command. + configContent := []byte(`{}`) hdr := &tar.Header{ Name: "config.json", Mode: 0600, diff --git a/docs/index.md b/docs/index.md index 4151a29..7dbe03d 100644 --- a/docs/index.md +++ b/docs/index.md @@ -47,6 +47,44 @@ borg collect website [url] [flags] ./borg collect website https://google.com --output website.dat --depth 1 ``` +#### `collect github repos` + +Collects all public repositories for a user or organization. + +**Usage:** +``` +borg collect github repos [user-or-org] [flags] +``` + +**Example:** +``` +./borg collect github repos Snider +``` + +#### `collect github release` + +Downloads the latest release of a file from GitHub releases. + +**Usage:** +``` +borg collect github release [repository-url] [flags] +``` + +**Flags:** +- `--output string`: Output directory for the downloaded file (default ".") +- `--pack`: Pack all assets into a DataNode +- `--file string`: The file to download from the release +- `--version string`: The version to check against + +**Example:** +``` +# Download the latest release of the 'borg' executable +./borg collect github release https://github.com/Snider/Borg --file borg + +# Pack all assets from the latest release into a DataNode +./borg collect github release https://github.com/Snider/Borg --pack --output borg-release.dat +``` + #### `collect pwa` Collects a single PWA and stores it in a DataNode. @@ -67,6 +105,24 @@ borg collect pwa [flags] ./borg collect pwa --uri https://squoosh.app --output squoosh.dat ``` +### `compile` + +Compiles a `Borgfile` into a Terminal Isolation Matrix. + +**Usage:** +``` +borg compile [flags] +``` + +**Flags:** +- `--file string`: Path to the Borgfile (default "Borgfile") +- `--output string`: Path to the output matrix file (default "a.matrix") + +**Example:** +``` +./borg compile -f my-borgfile -o my-app.matrix +``` + ### `serve` Serves the contents of a packaged DataNode or Terminal Isolation Matrix file using a static file server. @@ -116,33 +172,94 @@ To create a Matrix, use the `--format matrix` flag with any of the `collect` sub ./borg collect github repo https://github.com/Snider/Borg --output borg.matrix --format matrix ``` -You can then execute the Matrix with `runc`: +The `borg run` command is used to execute a Terminal Isolation Matrix. This command handles the unpacking and execution of the matrix in a secure, isolated environment using `runc`. This ensures that the payload can be safely analyzed without affecting the host system. + +**Example:** ``` -# Create a directory for the bundle -mkdir borg-bundle +./borg run borg.matrix +``` + +## Programmatic Usage + +The `examples` directory contains a number of Go programs that demonstrate how to use the `borg` package programmatically. -# Unpack the matrix into the bundle directory -tar -xf borg.matrix -C borg-bundle +### Inspecting a DataNode -# Run the bundle -cd borg-bundle -runc run borg +The `inspect_datanode` example demonstrates how to read, decompress, and walk a `.dat` file. + +**Usage:** +``` +go run examples/inspect_datanode/main.go ``` -## Inspecting a DataNode +### Creating a Matrix Programmatically -The `examples` directory contains a Go program that can be used to inspect the contents of a `.dat` file. +The `create_matrix_programmatically` example demonstrates how to create a Terminal Isolation Matrix from scratch. **Usage:** ``` -go run examples/inspect_datanode.go +go run examples/create_matrix_programmatically/main.go ``` -**Example:** +### Running a Matrix Programmatically + +The `run_matrix_programmatically` example demonstrates how to run a Terminal Isolation Matrix using the `borg` package. + +**Usage:** ``` -# First, create a .dat file -./borg collect github repo https://github.com/Snider/Borg --output borg.dat +go run examples/run_matrix_programmatically/main.go +``` + +### Collecting a Website + +The `collect_website` example demonstrates how to collect a website and package it into a `.dat` file. + +**Usage:** +``` +go run examples/collect_website/main.go +``` + +### Collecting a GitHub Release + +The `collect_github_release` example demonstrates how to collect the latest release of a GitHub repository. + +**Usage:** +``` +go run examples/collect_github_release/main.go +``` + +### Collecting All Repositories for a User + +The `all` example demonstrates how to collect all public repositories for a GitHub user. -# Then, inspect it -go run examples/inspect_datanode.go borg.dat +**Usage:** +``` +go run examples/all/main.go +``` + +### Collecting a PWA + +The `collect_pwa` example demonstrates how to collect a Progressive Web App and package it into a `.dat` file. + +**Usage:** +``` +go run examples/collect_pwa/main.go +``` + +### Collecting a GitHub Repository + +The `collect_github_repo` example demonstrates how to clone a GitHub repository and package it into a `.dat` file. + +**Usage:** +``` +go run examples/collect_github_repo/main.go +``` + +### Serving a DataNode + +The `serve` example demonstrates how to serve the contents of a `.dat` file over HTTP. + +**Usage:** +``` +go run examples/serve/main.go ``` diff --git a/examples/all/main.go b/examples/all/main.go index 25e4e58..6411baa 100644 --- a/examples/all/main.go +++ b/examples/all/main.go @@ -1,7 +1,44 @@ package main -import "fmt" +import ( + "context" + "fmt" + "log" + "os" + + "github.com/Snider/Borg/pkg/github" + "github.com/Snider/Borg/pkg/vcs" +) func main() { - fmt.Println("This is a placeholder for the 'all' example.") + log.Println("Collecting all repositories for a user...") + + repos, err := github.NewGithubClient().GetPublicRepos(context.Background(), "Snider") + if err != nil { + log.Fatalf("Failed to get public repos: %v", err) + } + + cloner := vcs.NewGitCloner() + + for _, repo := range repos { + log.Printf("Cloning %s...", repo) + dn, err := cloner.CloneGitRepository(fmt.Sprintf("https://github.com/%s", repo), nil) + if err != nil { + log.Printf("Failed to clone %s: %v", repo, err) + continue + } + + tarball, err := dn.ToTar() + if err != nil { + log.Printf("Failed to serialize %s to tar: %v", repo, err) + continue + } + + err = os.WriteFile(fmt.Sprintf("%s.dat", repo), tarball, 0644) + if err != nil { + log.Printf("Failed to write %s.dat: %v", repo, err) + continue + } + log.Printf("Successfully created %s.dat", repo) + } } diff --git a/examples/collect_github_release/main.go b/examples/collect_github_release/main.go index 332b208..63181ed 100644 --- a/examples/collect_github_release/main.go +++ b/examples/collect_github_release/main.go @@ -1,7 +1,42 @@ package main -import "fmt" +import ( + "log" + "os" + + "github.com/Snider/Borg/pkg/github" +) func main() { - fmt.Println("This is a placeholder for the 'collect github release' example.") + log.Println("Collecting GitHub release...") + + owner, repo, err := github.ParseRepoFromURL("https://github.com/Snider/Borg") + if err != nil { + log.Fatalf("Failed to parse repo from URL: %v", err) + } + + release, err := github.GetLatestRelease(owner, repo) + if err != nil { + log.Fatalf("Failed to get latest release: %v", err) + } + + if len(release.Assets) == 0 { + log.Println("No assets found in the latest release.") + return + } + + asset := release.Assets[0] + log.Printf("Downloading asset: %s", asset.GetName()) + + data, err := github.DownloadReleaseAsset(asset) + if err != nil { + log.Fatalf("Failed to download asset: %v", err) + } + + err = os.WriteFile(asset.GetName(), data, 0644) + if err != nil { + log.Fatalf("Failed to write asset to file: %v", err) + } + + log.Printf("Successfully downloaded asset to %s", asset.GetName()) } diff --git a/examples/collect_github_repo/main.go b/examples/collect_github_repo/main.go index df3182e..0fad8ef 100644 --- a/examples/collect_github_repo/main.go +++ b/examples/collect_github_repo/main.go @@ -1,7 +1,30 @@ package main -import "fmt" +import ( + "log" + "os" + + "github.com/Snider/Borg/pkg/vcs" +) func main() { - fmt.Println("This is a placeholder for the 'collect github repo' example.") + log.Println("Collecting GitHub repo...") + + cloner := vcs.NewGitCloner() + dn, err := cloner.CloneGitRepository("https://github.com/Snider/Borg", nil) + if err != nil { + log.Fatalf("Failed to clone repository: %v", err) + } + + tarball, err := dn.ToTar() + if err != nil { + log.Fatalf("Failed to serialize datanode to tar: %v", err) + } + + err = os.WriteFile("repo.dat", tarball, 0644) + if err != nil { + log.Fatalf("Failed to write datanode file: %v", err) + } + + log.Println("Successfully created repo.dat") } diff --git a/examples/collect_pwa/main.go b/examples/collect_pwa/main.go index 9ae5817..963ba62 100644 --- a/examples/collect_pwa/main.go +++ b/examples/collect_pwa/main.go @@ -1,7 +1,37 @@ package main -import "fmt" +import ( + "log" + "os" + + "github.com/Snider/Borg/pkg/pwa" +) func main() { - fmt.Println("This is a placeholder for the 'collect pwa' example.") + log.Println("Collecting PWA...") + + client := pwa.NewPWAClient() + pwaURL := "https://squoosh.app" + + manifestURL, err := client.FindManifest(pwaURL) + if err != nil { + log.Fatalf("Failed to find manifest: %v", err) + } + + dn, err := client.DownloadAndPackagePWA(pwaURL, manifestURL, nil) + if err != nil { + log.Fatalf("Failed to download and package PWA: %v", err) + } + + tarball, err := dn.ToTar() + if err != nil { + log.Fatalf("Failed to serialize datanode to tar: %v", err) + } + + err = os.WriteFile("pwa.dat", tarball, 0644) + if err != nil { + log.Fatalf("Failed to write datanode file: %v", err) + } + + log.Println("Successfully created pwa.dat") } diff --git a/examples/collect_website/main.go b/examples/collect_website/main.go index b363551..2e2f606 100644 --- a/examples/collect_website/main.go +++ b/examples/collect_website/main.go @@ -1,7 +1,32 @@ package main -import "fmt" +import ( + "log" + "os" + + "github.com/Snider/Borg/pkg/website" +) func main() { - fmt.Println("This is a placeholder for the 'collect website' example.") + log.Println("Collecting website...") + + // Download and package the website. + dn, err := website.DownloadAndPackageWebsite("https://example.com", 2, nil) + if err != nil { + log.Fatalf("Failed to collect website: %v", err) + } + + // Serialize the DataNode to a tarball. + tarball, err := dn.ToTar() + if err != nil { + log.Fatalf("Failed to serialize datanode to tar: %v", err) + } + + // Write the tarball to a file. + err = os.WriteFile("website.dat", tarball, 0644) + if err != nil { + log.Fatalf("Failed to write datanode file: %v", err) + } + + log.Println("Successfully created website.dat") } diff --git a/examples/run_matrix_programmatically/main.go b/examples/run_matrix_programmatically/main.go index 3807cdf..2204c6e 100644 --- a/examples/run_matrix_programmatically/main.go +++ b/examples/run_matrix_programmatically/main.go @@ -1,73 +1,16 @@ package main import ( - "archive/tar" - "io" "log" - "os" - "os/exec" - "path/filepath" + + "github.com/Snider/Borg/pkg/matrix" ) func main() { - // Open the matrix file. - matrixFile, err := os.Open("programmatic.matrix") - if err != nil { - log.Fatalf("Failed to open matrix file (run create_matrix_programmatically first): %v", err) - } - defer matrixFile.Close() - - // Create a temporary directory to unpack the matrix. - tempDir, err := os.MkdirTemp("", "borg-run-example-*") - if err != nil { - log.Fatalf("Failed to create temporary directory: %v", err) - } - defer os.RemoveAll(tempDir) - - log.Printf("Unpacking matrix to %s", tempDir) - - // Unpack the tar archive. - tr := tar.NewReader(matrixFile) - for { - header, err := tr.Next() - if err == io.EOF { - break // End of archive - } - if err != nil { - log.Fatalf("Failed to read tar header: %v", err) - } - - target := filepath.Join(tempDir, header.Name) - - switch header.Typeflag { - case tar.TypeDir: - if err := os.MkdirAll(target, 0755); err != nil { - log.Fatalf("Failed to create directory: %v", err) - } - case tar.TypeReg: - outFile, err := os.Create(target) - if err != nil { - log.Fatalf("Failed to create file: %v", err) - } - if _, err := io.Copy(outFile, tr); err != nil { - log.Fatalf("Failed to write file content: %v", err) - } - outFile.Close() - default: - log.Printf("Skipping unsupported type: %c in %s", header.Typeflag, header.Name) - } - } - - log.Println("Executing matrix with runc...") - - // Execute the matrix using runc. - cmd := exec.Command("runc", "run", "borg-container-example") - cmd.Dir = tempDir - cmd.Stdout = os.Stdout - cmd.Stderr = os.Stderr - cmd.Stdin = os.Stdin + log.Println("Executing matrix with Borg...") - if err := cmd.Run(); err != nil { + // Execute the matrix using the Borg package. + if err := matrix.Run("programmatic.matrix"); err != nil { log.Fatalf("Failed to run matrix: %v", err) } diff --git a/examples/serve/main.go b/examples/serve/main.go index cebb09d..10b5034 100644 --- a/examples/serve/main.go +++ b/examples/serve/main.go @@ -1,7 +1,42 @@ package main -import "fmt" +import ( + "log" + "net/http" + "os" + + "github.com/Snider/Borg/pkg/compress" + "github.com/Snider/Borg/pkg/tarfs" +) func main() { - fmt.Println("This is a placeholder for the 'serve' example.") + log.Println("Serving datanode...") + + // Create a dummy datanode for serving. + err := os.WriteFile("serve.dat", []byte{}, 0644) + if err != nil { + log.Fatalf("Failed to create dummy datanode: %v", err) + } + + data, err := os.ReadFile("serve.dat") + if err != nil { + log.Fatalf("Failed to read datanode: %v", err) + } + + decompressed, err := compress.Decompress(data) + if err != nil { + log.Fatalf("Failed to decompress datanode: %v", err) + } + + fs, err := tarfs.New(decompressed) + if err != nil { + log.Fatalf("Failed to create tarfs: %v", err) + } + + http.Handle("/", http.FileServer(fs)) + log.Println("Listening on :8080...") + err = http.ListenAndServe(":8080", nil) + if err != nil { + log.Fatalf("Failed to start server: %v", err) + } } diff --git a/pkg/matrix/run.go b/pkg/matrix/run.go new file mode 100644 index 0000000..1d1d1f8 --- /dev/null +++ b/pkg/matrix/run.go @@ -0,0 +1,65 @@ +package matrix + +import ( + "archive/tar" + "io" + "os" + "os/exec" + "path/filepath" +) + +// ExecCommand is a wrapper around exec.Command that can be overridden for testing. +var ExecCommand = exec.Command + +// Run executes a Terminal Isolation Matrix from a given path. +func Run(matrixPath string) error { + // Create a temporary directory to unpack the matrix file. + tempDir, err := os.MkdirTemp("", "borg-run-*") + if err != nil { + return err + } + defer os.RemoveAll(tempDir) + + // Unpack the matrix file. + file, err := os.Open(matrixPath) + if err != nil { + return err + } + defer file.Close() + + tr := tar.NewReader(file) + for { + header, err := tr.Next() + if err == io.EOF { + break + } + if err != nil { + return err + } + + path := filepath.Join(tempDir, header.Name) + if header.Typeflag == tar.TypeDir { + if err := os.MkdirAll(path, 0755); err != nil { + return err + } + continue + } + + outFile, err := os.Create(path) + if err != nil { + return err + } + defer outFile.Close() + if _, err := io.Copy(outFile, tr); err != nil { + return err + } + } + + // Run the matrix. + runc := ExecCommand("runc", "run", "borg-container") + runc.Dir = tempDir + runc.Stdout = os.Stdout + runc.Stderr = os.Stderr + runc.Stdin = os.Stdin + return runc.Run() +} diff --git a/pkg/matrix/run_test.go b/pkg/matrix/run_test.go new file mode 100644 index 0000000..c76a03c --- /dev/null +++ b/pkg/matrix/run_test.go @@ -0,0 +1,63 @@ +package matrix + +import ( + "fmt" + "os" + "os/exec" + "strings" + "testing" +) + +func fakeExecCommand(command string, args ...string) *exec.Cmd { + cs := []string{"-test.run=TestHelperProcess", "--", command} + cs = append(cs, args...) + cmd := exec.Command(os.Args[0], cs...) + cmd.Env = []string{"GO_WANT_HELPER_PROCESS=1"} + return cmd +} + +func TestRun_Good(t *testing.T) { + // Create a dummy matrix file. + file, err := os.CreateTemp("", "matrix-*.matrix") + if err != nil { + t.Fatal(err) + } + defer os.Remove(file.Name()) + + ExecCommand = fakeExecCommand + defer func() { ExecCommand = exec.Command }() + + err = Run(file.Name()) + if err != nil { + t.Errorf("Run() failed: %v", err) + } +} + +func TestHelperProcess(t *testing.T) { + if os.Getenv("GO_WANT_HELPER_PROCESS") != "1" { + return + } + defer os.Exit(0) + + args := os.Args + for len(args) > 0 { + if args[0] == "--" { + args = args[1:] + break + } + args = args[1:] + } + if len(args) == 0 { + fmt.Fprintf(os.Stderr, "No command\n") + os.Exit(2) + } + + cmd, args := args[0], args[1:] + if cmd == "runc" && args[0] == "run" { + fmt.Println("Success") + os.Exit(0) + } else { + fmt.Fprintf(os.Stderr, "Unknown command %s %s\n", cmd, strings.Join(args, " ")) + os.Exit(1) + } +}