From f5d7e06919e5e1adeb02804b786bdab802c9a76b Mon Sep 17 00:00:00 2001 From: John Starich Date: Sat, 4 Sep 2021 18:14:21 -0500 Subject: [PATCH 01/37] Build static sub-tasks in parallel --- server/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/package.json b/server/package.json index 764d179..7f400fb 100644 --- a/server/package.json +++ b/server/package.json @@ -20,7 +20,7 @@ }, "scripts": { "start": "react-scripts start", - "start-go": "cd .. && nodemon --signal SIGINT -e go -d 2 -x 'make static || exit 1'", + "start-go": "cd .. && nodemon --signal SIGINT -e go -d 2 -x 'make -j8 static || exit 1'", "build": "react-scripts build", "test": "react-scripts test", "eject": "react-scripts eject" From 45c7f1e5a1d29a2d2f8c3be0dcece5e49b5d30d0 Mon Sep 17 00:00:00 2001 From: John Starich Date: Sun, 6 Feb 2022 14:54:49 -0600 Subject: [PATCH 02/37] Bump to latest Go 1.16 release --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 77ce405..fc38f8d 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,5 @@ SHELL := /usr/bin/env bash -GO_VERSION = 1.16.6 +GO_VERSION = 1.16 GOROOT = PATH := ${PWD}/cache/go/bin:${PWD}/cache/go/misc/wasm:${PATH} GOOS = js @@ -71,7 +71,7 @@ cache/go${GO_VERSION}: cache git clone \ --depth 1 \ --single-branch \ - --branch hackpad-go${GO_VERSION} \ + --branch hackpad/release-branch.go${GO_VERSION} \ https://github.com/hack-pad/go.git \ "$$TMP"; \ pushd "$$TMP/src"; \ From a3fab9540d0816af8badd8010d5dad469e5fe7de Mon Sep 17 00:00:00 2001 From: John Starich Date: Sun, 6 Feb 2022 11:56:13 -0600 Subject: [PATCH 03/37] Add caller info to log output --- Makefile | 6 ++++-- internal/log/caller.go | 22 ++++++++++++++++++++++ internal/log/js_log.go | 12 +++++++----- internal/log/log.go | 30 +++++++++++++++++++----------- 4 files changed, 52 insertions(+), 18 deletions(-) create mode 100644 internal/log/caller.go diff --git a/Makefile b/Makefile index fc38f8d..cdebda1 100644 --- a/Makefile +++ b/Makefile @@ -7,6 +7,8 @@ GOARCH = wasm export LINT_VERSION=1.27.0 +BUILD_FLAGS = -trimpath + .PHONY: serve serve: go run ./server @@ -91,10 +93,10 @@ cache/go${GO_VERSION}: cache touch cache/go.mod # Makes it so linters will ignore this dir server/public/wasm/%.wasm: server/public/wasm go - go build -o $@ ./cmd/$* + go build ${BUILD_FLAGS} -o $@ ./cmd/$* server/public/wasm/main.wasm: server/public/wasm go - go build -o server/public/wasm/main.wasm . + go build ${BUILD_FLAGS} -o server/public/wasm/main.wasm . server/public/wasm/wasm_exec.js: go cp cache/go/misc/wasm/wasm_exec.js server/public/wasm/wasm_exec.js diff --git a/internal/log/caller.go b/internal/log/caller.go new file mode 100644 index 0000000..684582d --- /dev/null +++ b/internal/log/caller.go @@ -0,0 +1,22 @@ +package log + +import ( + "fmt" + "runtime" + "strings" +) + +const ( + hackpadCommonPrefix = "github.com/hack-pad/" +) + +func getCaller(skip int) string { + pc, file, line, ok := runtime.Caller(skip + 1) + if !ok { + return "" + } + file = strings.TrimPrefix(file, hackpadCommonPrefix) + fn := runtime.FuncForPC(pc).Name() + fn = fn[strings.LastIndexAny(fn, "./")+1:] + return fmt.Sprintf("%s:%d:%s()", file, line, fn) +} diff --git a/internal/log/js_log.go b/internal/log/js_log.go index d59c394..2b49fc2 100644 --- a/internal/log/js_log.go +++ b/internal/log/js_log.go @@ -34,25 +34,27 @@ func SetLevel(level consoleType) { } func DebugJSValues(args ...interface{}) int { - return logJSValues(LevelDebug, args...) + return logJSValues(LevelDebug, 1, args...) } func PrintJSValues(args ...interface{}) int { - return logJSValues(LevelLog, args...) + return logJSValues(LevelLog, 1, args...) } func WarnJSValues(args ...interface{}) int { - return logJSValues(LevelWarn, args...) + return logJSValues(LevelWarn, 1, args...) } func ErrorJSValues(args ...interface{}) int { - return logJSValues(LevelError, args...) + return logJSValues(LevelError, 1, args...) } -func logJSValues(kind consoleType, args ...interface{}) int { +func logJSValues(kind consoleType, skip int, args ...interface{}) int { if kind < logLevel { return 0 } + caller := getCaller(skip + 1) + args = append([]interface{}{caller}, args...) console.Call(kind.String(), args...) return 0 } diff --git a/internal/log/log.go b/internal/log/log.go index f3eebf4..8af8026 100644 --- a/internal/log/log.go +++ b/internal/log/log.go @@ -1,6 +1,8 @@ package log -import "fmt" +import ( + "fmt" +) type consoleType int @@ -51,51 +53,57 @@ func parseLevel(level string) consoleType { } func Debugf(format string, args ...interface{}) int { - return logf(LevelDebug, format, args...) + return logf(LevelDebug, 1, format, args...) } func Printf(format string, args ...interface{}) int { - return logf(LevelLog, format, args...) + return logf(LevelLog, 1, format, args...) } func Warnf(format string, args ...interface{}) int { - return logf(LevelWarn, format, args...) + return logf(LevelWarn, 1, format, args...) } func Errorf(format string, args ...interface{}) int { - return logf(LevelError, format, args...) + return logf(LevelError, 1, format, args...) } -func logf(kind consoleType, format string, args ...interface{}) int { +func logf(kind consoleType, skip int, format string, args ...interface{}) int { if kind < logLevel { return 0 } s := fmt.Sprintf(format, args...) + if caller := getCaller(skip + 1); caller != "" { + s = caller + " - " + s + } writeLog(kind, s) return len(s) } func Debug(args ...interface{}) int { - return log(LevelDebug, args...) + return log(LevelDebug, 1, args...) } func Print(args ...interface{}) int { - return log(LevelLog, args...) + return log(LevelLog, 1, args...) } func Warn(args ...interface{}) int { - return log(LevelWarn, args...) + return log(LevelWarn, 1, args...) } func Error(args ...interface{}) int { - return log(LevelError, args...) + return log(LevelError, 1, args...) } -func log(kind consoleType, args ...interface{}) int { +func log(kind consoleType, skip int, args ...interface{}) int { if kind < logLevel { return 0 } s := fmt.Sprint(args...) + if caller := getCaller(skip + 1); caller != "" { + s = caller + " - " + s + } writeLog(kind, s) return len(s) } From 408cf79c32913b061c4464160712c66f1fac94e5 Mon Sep 17 00:00:00 2001 From: John Starich Date: Sat, 2 Apr 2022 16:21:40 -0500 Subject: [PATCH 04/37] Replace process interface with only struct --- internal/js/process/spawn.go | 2 +- internal/process/context.go | 6 ++--- internal/process/process.go | 45 ++++++++++++------------------- internal/process/process_js.go | 4 +-- internal/process/process_other.go | 2 +- internal/process/wasm.go | 6 ++--- 6 files changed, 27 insertions(+), 38 deletions(-) diff --git a/internal/js/process/spawn.go b/internal/js/process/spawn.go index 81c487b..9b7c1d3 100644 --- a/internal/js/process/spawn.go +++ b/internal/js/process/spawn.go @@ -35,7 +35,7 @@ func spawn(args []js.Value) (interface{}, error) { return Spawn(command, argv, procAttr) } -func Spawn(command string, args []string, attr *process.ProcAttr) (process.Process, error) { +func Spawn(command string, args []string, attr *process.ProcAttr) (*process.Process, error) { p, err := process.New(command, args, attr) if err != nil { return p, err diff --git a/internal/process/context.go b/internal/process/context.go index f21ec3f..9d1760c 100644 --- a/internal/process/context.go +++ b/internal/process/context.go @@ -23,7 +23,7 @@ func Init(switchedContext func(PID, PID)) { panic(err) } p, err := newWithCurrent( - &process{fileDescriptors: fileDescriptors}, + &Process{fileDescriptors: fileDescriptors}, minPID, "", nil, @@ -51,12 +51,12 @@ func switchContext(pid PID) (prev PID) { return } -func Current() Process { +func Current() *Process { process, _ := Get(currentPID) return process } -func Get(pid PID) (process Process, ok bool) { +func Get(pid PID) (process *Process, ok bool) { p, ok := pids[pid] return p, ok } diff --git a/internal/process/process.go b/internal/process/process.go index 3c0404c..46f7c67 100644 --- a/internal/process/process.go +++ b/internal/process/process.go @@ -32,22 +32,11 @@ const ( ) var ( - pids = make(map[PID]*process) + pids = make(map[PID]*Process) lastPID = atomic.NewUint64(minPID) ) -type Process interface { - PID() PID - ParentPID() PID - - Start() error - Wait() (exitCode int, err error) - Files() *fs.FileDescriptors - WorkingDirectory() string - SetWorkingDirectory(wd string) error -} - -type process struct { +type Process struct { pid, parentPID PID command string args []string @@ -61,18 +50,18 @@ type process struct { setFilesWD func(wd string) error } -func New(command string, args []string, attr *ProcAttr) (Process, error) { +func New(command string, args []string, attr *ProcAttr) (*Process, error) { return newWithCurrent(Current(), PID(lastPID.Inc()), command, args, attr) } -func newWithCurrent(current Process, newPID PID, command string, args []string, attr *ProcAttr) (*process, error) { +func newWithCurrent(current *Process, newPID PID, command string, args []string, attr *ProcAttr) (*Process, error) { wd := current.WorkingDirectory() if attr.Dir != "" { wd = attr.Dir } files, setFilesWD, err := fs.NewFileDescriptors(newPID, wd, current.Files(), attr.Files) ctx, cancel := context.WithCancel(context.Background()) - return &process{ + return &Process{ pid: newPID, command: command, args: args, @@ -86,19 +75,19 @@ func newWithCurrent(current Process, newPID PID, command string, args []string, }, err } -func (p *process) PID() PID { +func (p *Process) PID() PID { return p.pid } -func (p *process) ParentPID() PID { +func (p *Process) ParentPID() PID { return p.parentPID } -func (p *process) Files() *fs.FileDescriptors { +func (p *Process) Files() *fs.FileDescriptors { return p.fileDescriptors } -func (p *process) Start() error { +func (p *Process) Start() error { err := p.start() if p.err == nil { p.err = err @@ -106,7 +95,7 @@ func (p *process) Start() error { return p.err } -func (p *process) start() error { +func (p *Process) start() error { pids[p.pid] = p log.Debugf("Spawning process: %v", p) go func() { @@ -120,7 +109,7 @@ func (p *process) start() error { return nil } -func (p *process) prepExecutable() (command string, err error) { +func (p *Process) prepExecutable() (command string, err error) { fs := p.Files() command, err = lookPath(fs.Stat, os.Getenv("PATH"), p.command) if err != nil { @@ -143,13 +132,13 @@ func (p *process) prepExecutable() (command string, err error) { return command, nil } -func (p *process) Done() { +func (p *Process) Done() { log.Debug("PID ", p.pid, " is done.\n", p.fileDescriptors) p.fileDescriptors.CloseAll() p.ctxDone() } -func (p *process) handleErr(err error) { +func (p *Process) handleErr(err error) { p.state = stateDone if err != nil { log.Errorf("Failed to start process: %s", err.Error()) @@ -159,20 +148,20 @@ func (p *process) handleErr(err error) { p.Done() } -func (p *process) Wait() (exitCode int, err error) { +func (p *Process) Wait() (exitCode int, err error) { <-p.ctx.Done() return p.exitCode, p.err } -func (p *process) WorkingDirectory() string { +func (p *Process) WorkingDirectory() string { return p.Files().WorkingDirectory() } -func (p *process) SetWorkingDirectory(wd string) error { +func (p *Process) SetWorkingDirectory(wd string) error { return p.setFilesWD(wd) } -func (p *process) String() string { +func (p *Process) String() string { return fmt.Sprintf("PID=%s, Command=%v, State=%s, WD=%s, Attr=%+v, Err=%+v, Files:\n%v", p.pid, p.args, p.state, p.WorkingDirectory(), p.attr, p.err, p.fileDescriptors) } diff --git a/internal/process/process_js.go b/internal/process/process_js.go index 3b152d7..504d9d6 100644 --- a/internal/process/process_js.go +++ b/internal/process/process_js.go @@ -12,7 +12,7 @@ var ( jsGo = js.Global().Get("Go") ) -func (p *process) JSValue() js.Value { +func (p *Process) JSValue() js.Value { return js.ValueOf(map[string]interface{}{ "pid": p.pid, "ppid": p.parentPID, @@ -20,6 +20,6 @@ func (p *process) JSValue() js.Value { }) } -func (p *process) StartCPUProfile() error { +func (p *Process) StartCPUProfile() error { return interop.StartCPUProfile(p.ctx) } diff --git a/internal/process/process_other.go b/internal/process/process_other.go index 867ede1..dd1cf50 100644 --- a/internal/process/process_other.go +++ b/internal/process/process_other.go @@ -8,7 +8,7 @@ import ( "os/exec" ) -func (p *process) run(path string) { +func (p *Process) run(path string) { cmd := exec.Command(path, p.args...) if p.attr.Env == nil { cmd.Env = os.Environ() diff --git a/internal/process/wasm.go b/internal/process/wasm.go index 058a7a1..6f47c53 100644 --- a/internal/process/wasm.go +++ b/internal/process/wasm.go @@ -16,11 +16,11 @@ var ( jsObject = js.Global().Get("Object") ) -func (p *process) newWasmInstance(path string, importObject js.Value) (js.Value, error) { +func (p *Process) newWasmInstance(path string, importObject js.Value) (js.Value, error) { return p.Files().WasmInstance(path, importObject) } -func (p *process) run(path string) { +func (p *Process) run(path string) { defer func() { go runtime.GC() }() @@ -36,7 +36,7 @@ func (p *process) run(path string) { p.handleErr(err) } -func (p *process) startWasmPromise(path string, exitChan chan<- int) (promise.Promise, error) { +func (p *Process) startWasmPromise(path string, exitChan chan<- int) (promise.Promise, error) { p.state = stateCompiling goInstance := jsGo.New() goInstance.Set("argv", interop.SliceFromStrings(p.args)) From 1264aba333a6c63e59930dbbf5280b2d683e26b3 Mon Sep 17 00:00:00 2001 From: John Starich Date: Sat, 2 Apr 2022 18:10:01 -0500 Subject: [PATCH 05/37] Add (js)worker concepts and message system, extract process construction, remove context switching --- internal/common/fid.go | 9 ++++ internal/fs/file_descriptors.go | 66 +++++++++++++++++-------- internal/jsworker/local.go | 38 +++++++++++++++ internal/jsworker/message_event.go | 30 ++++++++++++ internal/jsworker/message_port.go | 77 ++++++++++++++++++++++++++++++ internal/jsworker/remote.go | 58 ++++++++++++++++++++++ internal/jsworker/types.go | 35 ++++++++++++++ internal/kernel/kernel.go | 18 +++++++ internal/process/context.go | 76 ----------------------------- internal/process/process.go | 26 +++------- internal/process/wasm.go | 55 ++++++++------------- internal/worker/local.go | 38 +++++++++++++++ internal/worker/remote.go | 29 +++++++++++ 13 files changed, 404 insertions(+), 151 deletions(-) create mode 100644 internal/jsworker/local.go create mode 100644 internal/jsworker/message_event.go create mode 100644 internal/jsworker/message_port.go create mode 100644 internal/jsworker/remote.go create mode 100644 internal/jsworker/types.go create mode 100644 internal/kernel/kernel.go delete mode 100644 internal/process/context.go create mode 100644 internal/worker/local.go create mode 100644 internal/worker/remote.go diff --git a/internal/common/fid.go b/internal/common/fid.go index e34468f..bb51874 100644 --- a/internal/common/fid.go +++ b/internal/common/fid.go @@ -2,6 +2,8 @@ package common import ( "fmt" + + "github.com/hack-pad/hackpadfs" ) type FID uint64 @@ -12,3 +14,10 @@ func (f *FID) String() string { } return fmt.Sprintf("%d", *f) } + +type OpenFileAttr struct { + FilePath string + SeekOffset int64 + Flags int + Mode hackpadfs.FileMode +} diff --git a/internal/fs/file_descriptors.go b/internal/fs/file_descriptors.go index 3d5849c..07c93de 100644 --- a/internal/fs/file_descriptors.go +++ b/internal/fs/file_descriptors.go @@ -49,35 +49,63 @@ func NewStdFileDescriptors(parentPID common.PID, workingDirectory string) (*File return f, err } -func NewFileDescriptors(parentPID common.PID, workingDirectory string, parentFiles *FileDescriptors, inheritFDs []Attr) (*FileDescriptors, func(wd string) error, error) { +func NewFileDescriptors(parentPID common.PID, workingDirectory string, openFiles []common.OpenFileAttr) (_ *FileDescriptors, _ func(wd string) error, returnedErr error) { f := &FileDescriptors{ parentPID: parentPID, previousFID: 0, files: make(map[FID]*fileDescriptor), workingDirectory: newWorkingDirectory(workingDirectory), } - if len(inheritFDs) == 0 { - inheritFDs = []Attr{{FID: 0}, {FID: 1}, {FID: 2}} + type openFile struct { + attr common.OpenFileAttr + file hackpadfs.File } - if len(inheritFDs) < 3 { - return nil, nil, errors.Errorf("Invalid number of inherited file descriptors, must be 0 or at least 3: %#v", inheritFDs) - } - for _, attr := range inheritFDs { - var inheritFD FID - switch { - case attr.Ignore: - return nil, nil, errors.New("Ignored file descriptors are unsupported") // TODO be sure to align FDs properly when skipping iterations - case attr.Pipe: - return nil, nil, errors.New("Pipe file descriptors are unsupported") // TODO align FDs like Ignore, but child FIDs on stdio property must be different than the real FIDs (see node docs) - default: - inheritFD = attr.FID + var files []openFile + defer func() { + if returnedErr != nil { + for _, f := range files { + f.file.Close() + } + } + }() + switch { + case len(openFiles) == 0: + stdin, err := getFile("/dev/stdin", 0, 0) + if err != nil { + return nil, nil, err } - parentFD := parentFiles.files[inheritFD] - if parentFD == nil { - return nil, nil, errors.Errorf("Invalid parent FID %d", attr.FID) + stdout, err := getFile("/dev/stdout", 0, 0) + if err != nil { + return nil, nil, err } + stderr, err := getFile("/dev/stderr", 0, 0) + if err != nil { + return nil, nil, err + } + files = append(files, + openFile{common.OpenFileAttr{FilePath: "/dev/stdin"}, stdin}, + openFile{common.OpenFileAttr{FilePath: "/dev/stdout"}, stdout}, + openFile{common.OpenFileAttr{FilePath: "/dev/stderr"}, stderr}, + ) + case len(openFiles) < 3: + return nil, nil, errors.Errorf("Invalid number of inherited file descriptors, must be 0 or at least 3: %#v", openFiles) + default: + for _, attr := range openFiles { + file, err := getFile(attr.FilePath, attr.Flags, attr.Mode) + if err != nil { + return nil, nil, err + } + files = append(files, openFile{attr, file}) + _, err = hackpadfs.SeekFile(file, attr.SeekOffset, io.SeekStart) + if err != nil { + return nil, nil, err + } + } + } + + for _, file := range files { fid := f.newFID() - fd := parentFD.Dup(fid) + fd := newIrregularFileDescriptor(fid, path.Base(file.attr.FilePath), file.file, file.attr.Mode) f.addFileDescriptor(fd) fd.Open(parentPID) } diff --git a/internal/jsworker/local.go b/internal/jsworker/local.go new file mode 100644 index 0000000..df87f66 --- /dev/null +++ b/internal/jsworker/local.go @@ -0,0 +1,38 @@ +package jsworker + +import ( + "context" + "syscall/js" +) + +type Local struct { + port *MessagePort +} + +var self *Local + +func init() { + jsSelf := js.Global().Get("self") + if !jsSelf.Truthy() { + return + } + port, err := wrapMessagePort(jsSelf) + if err != nil { + panic(err) + } + self = &Local{ + port: port, + } +} + +func GetLocal() *Local { + return self +} + +func (l *Local) PostMessage(message js.Value, transfers []js.Value) error { + return l.port.PostMessage(message, transfers) +} + +func (l *Local) Listen(ctx context.Context, listener func(MessageEvent, error)) error { + return l.port.Listen(ctx, listener) +} diff --git a/internal/jsworker/message_event.go b/internal/jsworker/message_event.go new file mode 100644 index 0000000..6d328b3 --- /dev/null +++ b/internal/jsworker/message_event.go @@ -0,0 +1,30 @@ +package jsworker + +import ( + "fmt" + "syscall/js" + + "github.com/hack-pad/hackpad/internal/common" +) + +type MessageEvent struct { + Data js.Value + Target *MessagePort +} + +func parseMessageEvent(v js.Value) (_ MessageEvent, err error) { + defer common.CatchException(&err) + target, err := wrapMessagePort(v.Get("target")) + return MessageEvent{ + Data: v.Get("data"), + Target: target, + }, err +} + +type MessageEventErr struct { + MessageEvent +} + +func (m MessageEventErr) Error() string { + return fmt.Sprintf("Failed to deserialize message: %+v", m.MessageEvent) +} diff --git a/internal/jsworker/message_port.go b/internal/jsworker/message_port.go new file mode 100644 index 0000000..1c176ae --- /dev/null +++ b/internal/jsworker/message_port.go @@ -0,0 +1,77 @@ +package jsworker + +import ( + "context" + "errors" + "syscall/js" + + "github.com/hack-pad/hackpad/internal/common" + "github.com/hack-pad/hackpad/internal/interop" +) + +type MessagePort struct { + jsMessagePort js.Value +} + +var jsMessageChannel = js.Global().Get("MessageChannel") + +func NewChannel() (port1, port2 *MessagePort, err error) { + defer common.CatchException(&err) + channel := jsMessageChannel.New() + port1, err = wrapMessagePort(channel.Get("port1")) + if err != nil { + return + } + port2, err = wrapMessagePort(channel.Get("port2")) + return +} + +func wrapMessagePort(v js.Value) (*MessagePort, error) { + if !v.Get("postMessage").Truthy() { + return nil, errors.New("invalid MessagePort value: postMessage is not a function") + } + return &MessagePort{v}, nil +} + +func (p *MessagePort) PostMessage(message js.Value, transfers []js.Value) (err error) { + defer common.CatchException(&err) + args := append([]interface{}{message}, interop.SliceFromJSValues(transfers)...) + p.jsMessagePort.Call("postMessage", args...) + return nil +} + +func (p *MessagePort) Listen(ctx context.Context, listener func(MessageEvent, error)) (err error) { + ctx, cancel := context.WithCancel(ctx) + defer common.CatchExceptionHandler(func(e error) { + err = e + cancel() + }) + + messageHandler := js.FuncOf(func(this js.Value, args []js.Value) interface{} { + ev, err := parseMessageEvent(args[0]) + listener(ev, err) + return nil + }) + errorHandler := js.FuncOf(func(this js.Value, args []js.Value) interface{} { + ev, err := parseMessageEvent(args[0]) + if err == nil { + err = MessageEventErr{ev} + } + listener(MessageEvent{}, err) + return nil + }) + + go func() { + <-ctx.Done() + defer messageHandler.Release() + defer errorHandler.Release() + p.jsMessagePort.Call("removeEventListener", "message", messageHandler) + p.jsMessagePort.Call("removeEventListener", "messageerror", errorHandler) + }() + p.jsMessagePort.Call("addEventListener", "message", messageHandler) + p.jsMessagePort.Call("addEventListener", "messageerror", errorHandler) + if p.jsMessagePort.Get("start").Truthy() { + p.jsMessagePort.Call("start") + } + return nil +} diff --git a/internal/jsworker/remote.go b/internal/jsworker/remote.go new file mode 100644 index 0000000..ea36f70 --- /dev/null +++ b/internal/jsworker/remote.go @@ -0,0 +1,58 @@ +package jsworker + +import ( + "context" + "syscall/js" + + "github.com/hack-pad/hackpad/internal/common" +) + +var ( + jsWorker = js.Global().Get("Worker") +) + +const ( + wasmWorkerScript = "/wasmWorker.js" +) + +type Remote struct { + port *MessagePort + worker js.Value +} + +func NewRemote(name, url string) (_ *Remote, err error) { + defer common.CatchException(&err) + val := jsWorker.New(url, map[string]interface{}{ + "name": name, + }) + port, err := wrapMessagePort(val) + return &Remote{ + port: port, + worker: val, + }, err +} + +func NewRemoteWasm(ctx context.Context, name, wasmURL string) (*Remote, error) { + ctx, cancel := context.WithCancel(ctx) + defer cancel() + l, err := NewRemote(name, wasmWorkerScript+"?wasm="+wasmURL) + if err != nil { + return nil, err + } + err = l.port.Listen(ctx, func(ev MessageEvent, err error) { + if jsString(ev.Data) == "ready" { + cancel() + } + }) + if err != nil { + return nil, err + } + <-ctx.Done() + return l, err +} + +func (l *Remote) Terminate() (err error) { + defer common.CatchException(&err) + l.worker.Call("terminate") + return nil +} diff --git a/internal/jsworker/types.go b/internal/jsworker/types.go new file mode 100644 index 0000000..b48193d --- /dev/null +++ b/internal/jsworker/types.go @@ -0,0 +1,35 @@ +package jsworker + +import ( + "syscall/js" + + "github.com/pkg/errors" +) + +func jsInt(v js.Value) int { + if v.Type() != js.TypeNumber { + return 0 + } + return v.Int() +} + +func jsString(v js.Value) string { + if v.Type() != js.TypeString { + return "" + } + return v.String() +} + +func jsBool(v js.Value) bool { + if v.Type() != js.TypeBoolean { + return false + } + return v.Bool() +} + +func jsError(v js.Value) error { + if !v.Truthy() { + return nil + } + return errors.Errorf("%v", v) +} diff --git a/internal/kernel/kernel.go b/internal/kernel/kernel.go new file mode 100644 index 0000000..cd1a0a2 --- /dev/null +++ b/internal/kernel/kernel.go @@ -0,0 +1,18 @@ +package kernel + +import ( + "github.com/hack-pad/hackpad/internal/common" + "go.uber.org/atomic" +) + +const ( + minPID = 1 +) + +var ( + lastPID = atomic.NewUint64(minPID) +) + +func ReservePID() common.PID { + return common.PID(lastPID.Inc()) +} diff --git a/internal/process/context.go b/internal/process/context.go deleted file mode 100644 index 9d1760c..0000000 --- a/internal/process/context.go +++ /dev/null @@ -1,76 +0,0 @@ -package process - -import ( - "strings" - "syscall" - - "github.com/hack-pad/hackpad/internal/fs" - "github.com/hack-pad/hackpad/internal/log" -) - -const initialDirectory = "/home/me" - -var ( - currentPID PID - - switchedContextListener func(newPID, parentPID PID) -) - -func Init(switchedContext func(PID, PID)) { - // create 'init' process - fileDescriptors, err := fs.NewStdFileDescriptors(minPID, initialDirectory) - if err != nil { - panic(err) - } - p, err := newWithCurrent( - &Process{fileDescriptors: fileDescriptors}, - minPID, - "", - nil, - &ProcAttr{Env: splitEnvPairs(syscall.Environ())}, - ) - if err != nil { - panic(err) - } - p.state = stateRunning - pids[minPID] = p - - switchedContextListener = switchedContext - switchContext(minPID) -} - -func switchContext(pid PID) (prev PID) { - prev = currentPID - log.Debug("Switching context from PID ", prev, " to ", pid) - if pid == prev { - return - } - newProcess := pids[pid] - currentPID = pid - switchedContextListener(pid, newProcess.parentPID) - return -} - -func Current() *Process { - process, _ := Get(currentPID) - return process -} - -func Get(pid PID) (process *Process, ok bool) { - p, ok := pids[pid] - return p, ok -} - -func splitEnvPairs(pairs []string) map[string]string { - env := make(map[string]string) - for _, pair := range pairs { - equalIndex := strings.IndexRune(pair, '=') - if equalIndex == -1 { - env[pair] = "" - } else { - key, value := pair[:equalIndex], pair[equalIndex+1:] - env[key] = value - } - } - return env -} diff --git a/internal/process/process.go b/internal/process/process.go index 46f7c67..3b53d08 100644 --- a/internal/process/process.go +++ b/internal/process/process.go @@ -12,11 +12,6 @@ import ( "github.com/hack-pad/hackpad/internal/log" "github.com/hack-pad/hackpadfs/keyvalue/blob" "github.com/pkg/errors" - "go.uber.org/atomic" -) - -const ( - minPID = 1 ) type PID = common.PID @@ -32,8 +27,7 @@ const ( ) var ( - pids = make(map[PID]*Process) - lastPID = atomic.NewUint64(minPID) + pids = make(map[PID]*Process) ) type Process struct { @@ -41,7 +35,7 @@ type Process struct { command string args []string state processState - attr *ProcAttr + env map[string]string ctx context.Context ctxDone context.CancelFunc exitCode int @@ -50,23 +44,15 @@ type Process struct { setFilesWD func(wd string) error } -func New(command string, args []string, attr *ProcAttr) (*Process, error) { - return newWithCurrent(Current(), PID(lastPID.Inc()), command, args, attr) -} - -func newWithCurrent(current *Process, newPID PID, command string, args []string, attr *ProcAttr) (*Process, error) { - wd := current.WorkingDirectory() - if attr.Dir != "" { - wd = attr.Dir - } - files, setFilesWD, err := fs.NewFileDescriptors(newPID, wd, current.Files(), attr.Files) +func New(newPID PID, command string, args []string, workingDirectory string, openFiles []common.OpenFileAttr, env map[string]string) (*Process, error) { + files, setFilesWD, err := fs.NewFileDescriptors(newPID, workingDirectory, openFiles) ctx, cancel := context.WithCancel(context.Background()) return &Process{ pid: newPID, command: command, args: args, state: statePending, - attr: attr, + env: env, ctx: ctx, ctxDone: cancel, err: err, @@ -162,7 +148,7 @@ func (p *Process) SetWorkingDirectory(wd string) error { } func (p *Process) String() string { - return fmt.Sprintf("PID=%s, Command=%v, State=%s, WD=%s, Attr=%+v, Err=%+v, Files:\n%v", p.pid, p.args, p.state, p.WorkingDirectory(), p.attr, p.err, p.fileDescriptors) + return fmt.Sprintf("PID=%s, Command=%v, State=%s, WD=%s, Attr=%+v, Err=%+v, Files:\n%v", p.pid, p.args, p.state, p.WorkingDirectory(), p.env, p.err, p.fileDescriptors) } func Dump() interface{} { diff --git a/internal/process/wasm.go b/internal/process/wasm.go index 6f47c53..b9d144e 100644 --- a/internal/process/wasm.go +++ b/internal/process/wasm.go @@ -5,6 +5,7 @@ package process import ( "os" "runtime" + "strings" "syscall/js" "github.com/hack-pad/hackpad/internal/interop" @@ -40,16 +41,18 @@ func (p *Process) startWasmPromise(path string, exitChan chan<- int) (promise.Pr p.state = stateCompiling goInstance := jsGo.New() goInstance.Set("argv", interop.SliceFromStrings(p.args)) - if p.attr.Env == nil { - p.attr.Env = splitEnvPairs(os.Environ()) + if p.env == nil { + p.env = splitEnvPairs(os.Environ()) } - goInstance.Set("env", interop.StringMap(p.attr.Env)) + goInstance.Set("env", interop.StringMap(p.env)) var resumeFuncPtr *js.Func goInstance.Set("exit", interop.SingleUseFunc(func(this js.Value, args []js.Value) interface{} { defer func() { if resumeFuncPtr != nil { resumeFuncPtr.Release() } + // TODO exit hook for worker + // TODO free the whole goInstance to fix garbage issues entirely. Freeing individual properties appears to work for now, but is ultimately a bad long-term solution because memory still accumulates. goInstance.Set("mem", js.Null()) goInstance.Set("importObject", js.Null()) @@ -72,40 +75,20 @@ func (p *Process) startWasmPromise(path string, exitChan chan<- int) (promise.Pr return nil, err } - exports := instance.Get("exports") + p.state = stateRunning + return promise.From(goInstance.Call("run", instance)), nil +} - resumeFunc := js.FuncOf(func(this js.Value, args []js.Value) interface{} { - defer interop.PanicLogger() - prev := switchContext(p.pid) - ret := exports.Call("resume", interop.SliceFromJSValues(args)...) - switchContext(prev) - return ret - }) - resumeFuncPtr = &resumeFunc - wrapperExports := map[string]interface{}{ - "run": interop.SingleUseFunc(func(this js.Value, args []js.Value) interface{} { - defer interop.PanicLogger() - prev := switchContext(p.pid) - ret := exports.Call("run", interop.SliceFromJSValues(args)...) - switchContext(prev) - return ret - }), - "resume": resumeFunc, - } - for export, value := range interop.Entries(exports) { - _, overridden := wrapperExports[export] - if !overridden { - wrapperExports[export] = value +func splitEnvPairs(pairs []string) map[string]string { + env := make(map[string]string) + for _, pair := range pairs { + equalIndex := strings.IndexRune(pair, '=') + if equalIndex == -1 { + env[pair] = "" + } else { + key, value := pair[:equalIndex], pair[equalIndex+1:] + env[key] = value } } - wrapperInstance := jsObject.Call("defineProperty", - jsObject.Call("create", instance), - "exports", map[string]interface{}{ // Instance.exports is read-only, so create a shim - "value": wrapperExports, - "writable": false, - }, - ) - - p.state = stateRunning - return promise.From(goInstance.Call("run", wrapperInstance)), nil + return env } diff --git a/internal/worker/local.go b/internal/worker/local.go new file mode 100644 index 0000000..63077ed --- /dev/null +++ b/internal/worker/local.go @@ -0,0 +1,38 @@ +package worker + +import ( + "github.com/hack-pad/hackpad/internal/jsworker" + "github.com/hack-pad/hackpad/internal/kernel" + "github.com/hack-pad/hackpad/internal/log" + "github.com/hack-pad/hackpad/internal/process" +) + +type Local struct { + process *process.Process +} + +func New(localJS *jsworker.Local) (*Local, error) { + ctx, cancel := context.WithCancel(context.Background()) + localChan := make(chan *Local, 1) + localJS.Listen(ctx, func(me jsworker.MessageEvent, err error) { + if err != nil { + log.Error(err) + return + } + initData := me.Data.Get("init") + if !initData.Truthy() { + return + } + process := process.New() + localChan <- &Local{ + initData. + } + }) + return nil, nil +} + +func (l *Local) Fork(command string, args []string, attr *process.ProcAttr) (*Remote, error) { + remote, err := NewRemote(l, kernel.ReservePID(), command, args, attr) + // TODO worker init + return remote, err +} diff --git a/internal/worker/remote.go b/internal/worker/remote.go new file mode 100644 index 0000000..261beb3 --- /dev/null +++ b/internal/worker/remote.go @@ -0,0 +1,29 @@ +package worker + +import ( + "github.com/hack-pad/hackpad/internal/process" +) + +type Remote struct { +} + +type openFile struct { + filePath string + seekOffset uint +} + +func NewRemote(local *Local, pid process.PID, command string, args []string, attr *process.ProcAttr) (*Remote, error) { + var openFiles []openFile + for _, f := range attr.Files { + info, err := local.process.Files().Fstat(f.FID) + if err != nil { + return nil, err + } + openFiles = append(openFiles, openFile{ + filePath: info.Name(), + seekOffset: 0, // TODO expose seek offset in file descriptor + }) + } + // TODO + return &Remote{}, nil +} From 044a5a29033c0df42a0d8296742a1058cf8321b5 Mon Sep 17 00:00:00 2001 From: John Starich Date: Sat, 2 Apr 2022 21:48:11 -0500 Subject: [PATCH 06/37] Fix shims, pass process into init --- install.go | 14 ++-- internal/fs/file_descriptors.go | 6 +- internal/interop/values.go | 8 +++ internal/js/fs/chmod.go | 10 ++- internal/js/fs/chown.go | 10 +-- internal/js/fs/close.go | 10 ++- internal/js/fs/fchmod.go | 10 ++- internal/js/fs/flock.go | 14 ++-- internal/js/fs/fs.go | 119 ++++++++++++++++---------------- internal/js/fs/fstat.go | 10 ++- internal/js/fs/fsync.go | 10 ++- internal/js/fs/ftruncate.go | 10 ++- internal/js/fs/lstat.go | 10 ++- internal/js/fs/mkdir.go | 12 ++-- internal/js/fs/open.go | 10 ++- internal/js/fs/overlay.go | 15 ++-- internal/js/fs/pipe.go | 10 ++- internal/js/fs/read.go | 14 ++-- internal/js/fs/readdir.go | 10 ++- internal/js/fs/rename.go | 10 ++- internal/js/fs/rmdir.go | 10 ++- internal/js/fs/stat.go | 10 ++- internal/js/fs/unlink.go | 10 ++- internal/js/fs/utimes.go | 10 ++- internal/js/fs/write.go | 10 ++- internal/js/process/dir.go | 10 ++- internal/js/process/groups.go | 6 +- internal/js/process/process.go | 46 ++++++------ internal/js/process/spawn.go | 17 ++--- internal/js/process/umask.go | 2 +- internal/js/process/wait.go | 20 +++--- internal/terminal/term.go | 17 +++-- internal/worker/dom.go | 31 +++++++++ internal/worker/local.go | 52 +++++++++++--- main.go | 57 ++++++++------- server/src/Hackpad.js | 31 +++++++++ 36 files changed, 361 insertions(+), 300 deletions(-) create mode 100644 internal/worker/dom.go diff --git a/install.go b/install.go index 3d61311..58935b2 100644 --- a/install.go +++ b/install.go @@ -11,14 +11,13 @@ import ( "github.com/hack-pad/hackpad/internal/interop" "github.com/hack-pad/hackpad/internal/log" - "github.com/hack-pad/hackpad/internal/process" "github.com/hack-pad/hackpad/internal/promise" ) -func installFunc(this js.Value, args []js.Value) interface{} { +func (s domShim) installFunc(this js.Value, args []js.Value) interface{} { resolve, reject, prom := promise.New() go func() { - err := install(args) + err := s.install(args) if err != nil { reject(interop.WrapAsJSError(err, "Failed to install binary")) return @@ -28,7 +27,7 @@ func installFunc(this js.Value, args []js.Value) interface{} { return prom } -func install(args []js.Value) error { +func (s domShim) install(args []js.Value) error { if len(args) != 1 { return errors.New("Expected command name to install") } @@ -44,13 +43,12 @@ func install(args []js.Value) error { return err } defer runtime.GC() - fs := process.Current().Files() - fd, err := fs.Open("/bin/"+command, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0750) + file, err := os.OpenFile("/bin/"+command, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0750) if err != nil { return err } - defer fs.Close(fd) - if _, err := fs.Write(fd, body, 0, body.Len(), nil); err != nil { + defer file.Close() + if _, err := file.Write(body.Bytes()); err != nil { return err } log.Print("Install completed: ", command) diff --git a/internal/fs/file_descriptors.go b/internal/fs/file_descriptors.go index 07c93de..b7f2e75 100644 --- a/internal/fs/file_descriptors.go +++ b/internal/fs/file_descriptors.go @@ -70,15 +70,15 @@ func NewFileDescriptors(parentPID common.PID, workingDirectory string, openFiles }() switch { case len(openFiles) == 0: - stdin, err := getFile("/dev/stdin", 0, 0) + stdin, err := getFile("dev/stdin", 0, 0) if err != nil { return nil, nil, err } - stdout, err := getFile("/dev/stdout", 0, 0) + stdout, err := getFile("dev/stdout", 0, 0) if err != nil { return nil, nil, err } - stderr, err := getFile("/dev/stderr", 0, 0) + stderr, err := getFile("dev/stderr", 0, 0) if err != nil { return nil, nil, err } diff --git a/internal/interop/values.go b/internal/interop/values.go index eaf09af..2de0d8a 100644 --- a/internal/interop/values.go +++ b/internal/interop/values.go @@ -65,3 +65,11 @@ func StringMap(m map[string]string) js.Value { } return js.ValueOf(jsValue) } + +func StringMapFromJSObject(v js.Value) map[string]string { + m := make(map[string]string) + for key, value := range Entries(v) { + m[key] = value.String() + } + return m +} diff --git a/internal/js/fs/chmod.go b/internal/js/fs/chmod.go index 6977ba5..805d7c7 100644 --- a/internal/js/fs/chmod.go +++ b/internal/js/fs/chmod.go @@ -6,22 +6,20 @@ import ( "os" "syscall/js" - "github.com/hack-pad/hackpad/internal/process" "github.com/pkg/errors" ) -func chmod(args []js.Value) ([]interface{}, error) { - _, err := chmodSync(args) +func (s fileShim) chmod(args []js.Value) ([]interface{}, error) { + _, err := s.chmodSync(args) return nil, err } -func chmodSync(args []js.Value) (interface{}, error) { +func (s fileShim) chmodSync(args []js.Value) (interface{}, error) { if len(args) != 2 { return nil, errors.Errorf("Invalid number of args, expected 2: %v", args) } path := args[0].String() mode := os.FileMode(args[1].Int()) - p := process.Current() - return nil, p.Files().Chmod(path, mode) + return nil, s.process.Files().Chmod(path, mode) } diff --git a/internal/js/fs/chown.go b/internal/js/fs/chown.go index 779671a..62539b4 100644 --- a/internal/js/fs/chown.go +++ b/internal/js/fs/chown.go @@ -8,12 +8,12 @@ import ( "github.com/pkg/errors" ) -func chown(args []js.Value) ([]interface{}, error) { - _, err := chownSync(args) +func (s fileShim) chown(args []js.Value) ([]interface{}, error) { + _, err := s.chownSync(args) return nil, err } -func chownSync(args []js.Value) (interface{}, error) { +func (s fileShim) chownSync(args []js.Value) (interface{}, error) { if len(args) != 3 { return nil, errors.Errorf("Invalid number of args, expected 3: %v", args) } @@ -21,10 +21,10 @@ func chownSync(args []js.Value) (interface{}, error) { path := args[0].String() uid := args[1].Int() gid := args[2].Int() - return nil, Chown(path, uid, gid) + return nil, s.Chown(path, uid, gid) } -func Chown(path string, uid, gid int) error { +func (s fileShim) Chown(path string, uid, gid int) error { // TODO no-op, consider adding user and group ID support to hackpadfs return nil } diff --git a/internal/js/fs/close.go b/internal/js/fs/close.go index c1c8919..41c9b67 100644 --- a/internal/js/fs/close.go +++ b/internal/js/fs/close.go @@ -6,22 +6,20 @@ import ( "syscall/js" "github.com/hack-pad/hackpad/internal/fs" - "github.com/hack-pad/hackpad/internal/process" "github.com/pkg/errors" ) -func closeFn(args []js.Value) ([]interface{}, error) { - ret, err := closeSync(args) +func (s fileShim) closeFn(args []js.Value) ([]interface{}, error) { + ret, err := s.closeSync(args) return []interface{}{ret}, err } -func closeSync(args []js.Value) (interface{}, error) { +func (s fileShim) closeSync(args []js.Value) (interface{}, error) { if len(args) != 1 { return nil, errors.Errorf("not enough args %d", len(args)) } fd := fs.FID(args[0].Int()) - p := process.Current() - err := p.Files().Close(fd) + err := s.process.Files().Close(fd) return nil, err } diff --git a/internal/js/fs/fchmod.go b/internal/js/fs/fchmod.go index 8c04fa5..32e9589 100644 --- a/internal/js/fs/fchmod.go +++ b/internal/js/fs/fchmod.go @@ -7,22 +7,20 @@ import ( "syscall/js" "github.com/hack-pad/hackpad/internal/common" - "github.com/hack-pad/hackpad/internal/process" "github.com/pkg/errors" ) -func fchmod(args []js.Value) ([]interface{}, error) { - _, err := fchmodSync(args) +func (s fileShim) fchmod(args []js.Value) ([]interface{}, error) { + _, err := s.fchmodSync(args) return nil, err } -func fchmodSync(args []js.Value) (interface{}, error) { +func (s fileShim) fchmodSync(args []js.Value) (interface{}, error) { if len(args) != 2 { return nil, errors.Errorf("Invalid number of args, expected 2: %v", args) } fid := common.FID(args[0].Int()) mode := os.FileMode(args[1].Int()) - p := process.Current() - return nil, p.Files().Fchmod(fid, mode) + return nil, s.process.Files().Fchmod(fid, mode) } diff --git a/internal/js/fs/flock.go b/internal/js/fs/flock.go index cf9f041..7e68b1f 100644 --- a/internal/js/fs/flock.go +++ b/internal/js/fs/flock.go @@ -8,16 +8,15 @@ import ( "github.com/hack-pad/hackpad/internal/common" "github.com/hack-pad/hackpad/internal/fs" - "github.com/hack-pad/hackpad/internal/process" "github.com/pkg/errors" ) -func flock(args []js.Value) ([]interface{}, error) { - _, err := flockSync(args) +func (s fileShim) flock(args []js.Value) ([]interface{}, error) { + _, err := s.flockSync(args) return nil, err } -func flockSync(args []js.Value) (interface{}, error) { +func (s fileShim) flockSync(args []js.Value) (interface{}, error) { if len(args) != 2 { return nil, errors.Errorf("Invalid number of args, expected 2: %v", args) } @@ -34,10 +33,9 @@ func flockSync(args []js.Value) (interface{}, error) { action = fs.Unlock } - return nil, Flock(fid, action, shouldLock) + return nil, s.Flock(fid, action, shouldLock) } -func Flock(fid common.FID, action fs.LockAction, shouldLock bool) error { - p := process.Current() - return p.Files().Flock(fid, action) +func (s fileShim) Flock(fid common.FID, action fs.LockAction, shouldLock bool) error { + return s.process.Files().Flock(fid, action) } diff --git a/internal/js/fs/fs.go b/internal/js/fs/fs.go index 3d84e68..08fff2c 100644 --- a/internal/js/fs/fs.go +++ b/internal/js/fs/fs.go @@ -16,16 +16,13 @@ import ( "github.com/hack-pad/hackpad/internal/promise" ) -/* -fchown(fd, uid, gid, callback) { callback(enosys()); }, -lchown(path, uid, gid, callback) { callback(enosys()); }, -link(path, link, callback) { callback(enosys()); }, -readlink(path, callback) { callback(enosys()); }, -symlink(path, link, callback) { callback(enosys()); }, -truncate(path, length, callback) { callback(enosys()); }, -*/ +type fileShim struct { + process *process.Process +} + +func Init(process *process.Process) { + shim := fileShim{process} -func Init() { fs := js.Global().Get("fs") constants := fs.Get("constants") constants.Set("O_RDONLY", syscall.O_RDONLY) @@ -35,75 +32,75 @@ func Init() { constants.Set("O_TRUNC", syscall.O_TRUNC) constants.Set("O_APPEND", syscall.O_APPEND) constants.Set("O_EXCL", syscall.O_EXCL) - interop.SetFunc(fs, "chmod", chmod) - interop.SetFunc(fs, "chmodSync", chmodSync) - interop.SetFunc(fs, "chown", chown) - interop.SetFunc(fs, "chownSync", chownSync) - interop.SetFunc(fs, "close", closeFn) - interop.SetFunc(fs, "closeSync", closeSync) - interop.SetFunc(fs, "fchmod", fchmod) - interop.SetFunc(fs, "fchmodSync", fchmodSync) - interop.SetFunc(fs, "flock", flock) - interop.SetFunc(fs, "flockSync", flockSync) - interop.SetFunc(fs, "fstat", fstat) - interop.SetFunc(fs, "fstatSync", fstatSync) - interop.SetFunc(fs, "fsync", fsync) - interop.SetFunc(fs, "fsyncSync", fsyncSync) - interop.SetFunc(fs, "ftruncate", ftruncate) - interop.SetFunc(fs, "ftruncateSync", ftruncateSync) - interop.SetFunc(fs, "lstat", lstat) - interop.SetFunc(fs, "lstatSync", lstatSync) - interop.SetFunc(fs, "mkdir", mkdir) - interop.SetFunc(fs, "mkdirSync", mkdirSync) - interop.SetFunc(fs, "open", open) - interop.SetFunc(fs, "openSync", openSync) - interop.SetFunc(fs, "pipe", pipe) - interop.SetFunc(fs, "pipeSync", pipeSync) - interop.SetFunc(fs, "read", read) - interop.SetFunc(fs, "readSync", readSync) - interop.SetFunc(fs, "readdir", readdir) - interop.SetFunc(fs, "readdirSync", readdirSync) - interop.SetFunc(fs, "rename", rename) - interop.SetFunc(fs, "renameSync", renameSync) - interop.SetFunc(fs, "rmdir", rmdir) - interop.SetFunc(fs, "rmdirSync", rmdirSync) - interop.SetFunc(fs, "stat", stat) - interop.SetFunc(fs, "statSync", statSync) - interop.SetFunc(fs, "unlink", unlink) - interop.SetFunc(fs, "unlinkSync", unlinkSync) - interop.SetFunc(fs, "utimes", utimes) - interop.SetFunc(fs, "utimesSync", utimesSync) - interop.SetFunc(fs, "write", write) - interop.SetFunc(fs, "writeSync", writeSync) + interop.SetFunc(fs, "chmod", shim.chmod) + interop.SetFunc(fs, "chmodSync", shim.chmodSync) + interop.SetFunc(fs, "chown", shim.chown) + interop.SetFunc(fs, "chownSync", shim.chownSync) + interop.SetFunc(fs, "close", shim.closeFn) + interop.SetFunc(fs, "closeSync", shim.closeSync) + interop.SetFunc(fs, "fchmod", shim.fchmod) + interop.SetFunc(fs, "fchmodSync", shim.fchmodSync) + interop.SetFunc(fs, "flock", shim.flock) + interop.SetFunc(fs, "flockSync", shim.flockSync) + interop.SetFunc(fs, "fstat", shim.fstat) + interop.SetFunc(fs, "fstatSync", shim.fstatSync) + interop.SetFunc(fs, "fsync", shim.fsync) + interop.SetFunc(fs, "fsyncSync", shim.fsyncSync) + interop.SetFunc(fs, "ftruncate", shim.ftruncate) + interop.SetFunc(fs, "ftruncateSync", shim.ftruncateSync) + interop.SetFunc(fs, "lstat", shim.lstat) + interop.SetFunc(fs, "lstatSync", shim.lstatSync) + interop.SetFunc(fs, "mkdir", shim.mkdir) + interop.SetFunc(fs, "mkdirSync", shim.mkdirSync) + interop.SetFunc(fs, "open", shim.open) + interop.SetFunc(fs, "openSync", shim.openSync) + interop.SetFunc(fs, "pipe", shim.pipe) + interop.SetFunc(fs, "pipeSync", shim.pipeSync) + interop.SetFunc(fs, "read", shim.read) + interop.SetFunc(fs, "readSync", shim.readSync) + interop.SetFunc(fs, "readdir", shim.readdir) + interop.SetFunc(fs, "readdirSync", shim.readdirSync) + interop.SetFunc(fs, "rename", shim.rename) + interop.SetFunc(fs, "renameSync", shim.renameSync) + interop.SetFunc(fs, "rmdir", shim.rmdir) + interop.SetFunc(fs, "rmdirSync", shim.rmdirSync) + interop.SetFunc(fs, "stat", shim.stat) + interop.SetFunc(fs, "statSync", shim.statSync) + interop.SetFunc(fs, "unlink", shim.unlink) + interop.SetFunc(fs, "unlinkSync", shim.unlinkSync) + interop.SetFunc(fs, "utimes", shim.utimes) + interop.SetFunc(fs, "utimesSync", shim.utimesSync) + interop.SetFunc(fs, "write", shim.write) + interop.SetFunc(fs, "writeSync", shim.writeSync) - global.Set("getMounts", js.FuncOf(getMounts)) - global.Set("destroyMount", js.FuncOf(destroyMount)) - global.Set("overlayTarGzip", js.FuncOf(overlayTarGzip)) - global.Set("overlayIndexedDB", js.FuncOf(overlayIndexedDB)) - global.Set("dumpZip", js.FuncOf(dumpZip)) + global.Set("getMounts", js.FuncOf(shim.getMounts)) + global.Set("destroyMount", js.FuncOf(shim.destroyMount)) + global.Set("overlayTarGzip", js.FuncOf(shim.overlayTarGzip)) + global.Set("overlayIndexedDB", js.FuncOf(shim.overlayIndexedDB)) + global.Set("dumpZip", js.FuncOf(shim.dumpZip)) // Set up system directories - files := process.Current().Files() + files := process.Files() if err := files.MkdirAll(os.TempDir(), 0777); err != nil { panic(err) } } -func Dump(basePath string) interface{} { - basePath = common.ResolvePath(process.Current().WorkingDirectory(), basePath) +func (s fileShim) Dump(basePath string) interface{} { + basePath = common.ResolvePath(s.process.WorkingDirectory(), basePath) return fs.Dump(basePath) } -func dumpZip(this js.Value, args []js.Value) interface{} { +func (s fileShim) dumpZip(this js.Value, args []js.Value) interface{} { if len(args) != 1 { return interop.WrapAsJSError(errors.New("dumpZip: file path is required"), "EINVAL") } path := args[0].String() - path = common.ResolvePath(process.Current().WorkingDirectory(), path) + path = common.ResolvePath(s.process.WorkingDirectory(), path) return interop.WrapAsJSError(fs.DumpZip(path), "dumpZip") } -func getMounts(this js.Value, args []js.Value) interface{} { +func (s fileShim) getMounts(this js.Value, args []js.Value) interface{} { var mounts []string for _, p := range fs.Mounts() { mounts = append(mounts, p.Path) @@ -111,7 +108,7 @@ func getMounts(this js.Value, args []js.Value) interface{} { return interop.SliceFromStrings(mounts) } -func destroyMount(this js.Value, args []js.Value) interface{} { +func (s fileShim) destroyMount(this js.Value, args []js.Value) interface{} { if len(args) < 1 { return interop.WrapAsJSError(errors.New("destroyMount: mount path is required"), "EINVAL") } diff --git a/internal/js/fs/fstat.go b/internal/js/fs/fstat.go index aed0af1..887d42e 100644 --- a/internal/js/fs/fstat.go +++ b/internal/js/fs/fstat.go @@ -6,21 +6,19 @@ import ( "syscall/js" "github.com/hack-pad/hackpad/internal/fs" - "github.com/hack-pad/hackpad/internal/process" "github.com/pkg/errors" ) -func fstat(args []js.Value) ([]interface{}, error) { - info, err := fstatSync(args) +func (s fileShim) fstat(args []js.Value) ([]interface{}, error) { + info, err := s.fstatSync(args) return []interface{}{info}, err } -func fstatSync(args []js.Value) (interface{}, error) { +func (s fileShim) fstatSync(args []js.Value) (interface{}, error) { if len(args) != 1 { return nil, errors.Errorf("Invalid number of args, expected 1: %v", args) } fd := fs.FID(args[0].Int()) - p := process.Current() - info, err := p.Files().Fstat(fd) + info, err := s.process.Files().Fstat(fd) return jsStat(info), err } diff --git a/internal/js/fs/fsync.go b/internal/js/fs/fsync.go index a42fd37..05d7c9c 100644 --- a/internal/js/fs/fsync.go +++ b/internal/js/fs/fsync.go @@ -6,22 +6,20 @@ import ( "syscall/js" "github.com/hack-pad/hackpad/internal/fs" - "github.com/hack-pad/hackpad/internal/process" "github.com/pkg/errors" ) // fsync(fd, callback) { callback(null); }, -func fsync(args []js.Value) ([]interface{}, error) { - _, err := fsyncSync(args) +func (s fileShim) fsync(args []js.Value) ([]interface{}, error) { + _, err := s.fsyncSync(args) return nil, err } -func fsyncSync(args []js.Value) (interface{}, error) { +func (s fileShim) fsyncSync(args []js.Value) (interface{}, error) { if len(args) != 1 { return nil, errors.Errorf("Invalid number of args, expected 1: %v", args) } fd := fs.FID(args[0].Int()) - p := process.Current() - return nil, p.Files().Fsync(fd) + return nil, s.process.Files().Fsync(fd) } diff --git a/internal/js/fs/ftruncate.go b/internal/js/fs/ftruncate.go index cf8d478..547fdc4 100644 --- a/internal/js/fs/ftruncate.go +++ b/internal/js/fs/ftruncate.go @@ -6,16 +6,15 @@ import ( "syscall/js" "github.com/hack-pad/hackpad/internal/fs" - "github.com/hack-pad/hackpad/internal/process" "github.com/pkg/errors" ) -func ftruncateSync(args []js.Value) (interface{}, error) { - _, err := ftruncate(args) +func (s fileShim) ftruncateSync(args []js.Value) (interface{}, error) { + _, err := s.ftruncate(args) return nil, err } -func ftruncate(args []js.Value) ([]interface{}, error) { +func (s fileShim) ftruncate(args []js.Value) ([]interface{}, error) { // args: fd, len if len(args) == 0 { return nil, errors.Errorf("missing required args, expected fd: %+v", args) @@ -26,6 +25,5 @@ func ftruncate(args []js.Value) ([]interface{}, error) { length = args[1].Int() } - p := process.Current() - return nil, p.Files().Truncate(fd, int64(length)) + return nil, s.process.Files().Truncate(fd, int64(length)) } diff --git a/internal/js/fs/lstat.go b/internal/js/fs/lstat.go index 402c65a..b54798f 100644 --- a/internal/js/fs/lstat.go +++ b/internal/js/fs/lstat.go @@ -5,21 +5,19 @@ package fs import ( "syscall/js" - "github.com/hack-pad/hackpad/internal/process" "github.com/pkg/errors" ) -func lstat(args []js.Value) ([]interface{}, error) { - info, err := lstatSync(args) +func (s fileShim) lstat(args []js.Value) ([]interface{}, error) { + info, err := s.lstatSync(args) return []interface{}{info}, err } -func lstatSync(args []js.Value) (interface{}, error) { +func (s fileShim) lstatSync(args []js.Value) (interface{}, error) { if len(args) != 1 { return nil, errors.Errorf("Invalid number of args, expected 1: %v", args) } path := args[0].String() - p := process.Current() - info, err := p.Files().Lstat(path) + info, err := s.process.Files().Lstat(path) return jsStat(info), err } diff --git a/internal/js/fs/mkdir.go b/internal/js/fs/mkdir.go index 34a9cde..c9ffc9a 100644 --- a/internal/js/fs/mkdir.go +++ b/internal/js/fs/mkdir.go @@ -6,16 +6,15 @@ import ( "os" "syscall/js" - "github.com/hack-pad/hackpad/internal/process" "github.com/pkg/errors" ) -func mkdir(args []js.Value) ([]interface{}, error) { - _, err := mkdirSync(args) +func (s fileShim) mkdir(args []js.Value) ([]interface{}, error) { + _, err := s.mkdirSync(args) return nil, err } -func mkdirSync(args []js.Value) (interface{}, error) { +func (s fileShim) mkdirSync(args []js.Value) (interface{}, error) { if len(args) != 2 { return nil, errors.Errorf("Invalid number of args, expected 2: %v", args) } @@ -35,9 +34,8 @@ func mkdirSync(args []js.Value) (interface{}, error) { recursive = true } - p := process.Current() if recursive { - return nil, p.Files().MkdirAll(path, mode) + return nil, s.process.Files().MkdirAll(path, mode) } - return nil, p.Files().Mkdir(path, mode) + return nil, s.process.Files().Mkdir(path, mode) } diff --git a/internal/js/fs/open.go b/internal/js/fs/open.go index 6d48057..6ba381d 100644 --- a/internal/js/fs/open.go +++ b/internal/js/fs/open.go @@ -6,16 +6,15 @@ import ( "os" "syscall/js" - "github.com/hack-pad/hackpad/internal/process" "github.com/pkg/errors" ) -func open(args []js.Value) ([]interface{}, error) { - fd, err := openSync(args) +func (s fileShim) open(args []js.Value) ([]interface{}, error) { + fd, err := s.openSync(args) return []interface{}{fd}, err } -func openSync(args []js.Value) (interface{}, error) { +func (s fileShim) openSync(args []js.Value) (interface{}, error) { if len(args) == 0 { return nil, errors.Errorf("Expected path, received: %v", args) } @@ -29,7 +28,6 @@ func openSync(args []js.Value) (interface{}, error) { mode = os.FileMode(args[2].Int()) } - p := process.Current() - fd, err := p.Files().Open(path, flags, mode) + fd, err := s.process.Files().Open(path, flags, mode) return fd, err } diff --git a/internal/js/fs/overlay.go b/internal/js/fs/overlay.go index d7ab43b..4af9f43 100644 --- a/internal/js/fs/overlay.go +++ b/internal/js/fs/overlay.go @@ -21,15 +21,14 @@ import ( "github.com/hack-pad/hackpad/internal/fs" "github.com/hack-pad/hackpad/internal/interop" "github.com/hack-pad/hackpad/internal/log" - "github.com/hack-pad/hackpad/internal/process" "github.com/hack-pad/hackpad/internal/promise" "github.com/johnstarich/go/datasize" ) -func overlayIndexedDB(this js.Value, args []js.Value) interface{} { +func (s fileShim) overlayIndexedDB(this js.Value, args []js.Value) interface{} { resolve, reject, prom := promise.New() go func() { - err := OverlayIndexedDB(args) + err := s.OverlayIndexedDB(args) if err != nil { reject(interop.WrapAsJSError(err, "Failed overlaying IndexedDB FS")) } else { @@ -40,7 +39,7 @@ func overlayIndexedDB(this js.Value, args []js.Value) interface{} { return prom } -func OverlayIndexedDB(args []js.Value) (err error) { +func (s fileShim) OverlayIndexedDB(args []js.Value) (err error) { if len(args) == 0 { return errors.New("overlayIndexedDB: mount path is required") } @@ -64,11 +63,11 @@ func OverlayIndexedDB(args []js.Value) (err error) { return fs.Overlay(mountPath, idbFS) } -func overlayTarGzip(this js.Value, args []js.Value) interface{} { +func (s fileShim) overlayTarGzip(this js.Value, args []js.Value) interface{} { resolve, reject, prom := promise.New() log.Debug("Backgrounding overlay request") go func() { - err := OverlayTarGzip(args) + err := s.OverlayTarGzip(args) if err != nil { reject(interop.WrapAsJSError(err, "Failed overlaying .tar.gz FS")) } else { @@ -79,7 +78,7 @@ func overlayTarGzip(this js.Value, args []js.Value) interface{} { return prom } -func OverlayTarGzip(args []js.Value) error { +func (s fileShim) OverlayTarGzip(args []js.Value) error { if len(args) < 2 { return errors.New("overlayTarGzip: mount path and .tar.gz URL path is required") } @@ -113,7 +112,7 @@ func OverlayTarGzip(args []js.Value) error { if options["skipCacheDirs"].Type() == js.TypeObject { skipDirs := make(map[string]bool) for _, d := range interop.StringsFromJSValue(options["skipCacheDirs"]) { - skipDirs[common.ResolvePath(process.Current().WorkingDirectory(), d)] = true + skipDirs[common.ResolvePath(s.process.WorkingDirectory(), d)] = true } maxFileBytes := datasize.Kibibytes(100).Bytes() shouldCache = func(name string, info hackpadfs.FileInfo) bool { diff --git a/internal/js/fs/pipe.go b/internal/js/fs/pipe.go index 52808fc..bdcb7bd 100644 --- a/internal/js/fs/pipe.go +++ b/internal/js/fs/pipe.go @@ -5,20 +5,18 @@ package fs import ( "syscall/js" - "github.com/hack-pad/hackpad/internal/process" "github.com/pkg/errors" ) -func pipe(args []js.Value) ([]interface{}, error) { - fds, err := pipeSync(args) +func (s fileShim) pipe(args []js.Value) ([]interface{}, error) { + fds, err := s.pipeSync(args) return []interface{}{fds}, err } -func pipeSync(args []js.Value) (interface{}, error) { +func (s fileShim) pipeSync(args []js.Value) (interface{}, error) { if len(args) != 0 { return nil, errors.Errorf("Invalid number of args, expected 0: %v", args) } - p := process.Current() - fds := p.Files().Pipe() + fds := s.process.Files().Pipe() return []interface{}{fds[0], fds[1]}, nil } diff --git a/internal/js/fs/read.go b/internal/js/fs/read.go index ce69947..1f3f38d 100644 --- a/internal/js/fs/read.go +++ b/internal/js/fs/read.go @@ -6,22 +6,21 @@ import ( "syscall/js" "github.com/hack-pad/hackpad/internal/fs" - "github.com/hack-pad/hackpad/internal/process" "github.com/hack-pad/hackpadfs/indexeddb/idbblob" "github.com/pkg/errors" ) -func read(args []js.Value) ([]interface{}, error) { - n, buf, err := readSyncImpl(args) +func (s fileShim) read(args []js.Value) ([]interface{}, error) { + n, buf, err := s.readSyncImpl(args) return []interface{}{n, buf}, err } -func readSync(args []js.Value) (interface{}, error) { - n, _, err := readSyncImpl(args) +func (s fileShim) readSync(args []js.Value) (interface{}, error) { + n, _, err := s.readSyncImpl(args) return n, err } -func readSyncImpl(args []js.Value) (int, js.Value, error) { +func (s fileShim) readSyncImpl(args []js.Value) (int, js.Value, error) { // args: fd, buffer, offset, length, position if len(args) != 5 { return 0, js.Null(), errors.Errorf("missing required args, expected 5: %+v", args) @@ -39,7 +38,6 @@ func readSyncImpl(args []js.Value) (int, js.Value, error) { *position = int64(args[4].Int()) } - p := process.Current() - n, err := p.Files().Read(fd, buffer, offset, length, position) + n, err := s.process.Files().Read(fd, buffer, offset, length, position) return n, buffer.JSValue(), err } diff --git a/internal/js/fs/readdir.go b/internal/js/fs/readdir.go index c06ca17..f24489a 100644 --- a/internal/js/fs/readdir.go +++ b/internal/js/fs/readdir.go @@ -5,22 +5,20 @@ package fs import ( "syscall/js" - "github.com/hack-pad/hackpad/internal/process" "github.com/pkg/errors" ) -func readdir(args []js.Value) ([]interface{}, error) { - fileNames, err := readdirSync(args) +func (s fileShim) readdir(args []js.Value) ([]interface{}, error) { + fileNames, err := s.readdirSync(args) return []interface{}{fileNames}, err } -func readdirSync(args []js.Value) (interface{}, error) { +func (s fileShim) readdirSync(args []js.Value) (interface{}, error) { if len(args) != 1 { return nil, errors.Errorf("Invalid number of args, expected 1: %v", args) } path := args[0].String() - p := process.Current() - dir, err := p.Files().ReadDir(path) + dir, err := s.process.Files().ReadDir(path) if err != nil { return nil, err } diff --git a/internal/js/fs/rename.go b/internal/js/fs/rename.go index 86731cf..c0481a3 100644 --- a/internal/js/fs/rename.go +++ b/internal/js/fs/rename.go @@ -5,23 +5,21 @@ package fs import ( "syscall/js" - "github.com/hack-pad/hackpad/internal/process" "github.com/pkg/errors" ) // rename(from, to, callback) { callback(enosys()); }, -func rename(args []js.Value) ([]interface{}, error) { - _, err := renameSync(args) +func (s fileShim) rename(args []js.Value) ([]interface{}, error) { + _, err := s.renameSync(args) return nil, err } -func renameSync(args []js.Value) (interface{}, error) { +func (s fileShim) renameSync(args []js.Value) (interface{}, error) { if len(args) != 2 { return nil, errors.Errorf("Invalid number of args, expected 2: %v", args) } oldPath := args[0].String() newPath := args[1].String() - p := process.Current() - return nil, p.Files().Rename(oldPath, newPath) + return nil, s.process.Files().Rename(oldPath, newPath) } diff --git a/internal/js/fs/rmdir.go b/internal/js/fs/rmdir.go index f15e5cf..80673eb 100644 --- a/internal/js/fs/rmdir.go +++ b/internal/js/fs/rmdir.go @@ -5,20 +5,18 @@ package fs import ( "syscall/js" - "github.com/hack-pad/hackpad/internal/process" "github.com/pkg/errors" ) -func rmdir(args []js.Value) ([]interface{}, error) { - _, err := rmdirSync(args) +func (s fileShim) rmdir(args []js.Value) ([]interface{}, error) { + _, err := s.rmdirSync(args) return nil, err } -func rmdirSync(args []js.Value) (interface{}, error) { +func (s fileShim) rmdirSync(args []js.Value) (interface{}, error) { if len(args) != 1 { return nil, errors.Errorf("Invalid number of args, expected 1: %v", args) } path := args[0].String() - p := process.Current() - return nil, p.Files().RemoveDir(path) + return nil, s.process.Files().RemoveDir(path) } diff --git a/internal/js/fs/stat.go b/internal/js/fs/stat.go index 1abdb85..96ae112 100644 --- a/internal/js/fs/stat.go +++ b/internal/js/fs/stat.go @@ -7,22 +7,20 @@ import ( "syscall" "syscall/js" - "github.com/hack-pad/hackpad/internal/process" "github.com/pkg/errors" ) -func stat(args []js.Value) ([]interface{}, error) { - info, err := statSync(args) +func (s fileShim) stat(args []js.Value) ([]interface{}, error) { + info, err := s.statSync(args) return []interface{}{info}, err } -func statSync(args []js.Value) (interface{}, error) { +func (s fileShim) statSync(args []js.Value) (interface{}, error) { if len(args) != 1 { return nil, errors.Errorf("Invalid number of args, expected 1: %v", args) } path := args[0].String() - p := process.Current() - info, err := p.Files().Stat(path) + info, err := s.process.Files().Stat(path) return jsStat(info), err } diff --git a/internal/js/fs/unlink.go b/internal/js/fs/unlink.go index 4f396cc..86fb346 100644 --- a/internal/js/fs/unlink.go +++ b/internal/js/fs/unlink.go @@ -5,22 +5,20 @@ package fs import ( "syscall/js" - "github.com/hack-pad/hackpad/internal/process" "github.com/pkg/errors" ) // unlink(path, callback) { callback(enosys()); }, -func unlink(args []js.Value) ([]interface{}, error) { - _, err := unlinkSync(args) +func (s fileShim) unlink(args []js.Value) ([]interface{}, error) { + _, err := s.unlinkSync(args) return nil, err } -func unlinkSync(args []js.Value) (interface{}, error) { +func (s fileShim) unlinkSync(args []js.Value) (interface{}, error) { if len(args) != 1 { return nil, errors.Errorf("Invalid number of args, expected 1: %v", args) } path := args[0].String() - p := process.Current() - return nil, p.Files().Unlink(path) + return nil, s.process.Files().Unlink(path) } diff --git a/internal/js/fs/utimes.go b/internal/js/fs/utimes.go index 70765e7..cda71bb 100644 --- a/internal/js/fs/utimes.go +++ b/internal/js/fs/utimes.go @@ -6,16 +6,15 @@ import ( "syscall/js" "time" - "github.com/hack-pad/hackpad/internal/process" "github.com/pkg/errors" ) -func utimes(args []js.Value) ([]interface{}, error) { - _, err := utimesSync(args) +func (s fileShim) utimes(args []js.Value) ([]interface{}, error) { + _, err := s.utimesSync(args) return nil, err } -func utimesSync(args []js.Value) (interface{}, error) { +func (s fileShim) utimesSync(args []js.Value) (interface{}, error) { if len(args) != 3 { return nil, errors.Errorf("Invalid number of args, expected 3: %v", args) } @@ -23,6 +22,5 @@ func utimesSync(args []js.Value) (interface{}, error) { path := args[0].String() atime := time.Unix(int64(args[1].Int()), 0) mtime := time.Unix(int64(args[2].Int()), 0) - p := process.Current() - return nil, p.Files().Utimes(path, atime, mtime) + return nil, s.process.Files().Utimes(path, atime, mtime) } diff --git a/internal/js/fs/write.go b/internal/js/fs/write.go index 053d981..3cde9f5 100644 --- a/internal/js/fs/write.go +++ b/internal/js/fs/write.go @@ -6,20 +6,19 @@ import ( "syscall/js" "github.com/hack-pad/hackpad/internal/fs" - "github.com/hack-pad/hackpad/internal/process" "github.com/hack-pad/hackpadfs/indexeddb/idbblob" "github.com/pkg/errors" ) -func writeSync(args []js.Value) (interface{}, error) { - ret, err := write(args) +func (s fileShim) writeSync(args []js.Value) (interface{}, error) { + ret, err := s.write(args) if len(ret) > 1 { return ret[0], err } return ret, err } -func write(args []js.Value) ([]interface{}, error) { +func (s fileShim) write(args []js.Value) ([]interface{}, error) { // args: fd, buffer, offset, length, position if len(args) < 2 { return nil, errors.Errorf("missing required args, expected fd and buffer: %+v", args) @@ -43,7 +42,6 @@ func write(args []js.Value) ([]interface{}, error) { *position = int64(args[4].Int()) } - p := process.Current() - n, err := p.Files().Write(fd, buffer, offset, length, position) + n, err := s.process.Files().Write(fd, buffer, offset, length, position) return []interface{}{n, buffer}, err } diff --git a/internal/js/process/dir.go b/internal/js/process/dir.go index e29cf96..e861bd0 100644 --- a/internal/js/process/dir.go +++ b/internal/js/process/dir.go @@ -5,18 +5,16 @@ package process import ( "syscall/js" - "github.com/hack-pad/hackpad/internal/process" "github.com/pkg/errors" ) -func cwd(args []js.Value) (interface{}, error) { - return process.Current().WorkingDirectory(), nil +func (s processShim) cwd(args []js.Value) (interface{}, error) { + return s.process.WorkingDirectory(), nil } -func chdir(args []js.Value) (interface{}, error) { +func (s processShim) chdir(args []js.Value) (interface{}, error) { if len(args) == 0 { return nil, errors.New("a new directory argument is required") } - p := process.Current() - return nil, p.SetWorkingDirectory(args[0].String()) + return nil, s.process.SetWorkingDirectory(args[0].String()) } diff --git a/internal/js/process/groups.go b/internal/js/process/groups.go index 097cc96..679d993 100644 --- a/internal/js/process/groups.go +++ b/internal/js/process/groups.go @@ -9,14 +9,14 @@ const ( groupID = 0 ) -func geteuid(args []js.Value) (interface{}, error) { +func (s processShim) geteuid(args []js.Value) (interface{}, error) { return userID, nil } -func getegid(args []js.Value) (interface{}, error) { +func (s processShim) getegid(args []js.Value) (interface{}, error) { return groupID, nil } -func getgroups(args []js.Value) (interface{}, error) { +func (s processShim) getgroups(args []js.Value) (interface{}, error) { return groupID, nil } diff --git a/internal/js/process/process.go b/internal/js/process/process.go index f687eae..8479861 100644 --- a/internal/js/process/process.go +++ b/internal/js/process/process.go @@ -11,40 +11,38 @@ import ( var jsProcess = js.Global().Get("process") -func Init() { - process.Init(switchedContext) +type processShim struct { + process *process.Process +} + +func Init(process *process.Process) { + shim := processShim{process} - currentProcess := process.Current() - err := currentProcess.Files().MkdirAll(currentProcess.WorkingDirectory(), 0750) + err := process.Files().MkdirAll(process.WorkingDirectory(), 0750) // TODO move to parent initialization if err != nil { panic(err) } globals := js.Global() - interop.SetFunc(jsProcess, "getuid", geteuid) - interop.SetFunc(jsProcess, "geteuid", geteuid) - interop.SetFunc(jsProcess, "getgid", getegid) - interop.SetFunc(jsProcess, "getegid", getegid) - interop.SetFunc(jsProcess, "getgroups", getgroups) - jsProcess.Set("pid", currentProcess.PID()) - jsProcess.Set("ppid", currentProcess.ParentPID()) - interop.SetFunc(jsProcess, "umask", umask) - interop.SetFunc(jsProcess, "cwd", cwd) - interop.SetFunc(jsProcess, "chdir", chdir) + interop.SetFunc(jsProcess, "getuid", shim.geteuid) + interop.SetFunc(jsProcess, "geteuid", shim.geteuid) + interop.SetFunc(jsProcess, "getgid", shim.getegid) + interop.SetFunc(jsProcess, "getegid", shim.getegid) + interop.SetFunc(jsProcess, "getgroups", shim.getgroups) + jsProcess.Set("pid", process.PID()) + jsProcess.Set("ppid", process.ParentPID()) + interop.SetFunc(jsProcess, "umask", shim.umask) + interop.SetFunc(jsProcess, "cwd", shim.cwd) + interop.SetFunc(jsProcess, "chdir", shim.chdir) globals.Set("child_process", map[string]interface{}{}) childProcess := globals.Get("child_process") - interop.SetFunc(childProcess, "spawn", spawn) - //interop.SetFunc(childProcess, "spawnSync", spawnSync) // TODO is there any way to run spawnSync so we don't hit deadlock? - interop.SetFunc(childProcess, "wait", wait) - interop.SetFunc(childProcess, "waitSync", waitSync) -} - -func switchedContext(pid, ppid process.PID) { - jsProcess.Set("pid", pid) - jsProcess.Set("ppid", ppid) + interop.SetFunc(childProcess, "spawn", shim.spawn) + //interop.SetFunc(childProcess, "spawnSync", shim.spawnSync) // TODO is there any way to run spawnSync so we don't hit deadlock? + interop.SetFunc(childProcess, "wait", shim.wait) + interop.SetFunc(childProcess, "waitSync", shim.waitSync) } -func Dump() interface{} { +func (s processShim) Dump() interface{} { return process.Dump() } diff --git a/internal/js/process/spawn.go b/internal/js/process/spawn.go index 9b7c1d3..d219dea 100644 --- a/internal/js/process/spawn.go +++ b/internal/js/process/spawn.go @@ -11,7 +11,7 @@ import ( "github.com/pkg/errors" ) -func spawn(args []js.Value) (interface{}, error) { +func (s processShim) spawn(args []js.Value) (interface{}, error) { if len(args) == 0 { return nil, errors.Errorf("Invalid number of args, expected command name: %v", args) } @@ -32,15 +32,16 @@ func spawn(args []js.Value) (interface{}, error) { if len(args) >= 3 { argv[0], procAttr = parseProcAttr(command, args[2]) } - return Spawn(command, argv, procAttr) + return s.Spawn(command, argv, procAttr) } -func Spawn(command string, args []string, attr *process.ProcAttr) (*process.Process, error) { - p, err := process.New(command, args, attr) - if err != nil { - return p, err - } - return p, p.Start() +func (s processShim) Spawn(command string, args []string, attr *process.ProcAttr) (*process.Process, error) { + //p, err := process.New(nil, command, args, s.process.WorkingDirectory(), attr, nil) // TODO spawn new worker + //if err != nil { + //return p, err + //} + //return p, p.Start() + panic("not implemented") } func parseProcAttr(defaultCommand string, value js.Value) (argv0 string, attr *process.ProcAttr) { diff --git a/internal/js/process/umask.go b/internal/js/process/umask.go index 55b37d4..b2686b9 100644 --- a/internal/js/process/umask.go +++ b/internal/js/process/umask.go @@ -6,7 +6,7 @@ import "syscall/js" var currentUMask = 0755 -func umask(args []js.Value) (interface{}, error) { +func (s processShim) umask(args []js.Value) (interface{}, error) { if len(args) == 0 { return currentUMask, nil } diff --git a/internal/js/process/wait.go b/internal/js/process/wait.go index 98375cf..6dac7af 100644 --- a/internal/js/process/wait.go +++ b/internal/js/process/wait.go @@ -10,30 +10,32 @@ import ( "github.com/pkg/errors" ) -func wait(args []js.Value) ([]interface{}, error) { - ret, err := waitSync(args) +func (s processShim) wait(args []js.Value) ([]interface{}, error) { + ret, err := s.waitSync(args) return []interface{}{ret}, err } -func waitSync(args []js.Value) (interface{}, error) { +func (s processShim) waitSync(args []js.Value) (interface{}, error) { if len(args) != 1 { return nil, errors.Errorf("Invalid number of args, expected 1: %v", args) } pid := process.PID(args[0].Int()) waitStatus := new(syscall.WaitStatus) - wpid, err := Wait(pid, waitStatus, 0, nil) + wpid, err := s.Wait(pid, waitStatus, 0, nil) return map[string]interface{}{ "pid": wpid, "exitCode": waitStatus.ExitStatus(), }, err } -func Wait(pid process.PID, wstatus *syscall.WaitStatus, options int, rusage *syscall.Rusage) (wpid process.PID, err error) { +func (s processShim) Wait(pid process.PID, wstatus *syscall.WaitStatus, options int, rusage *syscall.Rusage) (wpid process.PID, err error) { + panic("not implemented") // TODO support options and rusage - p, ok := process.Get(pid) - if !ok { - return 0, errors.Errorf("Unknown child process: %d", pid) - } + var p *process.Process + //p, ok := process.Get(pid) + //if !ok { + //return 0, errors.Errorf("Unknown child process: %d", pid) + //} exitCode, err := p.Wait() if wstatus != nil { diff --git a/internal/terminal/term.go b/internal/terminal/term.go index f717d32..f287e2d 100644 --- a/internal/terminal/term.go +++ b/internal/terminal/term.go @@ -5,8 +5,10 @@ package terminal import ( "syscall/js" + "github.com/hack-pad/hackpad/internal/common" "github.com/hack-pad/hackpad/internal/fs" "github.com/hack-pad/hackpad/internal/interop" + "github.com/hack-pad/hackpad/internal/kernel" "github.com/hack-pad/hackpad/internal/log" "github.com/hack-pad/hackpad/internal/process" "github.com/hack-pad/hackpadfs/indexeddb/idbblob" @@ -50,19 +52,22 @@ func Open(args []js.Value) error { workingDirectory = wd.String() } - files := process.Current().Files() - stdinR, stdinW := pipe(files) - stdoutR, stdoutW := pipe(files) - stderrR, stderrW := pipe(files) + var files *fs.FileDescriptors + panic("not implemented") + //files := process.Current().Files() + //stdinR, stdinW := pipe(files) + //stdoutR, stdoutW := pipe(files) + //stderrR, stderrW := pipe(files) + var stdoutR, stderrR, stdinW common.FID - proc, err := process.New(procArgs[0], procArgs, &process.ProcAttr{ + proc, err := process.New(kernel.ReservePID(), procArgs[0], procArgs, workingDirectory, nil, nil) /*&process.ProcAttr{ Dir: workingDirectory, Files: []fs.Attr{ {FID: stdinR}, {FID: stdoutW}, {FID: stderrW}, }, - }) + }*/ if err != nil { return err } diff --git a/internal/worker/dom.go b/internal/worker/dom.go new file mode 100644 index 0000000..7f09c03 --- /dev/null +++ b/internal/worker/dom.go @@ -0,0 +1,31 @@ +package worker + +import ( + "syscall/js" + + "github.com/hack-pad/hackpad/internal/interop" + "github.com/hack-pad/hackpad/internal/jsworker" +) + +type DOM struct { + local *Local + port *jsworker.Local +} + +func ExecDOM(localJS *jsworker.Local, command string, args []string, workingDirectory string, env map[string]string) (*DOM, error) { + localJS.PostMessage(js.ValueOf(map[string]interface{}{ + "init": map[string]interface{}{ + "command": command, + "args": interop.SliceFromStrings(args), + "workingDirectory": workingDirectory, + "env": interop.StringMap(env), + }, + }), nil) + local, err := NewLocal(localJS) + if err != nil { + return nil, err + } + return &DOM{ + local: local, + }, nil +} diff --git a/internal/worker/local.go b/internal/worker/local.go index 63077ed..86ed8f2 100644 --- a/internal/worker/local.go +++ b/internal/worker/local.go @@ -1,9 +1,15 @@ package worker import ( + "context" + "syscall/js" + + "github.com/hack-pad/hackpad/internal/global" + "github.com/hack-pad/hackpad/internal/interop" + jsfs "github.com/hack-pad/hackpad/internal/js/fs" + jsprocess "github.com/hack-pad/hackpad/internal/js/process" "github.com/hack-pad/hackpad/internal/jsworker" "github.com/hack-pad/hackpad/internal/kernel" - "github.com/hack-pad/hackpad/internal/log" "github.com/hack-pad/hackpad/internal/process" ) @@ -11,24 +17,52 @@ type Local struct { process *process.Process } -func New(localJS *jsworker.Local) (*Local, error) { +func NewLocal(localJS *jsworker.Local) (*Local, error) { ctx, cancel := context.WithCancel(context.Background()) - localChan := make(chan *Local, 1) + type initMessage struct { + err error + init js.Value + } + initChan := make(chan initMessage, 1) localJS.Listen(ctx, func(me jsworker.MessageEvent, err error) { if err != nil { - log.Error(err) + initChan <- initMessage{err: err} + return + } + if !me.Data.Truthy() || me.Data.Type() != js.TypeObject { return } initData := me.Data.Get("init") if !initData.Truthy() { return } - process := process.New() - localChan <- &Local{ - initData. - } + initChan <- initMessage{init: initData} }) - return nil, nil + message := <-initChan + cancel() + if message.err != nil { + return nil, message.err + } + + process, err := process.New( + kernel.ReservePID(), + message.init.Get("command").String(), + interop.StringsFromJSValue(message.init.Get("args")), + message.init.Get("workingDirectory").String(), + nil, // TODO open files + interop.StringMapFromJSObject(message.init.Get("env")), + ) + if err != nil { + return nil, err + } + local := &Local{ + process: process, + } + jsprocess.Init(local.process) + jsfs.Init(local.process) + global.Set("ready", true) + localJS.PostMessage(js.ValueOf("ready"), nil) + return local, nil } func (l *Local) Fork(command string, args []string, attr *process.ProcAttr) (*Remote, error) { diff --git a/main.go b/main.go index 6b9661e..a3cc66e 100644 --- a/main.go +++ b/main.go @@ -1,45 +1,42 @@ +//go:build js // +build js package main import ( - "path/filepath" "syscall/js" "github.com/hack-pad/hackpad/internal/global" "github.com/hack-pad/hackpad/internal/interop" - "github.com/hack-pad/hackpad/internal/js/fs" - "github.com/hack-pad/hackpad/internal/js/process" - "github.com/hack-pad/hackpad/internal/log" - libProcess "github.com/hack-pad/hackpad/internal/process" - "github.com/hack-pad/hackpad/internal/terminal" + "github.com/hack-pad/hackpad/internal/jsworker" + "github.com/hack-pad/hackpad/internal/worker" ) +type domShim struct { + dom *worker.DOM +} + func main() { - process.Init() - fs.Init() - global.Set("spawnTerminal", js.FuncOf(terminal.SpawnTerminal)) - global.Set("dump", js.FuncOf(func(this js.Value, args []js.Value) interface{} { - go func() { - basePath := "" - if len(args) >= 1 { - basePath = args[0].String() - if filepath.IsAbs(basePath) { - basePath = filepath.Clean(basePath) - } else { - basePath = filepath.Join(libProcess.Current().WorkingDirectory(), basePath) - } - } - var fsDump interface{} - if basePath != "" { - fsDump = fs.Dump(basePath) - } - log.Error("Process:\n", process.Dump(), "\n\nFiles:\n", fsDump) - }() - return nil - })) + dom, err := worker.ExecDOM( + jsworker.GetLocal(), + "editor", + []string{"-editor=editor"}, + "/home/me", + map[string]string{ + "GOMODCACHE": "/home/me/.cache/go-mod", + "GOPROXY": "https://proxy.golang.org/", + "GOROOT": "/usr/local/go", + "HOME": "/home/me", + "PATH": "/bin:/home/me/go/bin:/usr/local/go/bin/js_wasm:/usr/local/go/pkg/tool/js_wasm", + }, + ) + if err != nil { + panic(err) + } + + shim := domShim{dom} global.Set("profile", js.FuncOf(interop.ProfileJS)) - global.Set("install", js.FuncOf(installFunc)) - interop.SetInitialized() + global.Set("install", js.FuncOf(shim.installFunc)) + //global.Set("spawnTerminal", js.FuncOf(terminal.SpawnTerminal)) select {} } diff --git a/server/src/Hackpad.js b/server/src/Hackpad.js index 2b76d69..c29366f 100644 --- a/server/src/Hackpad.js +++ b/server/src/Hackpad.js @@ -19,6 +19,11 @@ async function init() { } go.run(cmd.instance) const { hackpad, fs } = window + const maxInitWaitMillis = 3000 + await messageOrTimeout(message => { + console.debug("message:", message) + return message === "ready" + }, maxInitWaitMillis) console.debug(`hackpad status: ${hackpad.ready ? 'ready' : 'not ready'}`) const mkdir = promisify(fs.mkdir) @@ -115,3 +120,29 @@ function promisify(fn) { }) } } + +async function messageOrTimeout(doneListener, timeout) { + let messageListener, errorListener + let timeoutID + try { + await new Promise((resolve, reject) => { + messageListener = ev => { + if (doneListener(ev.data) === true) { + resolve({data: ev.data}) + } + } + errorListener = ev => { + if (messageListener(ev.data) === true) { + reject({error: ev.data}) + } + } + window.addEventListener("message", messageListener) + window.addEventListener("messageerror", errorListener) + timeoutID = setTimeout(() => reject({error: "timed out"}), timeout) + }) + } finally { + window.removeEventListener("message", messageListener) + window.removeEventListener("messageerror", errorListener) + clearTimeout(timeoutID) + } +} From 649d4ced7d926e6ed5a0a83dcad711079f29dd8e Mon Sep 17 00:00:00 2001 From: John Starich Date: Sun, 3 Apr 2022 00:04:47 -0500 Subject: [PATCH 07/37] Add local and remote worker init/start system; successful installs --- cmd/worker/main.go | 18 +++++ install.go | 4 ++ internal/js/process/process.go | 23 ++++++- internal/js/process/spawn.go | 16 +++-- internal/js/process/wait.go | 9 +-- internal/jsworker/remote.go | 31 ++++----- internal/worker/dom.go | 20 +++--- internal/worker/local.go | 122 +++++++++++++++++++++++++-------- internal/worker/remote.go | 85 ++++++++++++++++++++++- main.go | 14 ++++ server/public/wasmWorker.js | 17 +++++ server/src/App.js | 15 +--- server/src/Hackpad.js | 37 ++++------ 13 files changed, 300 insertions(+), 111 deletions(-) create mode 100644 cmd/worker/main.go create mode 100644 server/public/wasmWorker.js diff --git a/cmd/worker/main.go b/cmd/worker/main.go new file mode 100644 index 0000000..27bee0d --- /dev/null +++ b/cmd/worker/main.go @@ -0,0 +1,18 @@ +package main + +import ( + "github.com/hack-pad/hackpad/internal/jsworker" + "github.com/hack-pad/hackpad/internal/log" + "github.com/hack-pad/hackpad/internal/worker" +) + +func main() { + log.Warn("booting worker") + jsLocal := jsworker.GetLocal() + local, err := worker.NewLocal(jsLocal) + if err != nil { + panic(err) + } + log.Warn("worker started:", local) + select {} +} diff --git a/install.go b/install.go index 58935b2..5f14bcb 100644 --- a/install.go +++ b/install.go @@ -32,6 +32,10 @@ func (s domShim) install(args []js.Value) error { return errors.New("Expected command name to install") } command := args[0].String() + return s.Install(command) +} + +func (s domShim) Install(command string) error { command = filepath.Base(command) // ensure no path chars are present if err := os.MkdirAll("/bin", 0644); err != nil { diff --git a/internal/js/process/process.go b/internal/js/process/process.go index 8479861..7ca87a6 100644 --- a/internal/js/process/process.go +++ b/internal/js/process/process.go @@ -5,6 +5,7 @@ package process import ( "syscall/js" + "github.com/hack-pad/hackpad/internal/common" "github.com/hack-pad/hackpad/internal/interop" "github.com/hack-pad/hackpad/internal/process" ) @@ -13,10 +14,28 @@ var jsProcess = js.Global().Get("process") type processShim struct { process *process.Process + spawner Spawner + waiter Waiter } -func Init(process *process.Process) { - shim := processShim{process} +type PIDer interface { + PID() common.PID +} + +type Spawner interface { + Spawn(command string, args []string, attr *process.ProcAttr) (PIDer, error) +} + +type Waiter interface { + Wait(pid common.PID) (exitCode int, err error) +} + +func Init(process *process.Process, spawner Spawner, waiter Waiter) { + shim := processShim{ + process: process, + spawner: spawner, + waiter: waiter, + } err := process.Files().MkdirAll(process.WorkingDirectory(), 0750) // TODO move to parent initialization if err != nil { diff --git a/internal/js/process/spawn.go b/internal/js/process/spawn.go index d219dea..2a2ef4c 100644 --- a/internal/js/process/spawn.go +++ b/internal/js/process/spawn.go @@ -35,13 +35,15 @@ func (s processShim) spawn(args []js.Value) (interface{}, error) { return s.Spawn(command, argv, procAttr) } -func (s processShim) Spawn(command string, args []string, attr *process.ProcAttr) (*process.Process, error) { - //p, err := process.New(nil, command, args, s.process.WorkingDirectory(), attr, nil) // TODO spawn new worker - //if err != nil { - //return p, err - //} - //return p, p.Start() - panic("not implemented") +func (s processShim) Spawn(command string, args []string, attr *process.ProcAttr) (map[string]interface{}, error) { + pider, err := s.spawner.Spawn(command, args, attr) + if err != nil { + return nil, err + } + return map[string]interface{}{ + "pid": pider.PID(), + "ppid": s.process.PID(), + }, nil } func parseProcAttr(defaultCommand string, value js.Value) (argv0 string, attr *process.ProcAttr) { diff --git a/internal/js/process/wait.go b/internal/js/process/wait.go index 6dac7af..1b2133a 100644 --- a/internal/js/process/wait.go +++ b/internal/js/process/wait.go @@ -29,15 +29,8 @@ func (s processShim) waitSync(args []js.Value) (interface{}, error) { } func (s processShim) Wait(pid process.PID, wstatus *syscall.WaitStatus, options int, rusage *syscall.Rusage) (wpid process.PID, err error) { - panic("not implemented") // TODO support options and rusage - var p *process.Process - //p, ok := process.Get(pid) - //if !ok { - //return 0, errors.Errorf("Unknown child process: %d", pid) - //} - - exitCode, err := p.Wait() + exitCode, err := s.waiter.Wait(pid) if wstatus != nil { const ( // defined in syscall.WaitStatus diff --git a/internal/jsworker/remote.go b/internal/jsworker/remote.go index ea36f70..4ea19f5 100644 --- a/internal/jsworker/remote.go +++ b/internal/jsworker/remote.go @@ -32,27 +32,20 @@ func NewRemote(name, url string) (_ *Remote, err error) { }, err } -func NewRemoteWasm(ctx context.Context, name, wasmURL string) (*Remote, error) { - ctx, cancel := context.WithCancel(ctx) - defer cancel() - l, err := NewRemote(name, wasmWorkerScript+"?wasm="+wasmURL) - if err != nil { - return nil, err - } - err = l.port.Listen(ctx, func(ev MessageEvent, err error) { - if jsString(ev.Data) == "ready" { - cancel() - } - }) - if err != nil { - return nil, err - } - <-ctx.Done() - return l, err +func NewRemoteWasm(name, wasmURL string) (*Remote, error) { + return NewRemote(name, wasmWorkerScript+"?wasm="+wasmURL) } -func (l *Remote) Terminate() (err error) { +func (r *Remote) Terminate() (err error) { defer common.CatchException(&err) - l.worker.Call("terminate") + r.worker.Call("terminate") return nil } + +func (r *Remote) PostMessage(message js.Value, transfers []js.Value) error { + return r.port.PostMessage(message, transfers) +} + +func (r *Remote) Listen(ctx context.Context, listener func(MessageEvent, error)) error { + return r.port.Listen(ctx, listener) +} diff --git a/internal/worker/dom.go b/internal/worker/dom.go index 7f09c03..1c72dea 100644 --- a/internal/worker/dom.go +++ b/internal/worker/dom.go @@ -3,7 +3,6 @@ package worker import ( "syscall/js" - "github.com/hack-pad/hackpad/internal/interop" "github.com/hack-pad/hackpad/internal/jsworker" ) @@ -13,14 +12,7 @@ type DOM struct { } func ExecDOM(localJS *jsworker.Local, command string, args []string, workingDirectory string, env map[string]string) (*DOM, error) { - localJS.PostMessage(js.ValueOf(map[string]interface{}{ - "init": map[string]interface{}{ - "command": command, - "args": interop.SliceFromStrings(args), - "workingDirectory": workingDirectory, - "env": interop.StringMap(env), - }, - }), nil) + localJS.PostMessage(makeInitMessage(command, args, workingDirectory, env), nil) local, err := NewLocal(localJS) if err != nil { return nil, err @@ -29,3 +21,13 @@ func ExecDOM(localJS *jsworker.Local, command string, args []string, workingDire local: local, }, nil } + +func (d *DOM) Start() error { + return d.port.PostMessage(makeStartMessage(), nil) +} + +func makeStartMessage() js.Value { + return js.ValueOf(map[string]interface{}{ + "start": true, + }) +} diff --git a/internal/worker/local.go b/internal/worker/local.go index 86ed8f2..70dfd81 100644 --- a/internal/worker/local.go +++ b/internal/worker/local.go @@ -4,27 +4,69 @@ import ( "context" "syscall/js" + "github.com/hack-pad/hackpad/internal/common" "github.com/hack-pad/hackpad/internal/global" "github.com/hack-pad/hackpad/internal/interop" jsfs "github.com/hack-pad/hackpad/internal/js/fs" jsprocess "github.com/hack-pad/hackpad/internal/js/process" "github.com/hack-pad/hackpad/internal/jsworker" "github.com/hack-pad/hackpad/internal/kernel" + "github.com/hack-pad/hackpad/internal/log" "github.com/hack-pad/hackpad/internal/process" + "github.com/pkg/errors" ) type Local struct { + localJS *jsworker.Local process *process.Process + pids map[common.PID]*Remote } -func NewLocal(localJS *jsworker.Local) (*Local, error) { - ctx, cancel := context.WithCancel(context.Background()) +func NewLocal(localJS *jsworker.Local) (_ *Local, err error) { + local := &Local{ + localJS: localJS, + pids: make(map[common.PID]*Remote), + } + + init, err := local.awaitInit(context.Background()) + if err != nil { + return nil, err + } + + defer common.CatchException(&err) + local.process, err = process.New( + kernel.ReservePID(), + init.Get("command").String(), + interop.StringsFromJSValue(init.Get("args")), + init.Get("workingDirectory").String(), + nil, // TODO open files + interop.StringMapFromJSObject(init.Get("env")), + ) + if err != nil { + return nil, err + } + jsprocess.Init(local.process, local, local) + jsfs.Init(local.process) + global.Set("ready", true) + log.Debug("before ready post") + localJS.PostMessage(js.ValueOf("ready"), nil) + log.Debug("after ready post") + + local.listenStart() + + return local, nil +} + +func (l *Local) awaitInit(ctx context.Context) (js.Value, error) { + ctx, cancel := context.WithCancel(ctx) + defer cancel() + type initMessage struct { err error init js.Value } initChan := make(chan initMessage, 1) - localJS.Listen(ctx, func(me jsworker.MessageEvent, err error) { + err := l.localJS.Listen(ctx, func(me jsworker.MessageEvent, err error) { if err != nil { initChan <- initMessage{err: err} return @@ -38,35 +80,55 @@ func NewLocal(localJS *jsworker.Local) (*Local, error) { } initChan <- initMessage{init: initData} }) - message := <-initChan - cancel() - if message.err != nil { - return nil, message.err - } - - process, err := process.New( - kernel.ReservePID(), - message.init.Get("command").String(), - interop.StringsFromJSValue(message.init.Get("args")), - message.init.Get("workingDirectory").String(), - nil, // TODO open files - interop.StringMapFromJSObject(message.init.Get("env")), - ) if err != nil { - return nil, err - } - local := &Local{ - process: process, + return js.Value{}, err } - jsprocess.Init(local.process) - jsfs.Init(local.process) - global.Set("ready", true) - localJS.PostMessage(js.ValueOf("ready"), nil) - return local, nil + message := <-initChan + return message.init, message.err +} + +func (l *Local) listenStart() error { + startCtx, cancel := context.WithCancel(context.Background()) + return l.localJS.Listen(startCtx, func(me jsworker.MessageEvent, err error) { + if err != nil { + log.Error(err) + cancel() + return + } + defer common.CatchExceptionHandler(func(err error) { + log.Error(err) + cancel() + }) + if me.Data.Type() != js.TypeObject { + return + } + entries := interop.Entries(me.Data) + _, ok := entries["start"] + if !ok { + return + } + cancel() + + log.Print("Starting process: ", l.process.PID) + err = l.process.Start() + if err != nil { + log.Error(err) + return + } + }) } -func (l *Local) Fork(command string, args []string, attr *process.ProcAttr) (*Remote, error) { - remote, err := NewRemote(l, kernel.ReservePID(), command, args, attr) - // TODO worker init - return remote, err +func (l *Local) Spawn(command string, args []string, attr *process.ProcAttr) (jsprocess.PIDer, error) { + pid := kernel.ReservePID() + log.Print("Spawning pid: ", pid, " for command: ", command, args) + return NewRemote(l, pid, command, args, attr) +} + +func (l *Local) Wait(pid common.PID) (exitCode int, err error) { + log.Print("Waiting on pid: ", pid) + remote, ok := l.pids[pid] + if !ok { + return 0, errors.Errorf("Unknown child process: %d", pid) + } + return remote.Wait() } diff --git a/internal/worker/remote.go b/internal/worker/remote.go index 261beb3..ba77762 100644 --- a/internal/worker/remote.go +++ b/internal/worker/remote.go @@ -1,10 +1,22 @@ package worker import ( + "context" + "fmt" + "syscall/js" + + "github.com/hack-pad/hackpad/internal/common" + "github.com/hack-pad/hackpad/internal/interop" + "github.com/hack-pad/hackpad/internal/jsworker" "github.com/hack-pad/hackpad/internal/process" ) type Remote struct { + pid common.PID + port *jsworker.Remote + closeCtx context.Context + closeExitCode *int + closeErr error } type openFile struct { @@ -24,6 +36,75 @@ func NewRemote(local *Local, pid process.PID, command string, args []string, att seekOffset: 0, // TODO expose seek offset in file descriptor }) } - // TODO - return &Remote{}, nil + // TODO inherit file descriptors + port, err := jsworker.NewRemoteWasm(fmt.Sprintf("pid-%d", pid), "/wasm/worker.wasm") + if err != nil { + return nil, err + } + err = port.PostMessage(makeInitMessage(command, args, attr.Dir, attr.Env), nil) + if err != nil { + return nil, err + } + + closeCtx, cancel := context.WithCancel(context.Background()) + remote := &Remote{ + pid: pid, + port: port, + closeCtx: closeCtx, + } + + err = port.Listen(closeCtx, func(me jsworker.MessageEvent, err error) { + if err != nil { + remote.closeErr = err + cancel() + return + } + if me.Data.Type() != js.TypeObject { + return + } + data := interop.Entries(me.Data) + if jsExitCode, ok := data["exitCode"]; ok && jsExitCode.Type() == js.TypeNumber { + exitCode := jsExitCode.Int() + remote.closeExitCode = &exitCode + } + cancel() + }) + if err != nil { + return nil, err + } + + err = remote.port.PostMessage(makeStartMessage(), nil) + if err != nil { + return nil, err + } + + return remote, nil +} + +func (r *Remote) PID() common.PID { + return r.pid +} + +func makeInitMessage(command string, args []string, workingDirectory string, env map[string]string) js.Value { + return js.ValueOf(map[string]interface{}{ + "init": map[string]interface{}{ + "command": command, + "args": interop.SliceFromStrings(args), + "workingDirectory": workingDirectory, + "env": interop.StringMap(env), + }, + }) +} + +func (r *Remote) Wait() (exitCode int, err error) { + <-r.closeCtx.Done() + if r.closeExitCode == nil { + switch { + case r.closeErr != nil: + return 0, r.closeErr + default: + return 0, r.closeCtx.Err() + } + } + return *r.closeExitCode, r.closeErr } diff --git a/main.go b/main.go index a3cc66e..a906b9c 100644 --- a/main.go +++ b/main.go @@ -9,6 +9,7 @@ import ( "github.com/hack-pad/hackpad/internal/global" "github.com/hack-pad/hackpad/internal/interop" "github.com/hack-pad/hackpad/internal/jsworker" + "github.com/hack-pad/hackpad/internal/log" "github.com/hack-pad/hackpad/internal/worker" ) @@ -38,5 +39,18 @@ func main() { global.Set("profile", js.FuncOf(interop.ProfileJS)) global.Set("install", js.FuncOf(shim.installFunc)) //global.Set("spawnTerminal", js.FuncOf(terminal.SpawnTerminal)) + + if err := shim.Install("editor"); err != nil { + panic(err) + } + if err := shim.Install("sh"); err != nil { + panic(err) + } + + log.SetLevel(log.LevelDebug) + //if err := dom.Start(); err != nil { + //panic(err) + //} + select {} } diff --git a/server/public/wasmWorker.js b/server/public/wasmWorker.js new file mode 100644 index 0000000..32feb36 --- /dev/null +++ b/server/public/wasmWorker.js @@ -0,0 +1,17 @@ +"use strict"; + +async function runWasm(params) { + console.log("Loading Wasm:", params) + self.importScripts("wasm/wasm_exec.js") + const go = new Go() + const result = await WebAssembly.instantiateStreaming(fetch(params.wasm), go.importObject) + await go.run(result.instance) + close() +} + +const params = new URLSearchParams(self.location.search) +const paramsObj = {} +for (const [key, value] of params) { + paramsObj[key] = value +} +runWasm(paramsObj) diff --git a/server/src/App.js b/server/src/App.js index f6d24b0..d16856c 100644 --- a/server/src/App.js +++ b/server/src/App.js @@ -6,13 +6,12 @@ import "@fontsource/roboto"; import '@fortawesome/fontawesome-free/css/all.css'; import Compat from './Compat'; import Loading from './Loading'; -import { install, run, observeGoDownloadProgress } from './Hackpad'; +import { observeGoDownloadProgress } from './Hackpad'; import { newEditor } from './Editor'; import { newTerminal } from './Terminal'; function App() { const [percentage, setPercentage] = React.useState(0); - const [loading, setLoading] = React.useState(true); React.useEffect(() => { observeGoDownloadProgress(setPercentage) @@ -20,19 +19,11 @@ function App() { newTerminal, newEditor, } - Promise.all([ install('editor'), install('sh') ]) - .then(() => { - run('editor', '--editor=editor') - setLoading(false) - }) - }, [setLoading, setPercentage]) + }, [setPercentage]) return ( <> - { loading ? <> - - - : null } +
diff --git a/server/src/Hackpad.js b/server/src/Hackpad.js index c29366f..8477aff 100644 --- a/server/src/Hackpad.js +++ b/server/src/Hackpad.js @@ -10,13 +10,6 @@ async function init() { const startTime = new Date().getTime() const go = new Go(); const cmd = await WebAssembly.instantiateStreaming(fetch(`wasm/main.wasm`), go.importObject) - go.env = { - 'GOMODCACHE': '/home/me/.cache/go-mod', - 'GOPROXY': 'https://proxy.golang.org/', - 'GOROOT': '/usr/local/go', - 'HOME': '/home/me', - 'PATH': '/bin:/home/me/go/bin:/usr/local/go/bin/js_wasm:/usr/local/go/pkg/tool/js_wasm', - } go.run(cmd.instance) const { hackpad, fs } = window const maxInitWaitMillis = 3000 @@ -27,24 +20,24 @@ async function init() { console.debug(`hackpad status: ${hackpad.ready ? 'ready' : 'not ready'}`) const mkdir = promisify(fs.mkdir) - await mkdir("/bin", {mode: 0o700}) - await hackpad.overlayIndexedDB('/bin', {cache: true}) - await hackpad.overlayIndexedDB('/home/me') + await mkdir("/bin", {mode: 0o700, recursive: true}) + //await hackpad.overlayIndexedDB('/bin', {cache: true}) + //await hackpad.overlayIndexedDB('/home/me') await mkdir("/home/me/.cache", {recursive: true, mode: 0o700}) - await hackpad.overlayIndexedDB('/home/me/.cache', {cache: true}) + //await hackpad.overlayIndexedDB('/home/me/.cache', {cache: true}) await mkdir("/usr/local/go", {recursive: true, mode: 0o700}) - await hackpad.overlayTarGzip('/usr/local/go', 'wasm/go.tar.gz', { - persist: true, - skipCacheDirs: [ - '/usr/local/go/bin/js_wasm', - '/usr/local/go/pkg/tool/js_wasm', - ], - progress: percentage => { - overlayProgress = percentage - progressListeners.forEach(c => c(percentage)) - }, - }) + //await hackpad.overlayTarGzip('/usr/local/go', 'wasm/go.tar.gz', { + // persist: true, + // skipCacheDirs: [ + // '/usr/local/go/bin/js_wasm', + // '/usr/local/go/pkg/tool/js_wasm', + // ], + // progress: percentage => { + // overlayProgress = percentage + // progressListeners.forEach(c => c(percentage)) + // }, + //}) console.debug("Startup took", (new Date().getTime() - startTime) / 1000, "seconds") } From 893b7ab0d0e7903a1787390f19d60d266c1fa3ed Mon Sep 17 00:00:00 2001 From: John Starich Date: Sun, 3 Apr 2022 16:33:54 -0500 Subject: [PATCH 08/37] Fix DOM worker crashes, add panic handlers for better visibility --- cmd/worker/main.go | 7 +++++++ http_get.go | 1 + internal/js/process/process.go | 2 +- internal/js/process/spawn.go | 4 ++-- internal/process/process.go | 3 +-- internal/worker/dom.go | 3 ++- internal/worker/local.go | 8 ++++---- internal/worker/remote.go | 8 ++++---- main.go | 15 +++++++++++---- 9 files changed, 33 insertions(+), 18 deletions(-) diff --git a/cmd/worker/main.go b/cmd/worker/main.go index 27bee0d..e381aeb 100644 --- a/cmd/worker/main.go +++ b/cmd/worker/main.go @@ -1,12 +1,19 @@ package main import ( + "os" + + "github.com/hack-pad/hackpad/internal/common" "github.com/hack-pad/hackpad/internal/jsworker" "github.com/hack-pad/hackpad/internal/log" "github.com/hack-pad/hackpad/internal/worker" ) func main() { + defer common.CatchExceptionHandler(func(err error) { + log.Error("Worker panicked:", err) + os.Exit(1) + }) log.Warn("booting worker") jsLocal := jsworker.GetLocal() local, err := worker.NewLocal(jsLocal) diff --git a/http_get.go b/http_get.go index d00aaa0..1004e24 100644 --- a/http_get.go +++ b/http_get.go @@ -1,3 +1,4 @@ +//go:build js // +build js package main diff --git a/internal/js/process/process.go b/internal/js/process/process.go index 7ca87a6..f4d1800 100644 --- a/internal/js/process/process.go +++ b/internal/js/process/process.go @@ -23,7 +23,7 @@ type PIDer interface { } type Spawner interface { - Spawn(command string, args []string, attr *process.ProcAttr) (PIDer, error) + Spawn(command string, argv []string, attr *process.ProcAttr) (PIDer, error) } type Waiter interface { diff --git a/internal/js/process/spawn.go b/internal/js/process/spawn.go index 2a2ef4c..198f244 100644 --- a/internal/js/process/spawn.go +++ b/internal/js/process/spawn.go @@ -35,8 +35,8 @@ func (s processShim) spawn(args []js.Value) (interface{}, error) { return s.Spawn(command, argv, procAttr) } -func (s processShim) Spawn(command string, args []string, attr *process.ProcAttr) (map[string]interface{}, error) { - pider, err := s.spawner.Spawn(command, args, attr) +func (s processShim) Spawn(command string, argv []string, attr *process.ProcAttr) (map[string]interface{}, error) { + pider, err := s.spawner.Spawn(command, argv, attr) if err != nil { return nil, err } diff --git a/internal/process/process.go b/internal/process/process.go index 3b53d08..9b966e8 100644 --- a/internal/process/process.go +++ b/internal/process/process.go @@ -3,7 +3,6 @@ package process import ( "context" "fmt" - "os" "sort" "strings" @@ -97,7 +96,7 @@ func (p *Process) start() error { func (p *Process) prepExecutable() (command string, err error) { fs := p.Files() - command, err = lookPath(fs.Stat, os.Getenv("PATH"), p.command) + command, err = lookPath(fs.Stat, p.env["PATH"], p.command) if err != nil { return "", err } diff --git a/internal/worker/dom.go b/internal/worker/dom.go index 1c72dea..3a9cbf1 100644 --- a/internal/worker/dom.go +++ b/internal/worker/dom.go @@ -12,13 +12,14 @@ type DOM struct { } func ExecDOM(localJS *jsworker.Local, command string, args []string, workingDirectory string, env map[string]string) (*DOM, error) { - localJS.PostMessage(makeInitMessage(command, args, workingDirectory, env), nil) + localJS.PostMessage(makeInitMessage(command, append([]string{command}, args...), workingDirectory, env), nil) local, err := NewLocal(localJS) if err != nil { return nil, err } return &DOM{ local: local, + port: localJS, }, nil } diff --git a/internal/worker/local.go b/internal/worker/local.go index 70dfd81..4f3acff 100644 --- a/internal/worker/local.go +++ b/internal/worker/local.go @@ -37,7 +37,7 @@ func NewLocal(localJS *jsworker.Local) (_ *Local, err error) { local.process, err = process.New( kernel.ReservePID(), init.Get("command").String(), - interop.StringsFromJSValue(init.Get("args")), + interop.StringsFromJSValue(init.Get("argv")), init.Get("workingDirectory").String(), nil, // TODO open files interop.StringMapFromJSObject(init.Get("env")), @@ -118,10 +118,10 @@ func (l *Local) listenStart() error { }) } -func (l *Local) Spawn(command string, args []string, attr *process.ProcAttr) (jsprocess.PIDer, error) { +func (l *Local) Spawn(command string, argv []string, attr *process.ProcAttr) (jsprocess.PIDer, error) { pid := kernel.ReservePID() - log.Print("Spawning pid: ", pid, " for command: ", command, args) - return NewRemote(l, pid, command, args, attr) + log.Print("Spawning pid: ", pid, " for command: ", command, argv) + return NewRemote(l, pid, command, argv, attr) } func (l *Local) Wait(pid common.PID) (exitCode int, err error) { diff --git a/internal/worker/remote.go b/internal/worker/remote.go index ba77762..2f2f143 100644 --- a/internal/worker/remote.go +++ b/internal/worker/remote.go @@ -24,7 +24,7 @@ type openFile struct { seekOffset uint } -func NewRemote(local *Local, pid process.PID, command string, args []string, attr *process.ProcAttr) (*Remote, error) { +func NewRemote(local *Local, pid process.PID, command string, argv []string, attr *process.ProcAttr) (*Remote, error) { var openFiles []openFile for _, f := range attr.Files { info, err := local.process.Files().Fstat(f.FID) @@ -41,7 +41,7 @@ func NewRemote(local *Local, pid process.PID, command string, args []string, att if err != nil { return nil, err } - err = port.PostMessage(makeInitMessage(command, args, attr.Dir, attr.Env), nil) + err = port.PostMessage(makeInitMessage(command, argv, attr.Dir, attr.Env), nil) if err != nil { return nil, err } @@ -85,11 +85,11 @@ func (r *Remote) PID() common.PID { return r.pid } -func makeInitMessage(command string, args []string, workingDirectory string, env map[string]string) js.Value { +func makeInitMessage(command string, argv []string, workingDirectory string, env map[string]string) js.Value { return js.ValueOf(map[string]interface{}{ "init": map[string]interface{}{ "command": command, - "args": interop.SliceFromStrings(args), + "argv": interop.SliceFromStrings(argv), "workingDirectory": workingDirectory, "env": interop.StringMap(env), }, diff --git a/main.go b/main.go index a906b9c..0ad32df 100644 --- a/main.go +++ b/main.go @@ -4,8 +4,11 @@ package main import ( + "os" + "runtime/debug" "syscall/js" + "github.com/hack-pad/hackpad/internal/common" "github.com/hack-pad/hackpad/internal/global" "github.com/hack-pad/hackpad/internal/interop" "github.com/hack-pad/hackpad/internal/jsworker" @@ -18,6 +21,11 @@ type domShim struct { } func main() { + defer common.CatchExceptionHandler(func(err error) { + log.Error("Hackpad panic:", err, "\n", string(debug.Stack())) + os.Exit(1) + }) + dom, err := worker.ExecDOM( jsworker.GetLocal(), "editor", @@ -47,10 +55,9 @@ func main() { panic(err) } - log.SetLevel(log.LevelDebug) - //if err := dom.Start(); err != nil { - //panic(err) - //} + if err := dom.Start(); err != nil { + panic(err) + } select {} } From 228f942b140c92a4ee2e43d9fbeec11ccfd7ed98 Mon Sep 17 00:00:00 2001 From: John Starich Date: Sun, 3 Apr 2022 16:42:04 -0500 Subject: [PATCH 09/37] Track child pids --- internal/worker/local.go | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/internal/worker/local.go b/internal/worker/local.go index 4f3acff..c10e09e 100644 --- a/internal/worker/local.go +++ b/internal/worker/local.go @@ -109,7 +109,7 @@ func (l *Local) listenStart() error { } cancel() - log.Print("Starting process: ", l.process.PID) + log.Print("Starting process: ", l.process.PID()) err = l.process.Start() if err != nil { log.Error(err) @@ -121,7 +121,12 @@ func (l *Local) listenStart() error { func (l *Local) Spawn(command string, argv []string, attr *process.ProcAttr) (jsprocess.PIDer, error) { pid := kernel.ReservePID() log.Print("Spawning pid: ", pid, " for command: ", command, argv) - return NewRemote(l, pid, command, argv, attr) + remote, err := NewRemote(l, pid, command, argv, attr) + if err != nil { + return nil, err + } + l.pids[pid] = remote + return remote, nil } func (l *Local) Wait(pid common.PID) (exitCode int, err error) { From 31a9e5db22b7ec124fc0ad7f1ea98b64a5c3e974 Mon Sep 17 00:00:00 2001 From: John Starich Date: Sun, 3 Apr 2022 22:33:36 -0500 Subject: [PATCH 10/37] Add worker lifecycle management, move FS setup to main.go --- cmd/editor/main.go | 7 +++ cmd/worker/main.go | 25 +++++++-- internal/jsworker/local.go | 4 ++ internal/jsworker/message_port.go | 8 +++ internal/worker/dom.go | 10 ++-- internal/worker/local.go | 48 +++++++++++++---- internal/worker/remote.go | 38 ++++++++++++- main.go | 88 +++++++++++++++++++++++++++++-- server/src/Hackpad.js | 22 +------- 9 files changed, 206 insertions(+), 44 deletions(-) diff --git a/cmd/editor/main.go b/cmd/editor/main.go index 7c0c828..a4f1651 100644 --- a/cmd/editor/main.go +++ b/cmd/editor/main.go @@ -6,6 +6,7 @@ import ( "flag" "io/ioutil" "os" + "runtime/debug" "syscall/js" "github.com/hack-pad/hackpad/cmd/editor/dom" @@ -13,6 +14,7 @@ import ( "github.com/hack-pad/hackpad/cmd/editor/plaineditor" "github.com/hack-pad/hackpad/cmd/editor/taskconsole" "github.com/hack-pad/hackpad/cmd/editor/terminal" + "github.com/hack-pad/hackpad/internal/common" "github.com/hack-pad/hackpad/internal/interop" "github.com/hack-pad/hackpad/internal/log" ) @@ -22,6 +24,11 @@ const ( ) func main() { + defer common.CatchExceptionHandler(func(err error) { + log.Error("Editor panic:", err, "\n", string(debug.Stack())) + os.Exit(1) + }) + editorID := flag.String("editor", "", "Editor element ID to attach") flag.Parse() diff --git a/cmd/worker/main.go b/cmd/worker/main.go index e381aeb..747ded1 100644 --- a/cmd/worker/main.go +++ b/cmd/worker/main.go @@ -1,6 +1,7 @@ package main import ( + "context" "os" "github.com/hack-pad/hackpad/internal/common" @@ -14,12 +15,26 @@ func main() { log.Error("Worker panicked:", err) os.Exit(1) }) - log.Warn("booting worker") - jsLocal := jsworker.GetLocal() - local, err := worker.NewLocal(jsLocal) + + log.SetLevel(log.LevelDebug) + bootCtx := context.Background() + //bootCtx, bootCancel := context.WithTimeout(context.Background(), 30*time.Second) + //defer bootCancel() + log.Print("booting worker") + local, err := worker.NewLocal(bootCtx, jsworker.GetLocal()) if err != nil { panic(err) } - log.Warn("worker started:", local) - select {} + log.Print("worker started") + <-local.Started() + pid := local.PID() + log.Print("worker process started PID ", pid) + exitCode, err := local.Wait(pid) + if err != nil { + log.Error("Failed to wait for PID ", pid, ":", err) + exitCode = 1 + } + log.Warn("worker stopped for PID ", pid, "; exit code = ", exitCode) + local.Exit(exitCode) + os.Exit(exitCode) } diff --git a/internal/jsworker/local.go b/internal/jsworker/local.go index df87f66..ad2fc0a 100644 --- a/internal/jsworker/local.go +++ b/internal/jsworker/local.go @@ -36,3 +36,7 @@ func (l *Local) PostMessage(message js.Value, transfers []js.Value) error { func (l *Local) Listen(ctx context.Context, listener func(MessageEvent, error)) error { return l.port.Listen(ctx, listener) } + +func (l *Local) Close() error { + return l.port.Close() +} diff --git a/internal/jsworker/message_port.go b/internal/jsworker/message_port.go index 1c176ae..7f0a6ff 100644 --- a/internal/jsworker/message_port.go +++ b/internal/jsworker/message_port.go @@ -7,6 +7,7 @@ import ( "github.com/hack-pad/hackpad/internal/common" "github.com/hack-pad/hackpad/internal/interop" + "github.com/hack-pad/hackpad/internal/log" ) type MessagePort struct { @@ -36,6 +37,7 @@ func wrapMessagePort(v js.Value) (*MessagePort, error) { func (p *MessagePort) PostMessage(message js.Value, transfers []js.Value) (err error) { defer common.CatchException(&err) args := append([]interface{}{message}, interop.SliceFromJSValues(transfers)...) + log.PrintJSValues("Post message:", message) p.jsMessagePort.Call("postMessage", args...) return nil } @@ -75,3 +77,9 @@ func (p *MessagePort) Listen(ctx context.Context, listener func(MessageEvent, er } return nil } + +func (p *MessagePort) Close() (err error) { + defer common.CatchException(&err) + p.jsMessagePort.Call("close") + return nil +} diff --git a/internal/worker/dom.go b/internal/worker/dom.go index 3a9cbf1..78a3137 100644 --- a/internal/worker/dom.go +++ b/internal/worker/dom.go @@ -1,6 +1,7 @@ package worker import ( + "context" "syscall/js" "github.com/hack-pad/hackpad/internal/jsworker" @@ -11,9 +12,12 @@ type DOM struct { port *jsworker.Local } -func ExecDOM(localJS *jsworker.Local, command string, args []string, workingDirectory string, env map[string]string) (*DOM, error) { - localJS.PostMessage(makeInitMessage(command, append([]string{command}, args...), workingDirectory, env), nil) - local, err := NewLocal(localJS) +func ExecDOM(ctx context.Context, localJS *jsworker.Local, command string, args []string, workingDirectory string, env map[string]string) (*DOM, error) { + err := localJS.PostMessage(makeInitMessage(command, append([]string{command}, args...), workingDirectory, env), nil) + if err != nil { + return nil, err + } + local, err := NewLocal(ctx, localJS) if err != nil { return nil, err } diff --git a/internal/worker/local.go b/internal/worker/local.go index c10e09e..8076069 100644 --- a/internal/worker/local.go +++ b/internal/worker/local.go @@ -17,18 +17,18 @@ import ( ) type Local struct { - localJS *jsworker.Local - process *process.Process - pids map[common.PID]*Remote + localJS *jsworker.Local + process *process.Process + processStartCtx context.Context + pids map[common.PID]*Remote } -func NewLocal(localJS *jsworker.Local) (_ *Local, err error) { +func NewLocal(ctx context.Context, localJS *jsworker.Local) (_ *Local, err error) { local := &Local{ localJS: localJS, pids: make(map[common.PID]*Remote), } - - init, err := local.awaitInit(context.Background()) + init, err := local.awaitInit(ctx) if err != nil { return nil, err } @@ -52,7 +52,10 @@ func NewLocal(localJS *jsworker.Local) (_ *Local, err error) { localJS.PostMessage(js.ValueOf("ready"), nil) log.Debug("after ready post") - local.listenStart() + err = local.listenStart() + if err != nil { + return nil, err + } return local, nil } @@ -60,6 +63,7 @@ func NewLocal(localJS *jsworker.Local) (_ *Local, err error) { func (l *Local) awaitInit(ctx context.Context) (js.Value, error) { ctx, cancel := context.WithCancel(ctx) defer cancel() + l.processStartCtx = ctx type initMessage struct { err error @@ -83,6 +87,10 @@ func (l *Local) awaitInit(ctx context.Context) (js.Value, error) { if err != nil { return js.Value{}, err } + err = l.localJS.PostMessage(js.ValueOf("pending_init"), nil) + if err != nil { + return js.Value{}, err + } message := <-initChan return message.init, message.err } @@ -118,9 +126,17 @@ func (l *Local) listenStart() error { }) } +func (l *Local) Exit(exitCode int) error { + err := l.localJS.PostMessage(makeExitMessage(exitCode), nil) + if err != nil { + return err + } + return l.localJS.Close() +} + func (l *Local) Spawn(command string, argv []string, attr *process.ProcAttr) (jsprocess.PIDer, error) { pid := kernel.ReservePID() - log.Print("Spawning pid: ", pid, " for command: ", command, argv) + log.Print("Spawning pid ", pid, " for command: ", command, argv) remote, err := NewRemote(l, pid, command, argv, attr) if err != nil { return nil, err @@ -130,10 +146,24 @@ func (l *Local) Spawn(command string, argv []string, attr *process.ProcAttr) (js } func (l *Local) Wait(pid common.PID) (exitCode int, err error) { - log.Print("Waiting on pid: ", pid) + log.Print("Waiting on pid ", pid) remote, ok := l.pids[pid] if !ok { return 0, errors.Errorf("Unknown child process: %d", pid) } return remote.Wait() } + +func (l *Local) Started() <-chan struct{} { + return l.processStartCtx.Done() +} + +func (l *Local) PID() common.PID { + return l.process.PID() +} + +func makeExitMessage(exitCode int) js.Value { + return js.ValueOf(map[string]interface{}{ + "exitCode": exitCode, + }) +} diff --git a/internal/worker/remote.go b/internal/worker/remote.go index 2f2f143..397df4c 100644 --- a/internal/worker/remote.go +++ b/internal/worker/remote.go @@ -8,6 +8,7 @@ import ( "github.com/hack-pad/hackpad/internal/common" "github.com/hack-pad/hackpad/internal/interop" "github.com/hack-pad/hackpad/internal/jsworker" + "github.com/hack-pad/hackpad/internal/log" "github.com/hack-pad/hackpad/internal/process" ) @@ -25,6 +26,8 @@ type openFile struct { } func NewRemote(local *Local, pid process.PID, command string, argv []string, attr *process.ProcAttr) (*Remote, error) { + ctx := context.Background() + var openFiles []openFile for _, f := range attr.Files { info, err := local.process.Files().Fstat(f.FID) @@ -37,16 +40,23 @@ func NewRemote(local *Local, pid process.PID, command string, argv []string, att }) } // TODO inherit file descriptors - port, err := jsworker.NewRemoteWasm(fmt.Sprintf("pid-%d", pid), "/wasm/worker.wasm") + workerName := fmt.Sprintf("pid-%d", pid) + port, err := jsworker.NewRemoteWasm(workerName, "/wasm/worker.wasm") + if err != nil { + return nil, err + } + err = awaitMessage(ctx, port, "pending_init") if err != nil { return nil, err } + log.Warn("Sending init to worker ", workerName) err = port.PostMessage(makeInitMessage(command, argv, attr.Dir, attr.Env), nil) if err != nil { return nil, err } + log.Warn("init sent to worker ", workerName) - closeCtx, cancel := context.WithCancel(context.Background()) + closeCtx, cancel := context.WithCancel(ctx) remote := &Remote{ pid: pid, port: port, @@ -85,6 +95,30 @@ func (r *Remote) PID() common.PID { return r.pid } +func awaitMessage(ctx context.Context, port *jsworker.Remote, messageStr string) error { + ctx, cancel := context.WithCancel(ctx) + defer cancel() + result := make(chan error, 1) + err := port.Listen(ctx, func(me jsworker.MessageEvent, err error) { + if err != nil { + result <- err + return + } + if me.Data.Type() == js.TypeString && me.Data.String() == messageStr { + result <- nil + } + }) + if err != nil { + return err + } + select { + case <-ctx.Done(): + return ctx.Err() + case err := <-result: + return err + } +} + func makeInitMessage(command string, argv []string, workingDirectory string, env map[string]string) js.Value { return js.ValueOf(map[string]interface{}{ "init": map[string]interface{}{ diff --git a/main.go b/main.go index 0ad32df..0ee2857 100644 --- a/main.go +++ b/main.go @@ -4,16 +4,25 @@ package main import ( + "context" + "net/http" + "net/url" "os" + "path" "runtime/debug" "syscall/js" + "github.com/hack-pad/go-indexeddb/idb" "github.com/hack-pad/hackpad/internal/common" + "github.com/hack-pad/hackpad/internal/fs" "github.com/hack-pad/hackpad/internal/global" "github.com/hack-pad/hackpad/internal/interop" "github.com/hack-pad/hackpad/internal/jsworker" "github.com/hack-pad/hackpad/internal/log" "github.com/hack-pad/hackpad/internal/worker" + "github.com/hack-pad/hackpadfs" + "github.com/hack-pad/hackpadfs/indexeddb" + "github.com/johnstarich/go/datasize" ) type domShim struct { @@ -26,7 +35,11 @@ func main() { os.Exit(1) }) + bootCtx := context.Background() + //bootCtx, bootCancel := context.WithTimeout(context.Background(), 30*time.Second) + //defer bootCancel() dom, err := worker.ExecDOM( + bootCtx, jsworker.GetLocal(), "editor", []string{"-editor=editor"}, @@ -48,10 +61,7 @@ func main() { global.Set("install", js.FuncOf(shim.installFunc)) //global.Set("spawnTerminal", js.FuncOf(terminal.SpawnTerminal)) - if err := shim.Install("editor"); err != nil { - panic(err) - } - if err := shim.Install("sh"); err != nil { + if err := setUpFS(shim); err != nil { panic(err) } @@ -61,3 +71,73 @@ func main() { select {} } + +func setUpFS(shim domShim) error { + const dirPerm = 0700 + if err := os.MkdirAll("/bin", dirPerm); err != nil { + return err + } + if err := overlayIndexedDB("/bin", idb.DurabilityRelaxed); err != nil { + return err + } + if err := overlayIndexedDB("/home/me", idb.DurabilityDefault); err != nil { + return err + } + if err := os.MkdirAll("/home/me/.cache", dirPerm); err != nil { + return err + } + if err := overlayIndexedDB("/home/me/.cache", idb.DurabilityRelaxed); err != nil { + return err + } + if err := os.MkdirAll("/usr/local/go", dirPerm); err != nil { + return err + } + if err := overlayTarGzip("/usr/local/go", "wasm/go.tar.gz", []string{ + "/usr/local/go/bin/js_wasm", + "/usr/local/go/pkg/tool/js_wasm", + }); err != nil { + return err + } + + if err := shim.Install("editor"); err != nil { + return err + } + if err := shim.Install("sh"); err != nil { + return err + } + return nil +} + +func overlayIndexedDB(mountPath string, durability idb.TransactionDurability) error { + idbFS, err := indexeddb.NewFS(context.Background(), mountPath, indexeddb.Options{ + TransactionDurability: durability, + }) + if err != nil { + return err + } + return fs.Overlay(mountPath, idbFS) +} + +func overlayTarGzip(mountPath, downloadPath string, skipCacheDirs []string) error { + log.Debug("Downloading overlay .tar.gz FS: ", downloadPath) + u, err := url.Parse(downloadPath) + if err != nil { + return err + } + // only download from current server, not just any URL + resp, err := http.Get(u.Path) // nolint:bodyclose // Body is closed in OverlayTarGzip handler to keep this async + if err != nil { + return err + } + log.Debug("Download response received. Reading body...") + + skipDirs := make(map[string]bool) + for _, d := range skipCacheDirs { + skipDirs[common.ResolvePath("/", d)] = true + } + maxFileBytes := datasize.Kibibytes(100).Bytes() + shouldCache := func(name string, info hackpadfs.FileInfo) bool { + return !skipDirs[path.Dir(name)] && info.Size() < maxFileBytes + } + return fs.OverlayTarGzip(mountPath, resp.Body, true, shouldCache) +} diff --git a/server/src/Hackpad.js b/server/src/Hackpad.js index 8477aff..a0c7e7e 100644 --- a/server/src/Hackpad.js +++ b/server/src/Hackpad.js @@ -11,7 +11,7 @@ async function init() { const go = new Go(); const cmd = await WebAssembly.instantiateStreaming(fetch(`wasm/main.wasm`), go.importObject) go.run(cmd.instance) - const { hackpad, fs } = window + const { hackpad } = window const maxInitWaitMillis = 3000 await messageOrTimeout(message => { console.debug("message:", message) @@ -19,26 +19,6 @@ async function init() { }, maxInitWaitMillis) console.debug(`hackpad status: ${hackpad.ready ? 'ready' : 'not ready'}`) - const mkdir = promisify(fs.mkdir) - await mkdir("/bin", {mode: 0o700, recursive: true}) - //await hackpad.overlayIndexedDB('/bin', {cache: true}) - //await hackpad.overlayIndexedDB('/home/me') - await mkdir("/home/me/.cache", {recursive: true, mode: 0o700}) - //await hackpad.overlayIndexedDB('/home/me/.cache', {cache: true}) - - await mkdir("/usr/local/go", {recursive: true, mode: 0o700}) - //await hackpad.overlayTarGzip('/usr/local/go', 'wasm/go.tar.gz', { - // persist: true, - // skipCacheDirs: [ - // '/usr/local/go/bin/js_wasm', - // '/usr/local/go/pkg/tool/js_wasm', - // ], - // progress: percentage => { - // overlayProgress = percentage - // progressListeners.forEach(c => c(percentage)) - // }, - //}) - console.debug("Startup took", (new Date().getTime() - startTime) / 1000, "seconds") } From 7fccabd06be74367e4612ca922374114f5aec8d3 Mon Sep 17 00:00:00 2001 From: John Starich Date: Sun, 3 Apr 2022 22:37:38 -0500 Subject: [PATCH 11/37] Temporarily patch spawnTerminal back in --- internal/terminal/term.go | 1 + main.go | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/internal/terminal/term.go b/internal/terminal/term.go index f287e2d..1f74e28 100644 --- a/internal/terminal/term.go +++ b/internal/terminal/term.go @@ -31,6 +31,7 @@ func SpawnTerminal(this js.Value, args []js.Value) interface{} { } func Open(args []js.Value) error { + return errors.New("not implemented") if len(args) != 2 { return errors.New("Invalid number of args for spawnTerminal. Expected 2: term, options") } diff --git a/main.go b/main.go index 0ee2857..9163859 100644 --- a/main.go +++ b/main.go @@ -19,6 +19,7 @@ import ( "github.com/hack-pad/hackpad/internal/interop" "github.com/hack-pad/hackpad/internal/jsworker" "github.com/hack-pad/hackpad/internal/log" + "github.com/hack-pad/hackpad/internal/terminal" "github.com/hack-pad/hackpad/internal/worker" "github.com/hack-pad/hackpadfs" "github.com/hack-pad/hackpadfs/indexeddb" @@ -59,7 +60,7 @@ func main() { shim := domShim{dom} global.Set("profile", js.FuncOf(interop.ProfileJS)) global.Set("install", js.FuncOf(shim.installFunc)) - //global.Set("spawnTerminal", js.FuncOf(terminal.SpawnTerminal)) + global.Set("spawnTerminal", js.FuncOf(terminal.SpawnTerminal)) if err := setUpFS(shim); err != nil { panic(err) From bd139451ac8ce74fb37ec52f41f9d1b98b2d6489 Mon Sep 17 00:00:00 2001 From: John Starich Date: Sun, 3 Apr 2022 22:51:35 -0500 Subject: [PATCH 12/37] Add FS setup in worker, TODO reuse from main.go --- cmd/worker/main.go | 44 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/cmd/worker/main.go b/cmd/worker/main.go index 747ded1..34dcf89 100644 --- a/cmd/worker/main.go +++ b/cmd/worker/main.go @@ -4,10 +4,13 @@ import ( "context" "os" + "github.com/hack-pad/go-indexeddb/idb" "github.com/hack-pad/hackpad/internal/common" + "github.com/hack-pad/hackpad/internal/fs" "github.com/hack-pad/hackpad/internal/jsworker" "github.com/hack-pad/hackpad/internal/log" "github.com/hack-pad/hackpad/internal/worker" + "github.com/hack-pad/hackpadfs/indexeddb" ) func main() { @@ -26,6 +29,11 @@ func main() { panic(err) } log.Print("worker started") + + if err := setUpFS(); err != nil { + panic(err) + } + <-local.Started() pid := local.PID() log.Print("worker process started PID ", pid) @@ -38,3 +46,39 @@ func main() { local.Exit(exitCode) os.Exit(exitCode) } + +func setUpFS() error { + const dirPerm = 0700 + if err := os.MkdirAll("/bin", dirPerm); err != nil { + return err + } + if err := overlayIndexedDB("/bin", idb.DurabilityRelaxed); err != nil { + return err + } + if err := overlayIndexedDB("/home/me", idb.DurabilityDefault); err != nil { + return err + } + if err := os.MkdirAll("/home/me/.cache", dirPerm); err != nil { + return err + } + if err := overlayIndexedDB("/home/me/.cache", idb.DurabilityRelaxed); err != nil { + return err + } + if err := os.MkdirAll("/usr/local/go", dirPerm); err != nil { + return err + } + if err := overlayIndexedDB("/usr/local/go", idb.DurabilityRelaxed); err != nil { + return err + } + return nil +} + +func overlayIndexedDB(mountPath string, durability idb.TransactionDurability) error { + idbFS, err := indexeddb.NewFS(context.Background(), mountPath, indexeddb.Options{ + TransactionDurability: durability, + }) + if err != nil { + return err + } + return fs.Overlay(mountPath, idbFS) +} From 6c97ee78370292d51e58030cd94efc4e2b97e9d8 Mon Sep 17 00:00:00 2001 From: John Starich Date: Sun, 3 Apr 2022 22:51:52 -0500 Subject: [PATCH 13/37] Fix blocking event loop in event handler --- internal/jsworker/message_port.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/jsworker/message_port.go b/internal/jsworker/message_port.go index 7f0a6ff..5c25552 100644 --- a/internal/jsworker/message_port.go +++ b/internal/jsworker/message_port.go @@ -51,7 +51,7 @@ func (p *MessagePort) Listen(ctx context.Context, listener func(MessageEvent, er messageHandler := js.FuncOf(func(this js.Value, args []js.Value) interface{} { ev, err := parseMessageEvent(args[0]) - listener(ev, err) + go listener(ev, err) return nil }) errorHandler := js.FuncOf(func(this js.Value, args []js.Value) interface{} { @@ -59,7 +59,7 @@ func (p *MessagePort) Listen(ctx context.Context, listener func(MessageEvent, er if err == nil { err = MessageEventErr{ev} } - listener(MessageEvent{}, err) + go listener(MessageEvent{}, err) return nil }) From 5f3b386de38ba30af82f1155d0dd2943fe04e371 Mon Sep 17 00:00:00 2001 From: John Starich Date: Sun, 10 Apr 2022 16:16:37 -0500 Subject: [PATCH 14/37] Add .env file for automatic GOOS/GOARCH editor settings --- .env | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 .env diff --git a/.env b/.env new file mode 100644 index 0000000..d5fbc82 --- /dev/null +++ b/.env @@ -0,0 +1,2 @@ +GOOS=js +GOARCH=wasm From d0c3bac5a95b1c936b4775dbdf0440025c58a2d7 Mon Sep 17 00:00:00 2001 From: John Starich Date: Sun, 10 Apr 2022 17:14:57 -0500 Subject: [PATCH 15/37] Audit js.FuncOf usage, replace with jsfunc package; fix import cycle with new jserror pkg --- cmd/editor/dom/element.go | 3 +- cmd/editor/dom/window.go | 5 +- cmd/editor/editor.go | 26 +++---- cmd/editor/main.go | 3 +- cmd/editor/terminal/terminal.go | 3 +- install.go | 15 +--- internal/fs/file_descriptors.go | 3 +- internal/interop/error.go | 71 ++----------------- internal/interop/funcs.go | 4 +- internal/interop/profile.go | 11 ++- internal/js/fs/fs.go | 38 ++++------ internal/js/fs/overlay.go | 41 ++++------- internal/jserror/error.go | 69 ++++++++++++++++++ internal/{interop => jserror}/error_js.go | 8 +-- internal/jsfunc/non_block.go | 10 +++ internal/jsfunc/promise.go | 25 +++++++ .../once_func.go => jsfunc/single_use.go} | 6 +- internal/jsworker/message_port.go | 9 +-- internal/process/process_js.go | 3 +- internal/process/wasm.go | 3 +- internal/promise/js.go | 14 +++- internal/terminal/term.go | 20 +++--- main.go | 8 +-- 23 files changed, 214 insertions(+), 184 deletions(-) create mode 100644 internal/jserror/error.go rename internal/{interop => jserror}/error_js.go (69%) create mode 100644 internal/jsfunc/non_block.go create mode 100644 internal/jsfunc/promise.go rename internal/{interop/once_func.go => jsfunc/single_use.go} (64%) diff --git a/cmd/editor/dom/element.go b/cmd/editor/dom/element.go index d6f20aa..7240523 100644 --- a/cmd/editor/dom/element.go +++ b/cmd/editor/dom/element.go @@ -8,6 +8,7 @@ import ( "github.com/hack-pad/hackpad/internal/common" "github.com/hack-pad/hackpad/internal/interop" + "github.com/hack-pad/hackpad/internal/jsfunc" "github.com/hack-pad/hackpad/internal/log" ) @@ -112,7 +113,7 @@ func (e *Element) QuerySelectorAll(query string) []*Element { } func (e *Element) AddEventListener(name string, listener EventListener) { - e.elem.Call("addEventListener", name, js.FuncOf(func(this js.Value, args []js.Value) interface{} { + e.elem.Call("addEventListener", name, jsfunc.NonBlocking(func(this js.Value, args []js.Value) interface{} { defer common.CatchExceptionHandler(func(err error) { log.Error("recovered from panic: ", err, "\n", string(debug.Stack())) }) diff --git a/cmd/editor/dom/window.go b/cmd/editor/dom/window.go index 35f92fb..692f402 100644 --- a/cmd/editor/dom/window.go +++ b/cmd/editor/dom/window.go @@ -7,6 +7,7 @@ import ( "time" "github.com/hack-pad/hackpad/internal/interop" + "github.com/hack-pad/hackpad/internal/jsfunc" ) var ( @@ -15,7 +16,7 @@ var ( func SetTimeout(fn func(args []js.Value), delay time.Duration, args ...js.Value) int { intArgs := append([]interface{}{ - interop.SingleUseFunc(func(_ js.Value, args []js.Value) interface{} { + jsfunc.SingleUse(func(_ js.Value, args []js.Value) interface{} { fn(args) return nil }), @@ -28,7 +29,7 @@ func SetTimeout(fn func(args []js.Value), delay time.Duration, args ...js.Value) func QueueMicrotask(fn func()) { queueMicrotask := window.GetProperty("queueMicrotask") if queueMicrotask.Truthy() { - queueMicrotask.Invoke(interop.SingleUseFunc(func(this js.Value, args []js.Value) interface{} { + queueMicrotask.Invoke(jsfunc.SingleUse(func(this js.Value, args []js.Value) interface{} { fn() return nil })) diff --git a/cmd/editor/editor.go b/cmd/editor/editor.go index 1cd268b..9928c8d 100644 --- a/cmd/editor/editor.go +++ b/cmd/editor/editor.go @@ -1,3 +1,4 @@ +//go:build js // +build js package main @@ -9,6 +10,7 @@ import ( "github.com/hack-pad/hackpad/cmd/editor/dom" "github.com/hack-pad/hackpad/cmd/editor/ide" + "github.com/hack-pad/hackpad/internal/jsfunc" "github.com/hack-pad/hackpad/internal/log" ) @@ -25,7 +27,7 @@ func (e editorJSFunc) New(elem *dom.Element) ide.Editor { editor := &jsEditor{ titleChan: make(chan string, 1), } - editor.elem = js.Value(e).Invoke(elem, js.FuncOf(editor.onEdit)) + editor.elem = js.Value(e).Invoke(elem, jsfunc.NonBlocking(editor.onEdit)) return editor } @@ -36,18 +38,16 @@ type jsEditor struct { } func (j *jsEditor) onEdit(js.Value, []js.Value) interface{} { - go func() { - contents := j.elem.Call("getContents").String() - perm := os.FileMode(0700) - info, err := os.Stat(j.filePath) - if err == nil { - perm = info.Mode() - } - err = ioutil.WriteFile(j.filePath, []byte(contents), perm) - if err != nil { - log.Error("Failed to write file contents: ", err) - } - }() + contents := j.elem.Call("getContents").String() + perm := os.FileMode(0700) + info, err := os.Stat(j.filePath) + if err == nil { + perm = info.Mode() + } + err = ioutil.WriteFile(j.filePath, []byte(contents), perm) + if err != nil { + log.Error("Failed to write file contents: ", err) + } return nil } diff --git a/cmd/editor/main.go b/cmd/editor/main.go index a4f1651..86979d2 100644 --- a/cmd/editor/main.go +++ b/cmd/editor/main.go @@ -16,6 +16,7 @@ import ( "github.com/hack-pad/hackpad/cmd/editor/terminal" "github.com/hack-pad/hackpad/internal/common" "github.com/hack-pad/hackpad/internal/interop" + "github.com/hack-pad/hackpad/internal/jsfunc" "github.com/hack-pad/hackpad/internal/log" ) @@ -40,7 +41,7 @@ func main() { app := dom.GetDocument().GetElementByID(*editorID) app.AddClass("ide") globalEditorProps := js.Global().Get("editor") - globalEditorProps.Set("profile", js.FuncOf(interop.ProfileJS)) + globalEditorProps.Set("profile", jsfunc.NonBlocking(interop.ProfileJS)) newEditor := globalEditorProps.Get("newEditor") var editorBuilder ide.EditorBuilder = editorJSFunc(newEditor) if !newEditor.Truthy() { diff --git a/cmd/editor/terminal/terminal.go b/cmd/editor/terminal/terminal.go index abb485d..74488c0 100644 --- a/cmd/editor/terminal/terminal.go +++ b/cmd/editor/terminal/terminal.go @@ -10,6 +10,7 @@ import ( "github.com/hack-pad/hackpad/cmd/editor/dom" "github.com/hack-pad/hackpad/cmd/editor/ide" "github.com/hack-pad/hackpad/internal/common" + "github.com/hack-pad/hackpad/internal/jsfunc" "github.com/hack-pad/hackpad/internal/log" "github.com/hack-pad/hackpadfs/indexeddb/idbblob" "github.com/hack-pad/hackpadfs/keyvalue/blob" @@ -74,7 +75,7 @@ func (t *terminal) start(rawName, name string, args ...string) error { return err } - f := js.FuncOf(func(this js.Value, args []js.Value) interface{} { + f := jsfunc.NonBlocking(func(this js.Value, args []js.Value) interface{} { chunk := []byte(args[0].String()) _, err := stdin.Write(chunk) if err == io.EOF { diff --git a/install.go b/install.go index 5f14bcb..571d5f9 100644 --- a/install.go +++ b/install.go @@ -9,22 +9,11 @@ import ( "runtime" "syscall/js" - "github.com/hack-pad/hackpad/internal/interop" "github.com/hack-pad/hackpad/internal/log" - "github.com/hack-pad/hackpad/internal/promise" ) -func (s domShim) installFunc(this js.Value, args []js.Value) interface{} { - resolve, reject, prom := promise.New() - go func() { - err := s.install(args) - if err != nil { - reject(interop.WrapAsJSError(err, "Failed to install binary")) - return - } - resolve(nil) - }() - return prom +func (s domShim) installFunc(this js.Value, args []js.Value) (js.Wrapper, error) { + return nil, s.install(args) } func (s domShim) install(args []js.Value) error { diff --git a/internal/fs/file_descriptors.go b/internal/fs/file_descriptors.go index b7f2e75..21e8bb0 100644 --- a/internal/fs/file_descriptors.go +++ b/internal/fs/file_descriptors.go @@ -13,12 +13,13 @@ import ( "github.com/hack-pad/hackpad/internal/common" "github.com/hack-pad/hackpad/internal/interop" + "github.com/hack-pad/hackpad/internal/jserror" "github.com/hack-pad/hackpadfs" "github.com/pkg/errors" ) var ( - ErrNotDir = interop.NewError("not a directory", "ENOTDIR") + ErrNotDir = jserror.New("not a directory", "ENOTDIR") ) type FileDescriptors struct { diff --git a/internal/interop/error.go b/internal/interop/error.go index 6a22570..aeae714 100644 --- a/internal/interop/error.go +++ b/internal/interop/error.go @@ -2,82 +2,19 @@ package interop import ( "fmt" - "io" - "os/exec" "github.com/hack-pad/hackpad/internal/common" - "github.com/hack-pad/hackpad/internal/log" - "github.com/hack-pad/hackpadfs" - "github.com/pkg/errors" + "github.com/hack-pad/hackpad/internal/jserror" ) var ( - ErrNotImplemented = NewError("operation not supported", "ENOSYS") + ErrNotImplemented = jserror.New("operation not supported", "ENOSYS") ) -type Error interface { - error - Message() string - Code() string -} - -type interopErr struct { - error - code string -} - -func NewError(message, code string) Error { - return WrapErr(errors.New(message), code) -} - -func WrapErr(err error, code string) Error { - return &interopErr{ - error: err, - code: code, - } -} - -func (e *interopErr) Message() string { - return e.Error() -} - -func (e *interopErr) Code() string { - return e.code -} - -// errno names pulled from syscall/tables_js.go -func mapToErrNo(err error, debugMessage string) string { - if err, ok := err.(Error); ok { - return err.Code() - } - if err, ok := err.(interface{ Unwrap() error }); ok { - return mapToErrNo(err.Unwrap(), debugMessage) - } - switch err { - case io.EOF, exec.ErrNotFound: - return "ENOENT" - } - switch { - case errors.Is(err, hackpadfs.ErrClosed): - return "EBADF" // if it was already closed, then the file descriptor was invalid - case errors.Is(err, hackpadfs.ErrNotExist): - return "ENOENT" - case errors.Is(err, hackpadfs.ErrExist): - return "EEXIST" - case errors.Is(err, hackpadfs.ErrIsDir): - return "EISDIR" - case errors.Is(err, hackpadfs.ErrPermission): - return "EPERM" - default: - log.Errorf("Unknown error type: (%T) %+v\n\n%s", err, err, debugMessage) - return "EPERM" - } -} - func BadFileNumber(fd common.FID) error { - return NewError(fmt.Sprintf("Bad file number %d", fd), "EBADF") + return jserror.New(fmt.Sprintf("Bad file number %d", fd), "EBADF") } func BadFileErr(identifier string) error { - return NewError(fmt.Sprintf("Bad file %q", identifier), "EBADF") + return jserror.New(fmt.Sprintf("Bad file %q", identifier), "EBADF") } diff --git a/internal/interop/funcs.go b/internal/interop/funcs.go index 4881c7a..e3e4ced 100644 --- a/internal/interop/funcs.go +++ b/internal/interop/funcs.go @@ -1,3 +1,4 @@ +//go:build js // +build js package interop @@ -8,6 +9,7 @@ import ( "strings" "syscall/js" + "github.com/hack-pad/hackpad/internal/jserror" "github.com/hack-pad/hackpad/internal/log" "github.com/pkg/errors" ) @@ -68,7 +70,7 @@ func setFuncHandler(name string, fn interface{}, args []js.Value) (returnedVal i }() ret, err = fn(args) - err = wrapAsJSError(err, name, args...) + err = jserror.WrapArgs(err, name, args...) ret = append([]interface{}{err}, ret...) callback.Invoke(ret...) }() diff --git a/internal/interop/profile.go b/internal/interop/profile.go index 3b488ff..b880472 100644 --- a/internal/interop/profile.go +++ b/internal/interop/profile.go @@ -1,3 +1,4 @@ +//go:build js // +build js package interop @@ -15,12 +16,10 @@ import ( ) func ProfileJS(this js.Value, args []js.Value) interface{} { - go func() { - MemoryProfileJS(this, args) - // Re-enable once these profiles actually work in the browser. Currently produces 0 samples. - //TraceProfileJS(this, args) - //StartCPUProfileJS(this, args) - }() + MemoryProfileJS(this, args) + // Re-enable once these profiles actually work in the browser. Currently produces 0 samples. + //TraceProfileJS(this, args) + //StartCPUProfileJS(this, args) return nil } diff --git a/internal/js/fs/fs.go b/internal/js/fs/fs.go index 08fff2c..331d384 100644 --- a/internal/js/fs/fs.go +++ b/internal/js/fs/fs.go @@ -12,8 +12,9 @@ import ( "github.com/hack-pad/hackpad/internal/fs" "github.com/hack-pad/hackpad/internal/global" "github.com/hack-pad/hackpad/internal/interop" + "github.com/hack-pad/hackpad/internal/jserror" + "github.com/hack-pad/hackpad/internal/jsfunc" "github.com/hack-pad/hackpad/internal/process" - "github.com/hack-pad/hackpad/internal/promise" ) type fileShim struct { @@ -73,11 +74,11 @@ func Init(process *process.Process) { interop.SetFunc(fs, "write", shim.write) interop.SetFunc(fs, "writeSync", shim.writeSync) - global.Set("getMounts", js.FuncOf(shim.getMounts)) - global.Set("destroyMount", js.FuncOf(shim.destroyMount)) - global.Set("overlayTarGzip", js.FuncOf(shim.overlayTarGzip)) - global.Set("overlayIndexedDB", js.FuncOf(shim.overlayIndexedDB)) - global.Set("dumpZip", js.FuncOf(shim.dumpZip)) + global.Set("getMounts", jsfunc.Promise(shim.getMounts)) + global.Set("destroyMount", jsfunc.Promise(shim.destroyMount)) + global.Set("overlayTarGzip", jsfunc.Promise(shim.overlayTarGzip)) + global.Set("overlayIndexedDB", jsfunc.Promise(shim.overlayIndexedDB)) + global.Set("dumpZip", jsfunc.Promise(shim.dumpZip)) // Set up system directories files := process.Files() @@ -91,36 +92,27 @@ func (s fileShim) Dump(basePath string) interface{} { return fs.Dump(basePath) } -func (s fileShim) dumpZip(this js.Value, args []js.Value) interface{} { +func (s fileShim) dumpZip(this js.Value, args []js.Value) (js.Wrapper, error) { if len(args) != 1 { - return interop.WrapAsJSError(errors.New("dumpZip: file path is required"), "EINVAL") + return nil, jserror.Wrap(errors.New("dumpZip: file path is required"), "EINVAL") } path := args[0].String() path = common.ResolvePath(s.process.WorkingDirectory(), path) - return interop.WrapAsJSError(fs.DumpZip(path), "dumpZip") + return nil, jserror.Wrap(fs.DumpZip(path), "dumpZip") } -func (s fileShim) getMounts(this js.Value, args []js.Value) interface{} { +func (s fileShim) getMounts(this js.Value, args []js.Value) (js.Wrapper, error) { var mounts []string for _, p := range fs.Mounts() { mounts = append(mounts, p.Path) } - return interop.SliceFromStrings(mounts) + return interop.SliceFromStrings(mounts), nil } -func (s fileShim) destroyMount(this js.Value, args []js.Value) interface{} { +func (s fileShim) destroyMount(this js.Value, args []js.Value) (js.Wrapper, error) { if len(args) < 1 { - return interop.WrapAsJSError(errors.New("destroyMount: mount path is required"), "EINVAL") + return nil, jserror.Wrap(errors.New("destroyMount: mount path is required"), "EINVAL") } - resolve, reject, prom := promise.New() mountPath := args[0].String() - go func() { - err := interop.WrapAsJSError(fs.DestroyMount(mountPath), "destroyMount") - if err != nil { - reject(err) - } else { - resolve(nil) - } - }() - return prom + return nil, jserror.Wrap(fs.DestroyMount(mountPath), "destroyMount") } diff --git a/internal/js/fs/overlay.go b/internal/js/fs/overlay.go index 4af9f43..3e6633c 100644 --- a/internal/js/fs/overlay.go +++ b/internal/js/fs/overlay.go @@ -20,23 +20,18 @@ import ( "github.com/hack-pad/hackpad/internal/common" "github.com/hack-pad/hackpad/internal/fs" "github.com/hack-pad/hackpad/internal/interop" + "github.com/hack-pad/hackpad/internal/jserror" "github.com/hack-pad/hackpad/internal/log" - "github.com/hack-pad/hackpad/internal/promise" "github.com/johnstarich/go/datasize" ) -func (s fileShim) overlayIndexedDB(this js.Value, args []js.Value) interface{} { - resolve, reject, prom := promise.New() - go func() { - err := s.OverlayIndexedDB(args) - if err != nil { - reject(interop.WrapAsJSError(err, "Failed overlaying IndexedDB FS")) - } else { - log.Debug("Successfully overlayed IndexedDB FS") - resolve(nil) - } - }() - return prom +func (s fileShim) overlayIndexedDB(this js.Value, args []js.Value) (js.Wrapper, error) { + err := s.OverlayIndexedDB(args) + if err != nil { + return nil, jserror.Wrap(err, "Failed overlaying IndexedDB FS") + } + log.Debug("Successfully overlayed IndexedDB FS") + return nil, nil } func (s fileShim) OverlayIndexedDB(args []js.Value) (err error) { @@ -63,19 +58,13 @@ func (s fileShim) OverlayIndexedDB(args []js.Value) (err error) { return fs.Overlay(mountPath, idbFS) } -func (s fileShim) overlayTarGzip(this js.Value, args []js.Value) interface{} { - resolve, reject, prom := promise.New() - log.Debug("Backgrounding overlay request") - go func() { - err := s.OverlayTarGzip(args) - if err != nil { - reject(interop.WrapAsJSError(err, "Failed overlaying .tar.gz FS")) - } else { - log.Debug("Successfully overlayed .tar.gz FS") - resolve(nil) - } - }() - return prom +func (s fileShim) overlayTarGzip(this js.Value, args []js.Value) (js.Wrapper, error) { + err := s.OverlayTarGzip(args) + if err != nil { + return nil, jserror.Wrap(err, "Failed overlaying .tar.gz FS") + } + log.Debug("Successfully overlayed .tar.gz FS") + return nil, nil } func (s fileShim) OverlayTarGzip(args []js.Value) error { diff --git a/internal/jserror/error.go b/internal/jserror/error.go new file mode 100644 index 0000000..d059611 --- /dev/null +++ b/internal/jserror/error.go @@ -0,0 +1,69 @@ +package jserror + +import ( + "io" + "os/exec" + + "github.com/hack-pad/hackpad/internal/log" + "github.com/hack-pad/hackpadfs" + "github.com/pkg/errors" +) + +type Error interface { + error + Message() string + Code() string +} + +type jsErr struct { + error + code string +} + +func New(message, code string) Error { + return WrapErr(errors.New(message), code) +} + +func WrapErr(err error, code string) Error { + return &jsErr{ + error: err, + code: code, + } +} + +func (e *jsErr) Message() string { + return e.Error() +} + +func (e *jsErr) Code() string { + return e.code +} + +// errno names pulled from syscall/tables_js.go +func mapToErrNo(err error, debugMessage string) string { + if err, ok := err.(Error); ok { + return err.Code() + } + if err, ok := err.(interface{ Unwrap() error }); ok { + return mapToErrNo(err.Unwrap(), debugMessage) + } + switch err { + case io.EOF, exec.ErrNotFound: + return "ENOENT" + } + switch { + case errors.Is(err, hackpadfs.ErrClosed): + return "EBADF" // if it was already closed, then the file descriptor was invalid + case errors.Is(err, hackpadfs.ErrNotExist): + return "ENOENT" + case errors.Is(err, hackpadfs.ErrExist): + return "EEXIST" + case errors.Is(err, hackpadfs.ErrIsDir): + return "EISDIR" + case errors.Is(err, hackpadfs.ErrPermission): + return "EPERM" + default: + log.Errorf("Unknown error type: (%T) %+v\n\n%s", err, err, debugMessage) + return "EPERM" + } +} diff --git a/internal/interop/error_js.go b/internal/jserror/error_js.go similarity index 69% rename from internal/interop/error_js.go rename to internal/jserror/error_js.go index e1ca60c..400c550 100644 --- a/internal/interop/error_js.go +++ b/internal/jserror/error_js.go @@ -1,6 +1,6 @@ // +build js -package interop +package jserror import ( "fmt" @@ -9,11 +9,11 @@ import ( "github.com/pkg/errors" ) -func WrapAsJSError(err error, message string) error { - return wrapAsJSError(err, message) +func Wrap(err error, message string) error { + return WrapArgs(err, message) } -func wrapAsJSError(err error, message string, args ...js.Value) error { +func WrapArgs(err error, message string, args ...js.Value) error { if err == nil { return nil } diff --git a/internal/jsfunc/non_block.go b/internal/jsfunc/non_block.go new file mode 100644 index 0000000..1b7fca1 --- /dev/null +++ b/internal/jsfunc/non_block.go @@ -0,0 +1,10 @@ +package jsfunc + +import "syscall/js" + +func NonBlocking(fn Func) js.Func { + return js.FuncOf(func(this js.Value, args []js.Value) interface{} { + go fn(this, args) + return nil + }) +} diff --git a/internal/jsfunc/promise.go b/internal/jsfunc/promise.go new file mode 100644 index 0000000..8fbd691 --- /dev/null +++ b/internal/jsfunc/promise.go @@ -0,0 +1,25 @@ +package jsfunc + +import ( + "syscall/js" + + "github.com/hack-pad/hackpad/internal/jserror" + "github.com/hack-pad/hackpad/internal/promise" +) + +type ErrFunc = func(this js.Value, args []js.Value) (js.Wrapper, error) + +func Promise(fn ErrFunc) js.Func { + return js.FuncOf(func(this js.Value, args []js.Value) interface{} { + resolve, reject, prom := promise.New() + go func() { + value, err := fn(this, args) + if err != nil { + reject(jserror.Wrap(err, "Failed to install binary")) + return + } + resolve(value) + }() + return prom + }) +} diff --git a/internal/interop/once_func.go b/internal/jsfunc/single_use.go similarity index 64% rename from internal/interop/once_func.go rename to internal/jsfunc/single_use.go index 9f24635..8e684ba 100644 --- a/internal/interop/once_func.go +++ b/internal/jsfunc/single_use.go @@ -1,10 +1,12 @@ // +build js -package interop +package jsfunc import "syscall/js" -func SingleUseFunc(fn func(this js.Value, args []js.Value) interface{}) js.Func { +type Func = func(this js.Value, args []js.Value) interface{} + +func SingleUse(fn Func) js.Func { var wrapperFn js.Func wrapperFn = js.FuncOf(func(this js.Value, args []js.Value) interface{} { wrapperFn.Release() diff --git a/internal/jsworker/message_port.go b/internal/jsworker/message_port.go index 5c25552..73fc69b 100644 --- a/internal/jsworker/message_port.go +++ b/internal/jsworker/message_port.go @@ -7,6 +7,7 @@ import ( "github.com/hack-pad/hackpad/internal/common" "github.com/hack-pad/hackpad/internal/interop" + "github.com/hack-pad/hackpad/internal/jsfunc" "github.com/hack-pad/hackpad/internal/log" ) @@ -49,17 +50,17 @@ func (p *MessagePort) Listen(ctx context.Context, listener func(MessageEvent, er cancel() }) - messageHandler := js.FuncOf(func(this js.Value, args []js.Value) interface{} { + messageHandler := jsfunc.NonBlocking(func(this js.Value, args []js.Value) interface{} { ev, err := parseMessageEvent(args[0]) - go listener(ev, err) + listener(ev, err) return nil }) - errorHandler := js.FuncOf(func(this js.Value, args []js.Value) interface{} { + errorHandler := jsfunc.NonBlocking(func(this js.Value, args []js.Value) interface{} { ev, err := parseMessageEvent(args[0]) if err == nil { err = MessageEventErr{ev} } - go listener(MessageEvent{}, err) + listener(MessageEvent{}, err) return nil }) diff --git a/internal/process/process_js.go b/internal/process/process_js.go index 504d9d6..32511da 100644 --- a/internal/process/process_js.go +++ b/internal/process/process_js.go @@ -6,6 +6,7 @@ import ( "syscall/js" "github.com/hack-pad/hackpad/internal/interop" + "github.com/hack-pad/hackpad/internal/jserror" ) var ( @@ -16,7 +17,7 @@ func (p *Process) JSValue() js.Value { return js.ValueOf(map[string]interface{}{ "pid": p.pid, "ppid": p.parentPID, - "error": interop.WrapAsJSError(p.err, "spawn"), + "error": jserror.Wrap(p.err, "spawn"), }) } diff --git a/internal/process/wasm.go b/internal/process/wasm.go index b9d144e..eacaac5 100644 --- a/internal/process/wasm.go +++ b/internal/process/wasm.go @@ -9,6 +9,7 @@ import ( "syscall/js" "github.com/hack-pad/hackpad/internal/interop" + "github.com/hack-pad/hackpad/internal/jsfunc" "github.com/hack-pad/hackpad/internal/log" "github.com/hack-pad/hackpad/internal/promise" ) @@ -46,7 +47,7 @@ func (p *Process) startWasmPromise(path string, exitChan chan<- int) (promise.Pr } goInstance.Set("env", interop.StringMap(p.env)) var resumeFuncPtr *js.Func - goInstance.Set("exit", interop.SingleUseFunc(func(this js.Value, args []js.Value) interface{} { + goInstance.Set("exit", jsfunc.SingleUse(func(this js.Value, args []js.Value) interface{} { defer func() { if resumeFuncPtr != nil { resumeFuncPtr.Release() diff --git a/internal/promise/js.go b/internal/promise/js.go index 88fe6b7..c0d00ba 100644 --- a/internal/promise/js.go +++ b/internal/promise/js.go @@ -6,7 +6,6 @@ import ( "runtime/debug" "syscall/js" - "github.com/hack-pad/hackpad/internal/interop" "github.com/hack-pad/hackpad/internal/log" ) @@ -23,7 +22,7 @@ func From(promiseValue js.Value) JS { func New() (resolve, reject Resolver, promise JS) { resolvers := make(chan Resolver, 2) promise = From( - jsPromise.New(interop.SingleUseFunc(func(this js.Value, args []js.Value) interface{} { + jsPromise.New(singleUseFunc(func(this js.Value, args []js.Value) interface{} { resolve, reject := args[0], args[1] resolvers <- func(result interface{}) { resolve.Invoke(result) } resolvers <- func(result interface{}) { reject.Invoke(result) } @@ -34,13 +33,22 @@ func New() (resolve, reject Resolver, promise JS) { return } +func singleUseFunc(fn func(this js.Value, args []js.Value) interface{}) js.Func { + var wrapperFn js.Func + wrapperFn = js.FuncOf(func(this js.Value, args []js.Value) interface{} { + wrapperFn.Release() + return fn(this, args) + }) + return wrapperFn +} + func (p JS) Then(fn func(value interface{}) interface{}) Promise { return p.do("then", fn) } func (p JS) do(methodName string, fn func(value interface{}) interface{}) Promise { return JS{ - value: p.value.Call(methodName, interop.SingleUseFunc(func(this js.Value, args []js.Value) interface{} { + value: p.value.Call(methodName, singleUseFunc(func(this js.Value, args []js.Value) interface{} { var value js.Value if len(args) > 0 { value = args[0] diff --git a/internal/terminal/term.go b/internal/terminal/term.go index 1f74e28..9b751a1 100644 --- a/internal/terminal/term.go +++ b/internal/terminal/term.go @@ -1,3 +1,4 @@ +//go:build js // +build js package terminal @@ -8,6 +9,7 @@ import ( "github.com/hack-pad/hackpad/internal/common" "github.com/hack-pad/hackpad/internal/fs" "github.com/hack-pad/hackpad/internal/interop" + "github.com/hack-pad/hackpad/internal/jsfunc" "github.com/hack-pad/hackpad/internal/kernel" "github.com/hack-pad/hackpad/internal/log" "github.com/hack-pad/hackpad/internal/process" @@ -16,17 +18,15 @@ import ( ) func SpawnTerminal(this js.Value, args []js.Value) interface{} { - go func() { - defer func() { - if r := recover(); r != nil { - log.Error("Recovered from panic:", r) - } - }() - err := Open(args) - if err != nil { - log.Error(err) + defer func() { + if r := recover(); r != nil { + log.Error("Recovered from panic:", r) } }() + err := Open(args) + if err != nil { + log.Error(err) + } return nil } @@ -77,7 +77,7 @@ func Open(args []js.Value) error { return err } - f := js.FuncOf(func(this js.Value, args []js.Value) interface{} { + f := jsfunc.NonBlocking(func(this js.Value, args []js.Value) interface{} { chunk, err := idbblob.New(args[0]) if err != nil { log.Error("blob: Failed to write to terminal:", err) diff --git a/main.go b/main.go index 9163859..f6c92ad 100644 --- a/main.go +++ b/main.go @@ -10,13 +10,13 @@ import ( "os" "path" "runtime/debug" - "syscall/js" "github.com/hack-pad/go-indexeddb/idb" "github.com/hack-pad/hackpad/internal/common" "github.com/hack-pad/hackpad/internal/fs" "github.com/hack-pad/hackpad/internal/global" "github.com/hack-pad/hackpad/internal/interop" + "github.com/hack-pad/hackpad/internal/jsfunc" "github.com/hack-pad/hackpad/internal/jsworker" "github.com/hack-pad/hackpad/internal/log" "github.com/hack-pad/hackpad/internal/terminal" @@ -58,9 +58,9 @@ func main() { } shim := domShim{dom} - global.Set("profile", js.FuncOf(interop.ProfileJS)) - global.Set("install", js.FuncOf(shim.installFunc)) - global.Set("spawnTerminal", js.FuncOf(terminal.SpawnTerminal)) + global.Set("profile", jsfunc.NonBlocking(interop.ProfileJS)) + global.Set("install", jsfunc.Promise(shim.installFunc)) + global.Set("spawnTerminal", jsfunc.NonBlocking(terminal.SpawnTerminal)) if err := setUpFS(shim); err != nil { panic(err) From d2ed01859c701cbac4ae0e6e2c46a9821a4512ab Mon Sep 17 00:00:00 2001 From: John Starich Date: Mon, 18 Apr 2022 21:01:33 -0500 Subject: [PATCH 16/37] Remove unused func --- internal/process/wasm.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/internal/process/wasm.go b/internal/process/wasm.go index eacaac5..4a4b203 100644 --- a/internal/process/wasm.go +++ b/internal/process/wasm.go @@ -46,12 +46,8 @@ func (p *Process) startWasmPromise(path string, exitChan chan<- int) (promise.Pr p.env = splitEnvPairs(os.Environ()) } goInstance.Set("env", interop.StringMap(p.env)) - var resumeFuncPtr *js.Func goInstance.Set("exit", jsfunc.SingleUse(func(this js.Value, args []js.Value) interface{} { defer func() { - if resumeFuncPtr != nil { - resumeFuncPtr.Release() - } // TODO exit hook for worker // TODO free the whole goInstance to fix garbage issues entirely. Freeing individual properties appears to work for now, but is ultimately a bad long-term solution because memory still accumulates. From f395e40a03e80d02d6ebb6216c0c6c8aba62bc12 Mon Sep 17 00:00:00 2001 From: John Starich Date: Mon, 18 Apr 2022 21:02:06 -0500 Subject: [PATCH 17/37] Make window event handlers async --- cmd/editor/dom/window.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/editor/dom/window.go b/cmd/editor/dom/window.go index 692f402..c2ed546 100644 --- a/cmd/editor/dom/window.go +++ b/cmd/editor/dom/window.go @@ -17,7 +17,7 @@ var ( func SetTimeout(fn func(args []js.Value), delay time.Duration, args ...js.Value) int { intArgs := append([]interface{}{ jsfunc.SingleUse(func(_ js.Value, args []js.Value) interface{} { - fn(args) + go fn(args) return nil }), delay.Milliseconds(), @@ -30,7 +30,7 @@ func QueueMicrotask(fn func()) { queueMicrotask := window.GetProperty("queueMicrotask") if queueMicrotask.Truthy() { queueMicrotask.Invoke(jsfunc.SingleUse(func(this js.Value, args []js.Value) interface{} { - fn() + go fn() return nil })) } else { From 4c1bdea50de73625ca724985cccc1e37113a193c Mon Sep 17 00:00:00 2001 From: John Starich Date: Mon, 18 Apr 2022 21:32:00 -0500 Subject: [PATCH 18/37] Fix wasm boot crash due to synchronous spawn --- internal/jsworker/message_port.go | 2 -- internal/worker/remote.go | 31 +++++++++++++++++-------------- 2 files changed, 17 insertions(+), 16 deletions(-) diff --git a/internal/jsworker/message_port.go b/internal/jsworker/message_port.go index 73fc69b..5f2fb0a 100644 --- a/internal/jsworker/message_port.go +++ b/internal/jsworker/message_port.go @@ -8,7 +8,6 @@ import ( "github.com/hack-pad/hackpad/internal/common" "github.com/hack-pad/hackpad/internal/interop" "github.com/hack-pad/hackpad/internal/jsfunc" - "github.com/hack-pad/hackpad/internal/log" ) type MessagePort struct { @@ -38,7 +37,6 @@ func wrapMessagePort(v js.Value) (*MessagePort, error) { func (p *MessagePort) PostMessage(message js.Value, transfers []js.Value) (err error) { defer common.CatchException(&err) args := append([]interface{}{message}, interop.SliceFromJSValues(transfers)...) - log.PrintJSValues("Post message:", message) p.jsMessagePort.Call("postMessage", args...) return nil } diff --git a/internal/worker/remote.go b/internal/worker/remote.go index 397df4c..d4cec9a 100644 --- a/internal/worker/remote.go +++ b/internal/worker/remote.go @@ -45,16 +45,6 @@ func NewRemote(local *Local, pid process.PID, command string, argv []string, att if err != nil { return nil, err } - err = awaitMessage(ctx, port, "pending_init") - if err != nil { - return nil, err - } - log.Warn("Sending init to worker ", workerName) - err = port.PostMessage(makeInitMessage(command, argv, attr.Dir, attr.Env), nil) - if err != nil { - return nil, err - } - log.Warn("init sent to worker ", workerName) closeCtx, cancel := context.WithCancel(ctx) remote := &Remote{ @@ -83,10 +73,23 @@ func NewRemote(local *Local, pid process.PID, command string, argv []string, att return nil, err } - err = remote.port.PostMessage(makeStartMessage(), nil) - if err != nil { - return nil, err - } + go func() { + err := awaitMessage(ctx, port, "pending_init") + if err != nil { + log.Error("Failed awaiting pending_init:", workerName, err) + return + } + err = port.PostMessage(makeInitMessage(command, argv, attr.Dir, attr.Env), nil) + if err != nil { + log.Error("Failed sending init to worker:", workerName, err) + return + } + err = remote.port.PostMessage(makeStartMessage(), nil) + if err != nil { + log.Error("Failed sending start to worker:", workerName, err) + return + } + }() return remote, nil } From 99aad3126bf54cd10dfe7586a4127f6be52d4794 Mon Sep 17 00:00:00 2001 From: John Starich Date: Mon, 18 Apr 2022 21:54:48 -0500 Subject: [PATCH 19/37] Fix FS not fully initialized at process start --- cmd/worker/main.go | 16 +++++++++++----- internal/worker/dom.go | 3 +++ internal/worker/local.go | 9 ++------- 3 files changed, 16 insertions(+), 12 deletions(-) diff --git a/cmd/worker/main.go b/cmd/worker/main.go index 34dcf89..96cf23b 100644 --- a/cmd/worker/main.go +++ b/cmd/worker/main.go @@ -23,20 +23,23 @@ func main() { bootCtx := context.Background() //bootCtx, bootCancel := context.WithTimeout(context.Background(), 30*time.Second) //defer bootCancel() - log.Print("booting worker") + log.Debug("booting worker") local, err := worker.NewLocal(bootCtx, jsworker.GetLocal()) if err != nil { panic(err) } - log.Print("worker started") - + log.Debug("worker inited") if err := setUpFS(); err != nil { panic(err) } - + log.Debug("fs is setup") + if err := local.Start(); err != nil { + panic(err) + } + log.Debug("worker starting...") <-local.Started() pid := local.PID() - log.Print("worker process started PID ", pid) + log.Debug("worker process started PID ", pid) exitCode, err := local.Wait(pid) if err != nil { log.Error("Failed to wait for PID ", pid, ":", err) @@ -55,6 +58,9 @@ func setUpFS() error { if err := overlayIndexedDB("/bin", idb.DurabilityRelaxed); err != nil { return err } + if err := os.MkdirAll("/home/me", dirPerm); err != nil { + return err + } if err := overlayIndexedDB("/home/me", idb.DurabilityDefault); err != nil { return err } diff --git a/internal/worker/dom.go b/internal/worker/dom.go index 78a3137..d28b4cb 100644 --- a/internal/worker/dom.go +++ b/internal/worker/dom.go @@ -21,6 +21,9 @@ func ExecDOM(ctx context.Context, localJS *jsworker.Local, command string, args if err != nil { return nil, err } + if err := local.Start(); err != nil { + return nil, err + } return &DOM{ local: local, port: localJS, diff --git a/internal/worker/local.go b/internal/worker/local.go index 8076069..396c1c3 100644 --- a/internal/worker/local.go +++ b/internal/worker/local.go @@ -51,12 +51,6 @@ func NewLocal(ctx context.Context, localJS *jsworker.Local) (_ *Local, err error log.Debug("before ready post") localJS.PostMessage(js.ValueOf("ready"), nil) log.Debug("after ready post") - - err = local.listenStart() - if err != nil { - return nil, err - } - return local, nil } @@ -95,7 +89,8 @@ func (l *Local) awaitInit(ctx context.Context) (js.Value, error) { return message.init, message.err } -func (l *Local) listenStart() error { +func (l *Local) Start() (err error) { + defer common.CatchException(&err) startCtx, cancel := context.WithCancel(context.Background()) return l.localJS.Listen(startCtx, func(me jsworker.MessageEvent, err error) { if err != nil { From 8fef316db34053d6613b1358f59a3ded2ac1b029 Mon Sep 17 00:00:00 2001 From: John Starich Date: Mon, 18 Apr 2022 22:15:29 -0500 Subject: [PATCH 20/37] Add worker name to logs --- internal/log/js_log.go | 19 ++++++++++++++++--- internal/worker/dom.go | 2 +- internal/worker/local.go | 1 + internal/worker/remote.go | 5 +++-- 4 files changed, 21 insertions(+), 6 deletions(-) diff --git a/internal/log/js_log.go b/internal/log/js_log.go index 2b49fc2..779bfce 100644 --- a/internal/log/js_log.go +++ b/internal/log/js_log.go @@ -54,11 +54,24 @@ func logJSValues(kind consoleType, skip int, args ...interface{}) int { return 0 } caller := getCaller(skip + 1) - args = append([]interface{}{caller}, args...) - console.Call(kind.String(), args...) + var newArgs []interface{} + if name := workerNamePrefix(); name != "" { + newArgs = append(newArgs, name) + } + newArgs = append(newArgs, caller) + newArgs = append(newArgs, args...) + console.Call(kind.String(), newArgs...) return 0 } func writeLog(c consoleType, s string) { - console.Call(c.String(), s) + console.Call(c.String(), workerNamePrefix(), s) +} + +func workerNamePrefix() string { + name := global.Get("workerName") + if name.Type() == js.TypeString { + return name.String() + ": " + } + return "" } diff --git a/internal/worker/dom.go b/internal/worker/dom.go index d28b4cb..63a0f85 100644 --- a/internal/worker/dom.go +++ b/internal/worker/dom.go @@ -13,7 +13,7 @@ type DOM struct { } func ExecDOM(ctx context.Context, localJS *jsworker.Local, command string, args []string, workingDirectory string, env map[string]string) (*DOM, error) { - err := localJS.PostMessage(makeInitMessage(command, append([]string{command}, args...), workingDirectory, env), nil) + err := localJS.PostMessage(makeInitMessage("dom", command, append([]string{command}, args...), workingDirectory, env), nil) if err != nil { return nil, err } diff --git a/internal/worker/local.go b/internal/worker/local.go index 396c1c3..0ff0284 100644 --- a/internal/worker/local.go +++ b/internal/worker/local.go @@ -47,6 +47,7 @@ func NewLocal(ctx context.Context, localJS *jsworker.Local) (_ *Local, err error } jsprocess.Init(local.process, local, local) jsfs.Init(local.process) + global.Set("workerName", init.Get("workerName")) global.Set("ready", true) log.Debug("before ready post") localJS.PostMessage(js.ValueOf("ready"), nil) diff --git a/internal/worker/remote.go b/internal/worker/remote.go index d4cec9a..954b0f4 100644 --- a/internal/worker/remote.go +++ b/internal/worker/remote.go @@ -79,7 +79,7 @@ func NewRemote(local *Local, pid process.PID, command string, argv []string, att log.Error("Failed awaiting pending_init:", workerName, err) return } - err = port.PostMessage(makeInitMessage(command, argv, attr.Dir, attr.Env), nil) + err = port.PostMessage(makeInitMessage(workerName, command, argv, attr.Dir, attr.Env), nil) if err != nil { log.Error("Failed sending init to worker:", workerName, err) return @@ -122,9 +122,10 @@ func awaitMessage(ctx context.Context, port *jsworker.Remote, messageStr string) } } -func makeInitMessage(command string, argv []string, workingDirectory string, env map[string]string) js.Value { +func makeInitMessage(workerName, command string, argv []string, workingDirectory string, env map[string]string) js.Value { return js.ValueOf(map[string]interface{}{ "init": map[string]interface{}{ + "workerName": workerName, "command": command, "argv": interop.SliceFromStrings(argv), "workingDirectory": workingDirectory, From 1c7753c85b0b0e8bdd1cde12b68268427b6df83b Mon Sep 17 00:00:00 2001 From: John Starich Date: Mon, 18 Apr 2022 22:32:23 -0500 Subject: [PATCH 21/37] Allow waiting on self --- internal/worker/local.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/internal/worker/local.go b/internal/worker/local.go index 0ff0284..007f4b6 100644 --- a/internal/worker/local.go +++ b/internal/worker/local.go @@ -143,6 +143,9 @@ func (l *Local) Spawn(command string, argv []string, attr *process.ProcAttr) (js func (l *Local) Wait(pid common.PID) (exitCode int, err error) { log.Print("Waiting on pid ", pid) + if pid == l.process.PID() { + return l.process.Wait() + } remote, ok := l.pids[pid] if !ok { return 0, errors.Errorf("Unknown child process: %d", pid) From 341e8308d9b9fb91117153fcae5ae05e44dbe160 Mon Sep 17 00:00:00 2001 From: John Starich Date: Thu, 16 Jun 2022 23:08:27 -0500 Subject: [PATCH 22/37] WIP: Fix MessagePort transfers, process start works now --- cmd/editor/main.go | 2 + cmd/worker/main.go | 4 +- internal/common/fid.go | 2 + internal/fs/device_file.go | 33 +++++++++++ internal/fs/file_descriptors.go | 26 ++++++--- internal/fs/null_file.go | 14 +---- internal/fs/stdout.go | 19 ++++++ internal/jsworker/local.go | 2 +- internal/jsworker/message_event.go | 2 +- internal/jsworker/message_port.go | 15 +++-- internal/jsworker/remote.go | 2 +- internal/worker/dom.go | 13 ++++- internal/worker/local.go | 56 +++++++++++++++--- internal/worker/open_file.go | 51 ++++++++++++++++ internal/worker/pipe.go | 83 ++++++++++++++++++++++++++ internal/worker/remote.go | 93 +++++++++++++++++++++++++----- main.go | 1 + 17 files changed, 363 insertions(+), 55 deletions(-) create mode 100644 internal/fs/device_file.go create mode 100644 internal/worker/open_file.go create mode 100644 internal/worker/pipe.go diff --git a/cmd/editor/main.go b/cmd/editor/main.go index 86979d2..4527443 100644 --- a/cmd/editor/main.go +++ b/cmd/editor/main.go @@ -61,6 +61,8 @@ func main() { return } + select {} // TODO: remove + if err := os.MkdirAll("playground", 0700); err != nil { log.Error("Failed to make playground dir", err) return diff --git a/cmd/worker/main.go b/cmd/worker/main.go index 96cf23b..ef2a4fa 100644 --- a/cmd/worker/main.go +++ b/cmd/worker/main.go @@ -3,6 +3,7 @@ package main import ( "context" "os" + "runtime/debug" "github.com/hack-pad/go-indexeddb/idb" "github.com/hack-pad/hackpad/internal/common" @@ -15,7 +16,8 @@ import ( func main() { defer common.CatchExceptionHandler(func(err error) { - log.Error("Worker panicked:", err) + log.Errorf("Worker panicked: %+v", err) + log.Error(string(debug.Stack())) os.Exit(1) }) diff --git a/internal/common/fid.go b/internal/common/fid.go index bb51874..0d57972 100644 --- a/internal/common/fid.go +++ b/internal/common/fid.go @@ -2,6 +2,7 @@ package common import ( "fmt" + "io" "github.com/hack-pad/hackpadfs" ) @@ -20,4 +21,5 @@ type OpenFileAttr struct { SeekOffset int64 Flags int Mode hackpadfs.FileMode + RawDevice io.ReadWriteCloser } diff --git a/internal/fs/device_file.go b/internal/fs/device_file.go new file mode 100644 index 0000000..c11e096 --- /dev/null +++ b/internal/fs/device_file.go @@ -0,0 +1,33 @@ +package fs + +import ( + "io" + + "github.com/hack-pad/hackpadfs" +) + +type deviceFile struct { + name string + rawDevice io.ReadWriteCloser +} + +var _ hackpadfs.File = &deviceFile{} + +func newDeviceFile(name string, rawDevice io.ReadWriteCloser) *deviceFile { + return &deviceFile{ + name: name, + rawDevice: rawDevice, + } +} + +func (d *deviceFile) Read(p []byte) (n int, err error) { + return d.Read(p) +} + +func (d *deviceFile) Close() error { + return d.rawDevice.Close() +} + +func (d *deviceFile) Stat() (hackpadfs.FileInfo, error) { + return newNamedFileInfo(d.name), nil +} diff --git a/internal/fs/file_descriptors.go b/internal/fs/file_descriptors.go index 21e8bb0..75c5da2 100644 --- a/internal/fs/file_descriptors.go +++ b/internal/fs/file_descriptors.go @@ -14,6 +14,7 @@ import ( "github.com/hack-pad/hackpad/internal/common" "github.com/hack-pad/hackpad/internal/interop" "github.com/hack-pad/hackpad/internal/jserror" + "github.com/hack-pad/hackpad/internal/log" "github.com/hack-pad/hackpadfs" "github.com/pkg/errors" ) @@ -64,6 +65,7 @@ func NewFileDescriptors(parentPID common.PID, workingDirectory string, openFiles var files []openFile defer func() { if returnedErr != nil { + returnedErr = errors.WithStack(returnedErr) for _, f := range files { f.file.Close() } @@ -92,14 +94,19 @@ func NewFileDescriptors(parentPID common.PID, workingDirectory string, openFiles return nil, nil, errors.Errorf("Invalid number of inherited file descriptors, must be 0 or at least 3: %#v", openFiles) default: for _, attr := range openFiles { - file, err := getFile(attr.FilePath, attr.Flags, attr.Mode) - if err != nil { - return nil, nil, err - } - files = append(files, openFile{attr, file}) - _, err = hackpadfs.SeekFile(file, attr.SeekOffset, io.SeekStart) - if err != nil { - return nil, nil, err + if attr.RawDevice == nil { + file, err := getFile(attr.FilePath, attr.Flags, attr.Mode) + if err != nil { + return nil, nil, err + } + _, err = hackpadfs.SeekFile(file, attr.SeekOffset, io.SeekStart) + if err != nil { + return nil, nil, err + } + files = append(files, openFile{attr, file}) + } else { + file := newDeviceFile("", attr.RawDevice) + files = append(files, openFile{attr, file}) } } } @@ -168,6 +175,7 @@ func getFile(absPath string, flags int, mode os.FileMode) (hackpadfs.File, error case "dev/stderr": return stderr, nil } + log.Debugf("Opening: %q %v %v", absPath, flags, mode) return hackpadfs.OpenFile(filesystem, absPath, flags, mode) } @@ -337,7 +345,7 @@ func (f *FileDescriptors) Flock(fd FID, action LockAction) error { return nil } -func (f *FileDescriptors) RawFID(fid FID) (io.Reader, error) { +func (f *FileDescriptors) RawFID(fid FID) (hackpadfs.File, error) { if _, ok := f.files[fid]; !ok { return nil, interop.BadFileNumber(fid) } diff --git a/internal/fs/null_file.go b/internal/fs/null_file.go index bab7f53..884275c 100644 --- a/internal/fs/null_file.go +++ b/internal/fs/null_file.go @@ -3,7 +3,6 @@ package fs import ( "io" "os" - "time" "github.com/hack-pad/hackpadfs" ) @@ -22,16 +21,5 @@ func (f nullFile) ReadAt(p []byte, off int64) (n int, err error) { return 0, io func (f nullFile) Seek(offset int64, whence int) (int64, error) { return 0, nil } func (f nullFile) Write(p []byte) (n int, err error) { return len(p), nil } func (f nullFile) WriteAt(p []byte, off int64) (n int, err error) { return len(p), nil } -func (f nullFile) Stat() (os.FileInfo, error) { return nullStat{f}, nil } +func (f nullFile) Stat() (os.FileInfo, error) { return namedFileInfo{f.name}, nil } func (f nullFile) Truncate(size int64) error { return nil } - -type nullStat struct { - f nullFile -} - -func (s nullStat) Name() string { return s.f.name } -func (s nullStat) Size() int64 { return 0 } -func (s nullStat) Mode() os.FileMode { return 0 } -func (s nullStat) ModTime() time.Time { return time.Time{} } -func (s nullStat) IsDir() bool { return false } -func (s nullStat) Sys() interface{} { return nil } diff --git a/internal/fs/stdout.go b/internal/fs/stdout.go index 41a0595..b6f3682 100644 --- a/internal/fs/stdout.go +++ b/internal/fs/stdout.go @@ -79,3 +79,22 @@ func (b *bufferedLogger) Close() error { // TODO prevent writes and return os.ErrClosed return nil } + +func (b *bufferedLogger) Stat() (hackpadfs.FileInfo, error) { + return namedFileInfo{b.name}, nil +} + +type namedFileInfo struct { + name string +} + +func newNamedFileInfo(name string) hackpadfs.FileInfo { + return namedFileInfo{name: name} +} + +func (i namedFileInfo) Name() string { return i.name } +func (i namedFileInfo) Size() int64 { return 0 } +func (i namedFileInfo) Mode() hackpadfs.FileMode { return 0 } +func (i namedFileInfo) ModTime() time.Time { return time.Time{} } +func (i namedFileInfo) IsDir() bool { return false } +func (i namedFileInfo) Sys() interface{} { return nil } diff --git a/internal/jsworker/local.go b/internal/jsworker/local.go index ad2fc0a..d515775 100644 --- a/internal/jsworker/local.go +++ b/internal/jsworker/local.go @@ -16,7 +16,7 @@ func init() { if !jsSelf.Truthy() { return } - port, err := wrapMessagePort(jsSelf) + port, err := WrapMessagePort(jsSelf) if err != nil { panic(err) } diff --git a/internal/jsworker/message_event.go b/internal/jsworker/message_event.go index 6d328b3..c0897c5 100644 --- a/internal/jsworker/message_event.go +++ b/internal/jsworker/message_event.go @@ -14,7 +14,7 @@ type MessageEvent struct { func parseMessageEvent(v js.Value) (_ MessageEvent, err error) { defer common.CatchException(&err) - target, err := wrapMessagePort(v.Get("target")) + target, err := WrapMessagePort(v.Get("target")) return MessageEvent{ Data: v.Get("data"), Target: target, diff --git a/internal/jsworker/message_port.go b/internal/jsworker/message_port.go index 5f2fb0a..96bbf84 100644 --- a/internal/jsworker/message_port.go +++ b/internal/jsworker/message_port.go @@ -19,15 +19,15 @@ var jsMessageChannel = js.Global().Get("MessageChannel") func NewChannel() (port1, port2 *MessagePort, err error) { defer common.CatchException(&err) channel := jsMessageChannel.New() - port1, err = wrapMessagePort(channel.Get("port1")) + port1, err = WrapMessagePort(channel.Get("port1")) if err != nil { return } - port2, err = wrapMessagePort(channel.Get("port2")) + port2, err = WrapMessagePort(channel.Get("port2")) return } -func wrapMessagePort(v js.Value) (*MessagePort, error) { +func WrapMessagePort(v js.Value) (*MessagePort, error) { if !v.Get("postMessage").Truthy() { return nil, errors.New("invalid MessagePort value: postMessage is not a function") } @@ -36,7 +36,7 @@ func wrapMessagePort(v js.Value) (*MessagePort, error) { func (p *MessagePort) PostMessage(message js.Value, transfers []js.Value) (err error) { defer common.CatchException(&err) - args := append([]interface{}{message}, interop.SliceFromJSValues(transfers)...) + args := append([]interface{}{message}, interop.SliceFromJSValues(transfers)) p.jsMessagePort.Call("postMessage", args...) return nil } @@ -82,3 +82,10 @@ func (p *MessagePort) Close() (err error) { p.jsMessagePort.Call("close") return nil } + +func (p *MessagePort) JSValue() js.Value { + if p == nil { + return js.Null() + } + return p.jsMessagePort +} diff --git a/internal/jsworker/remote.go b/internal/jsworker/remote.go index 4ea19f5..7c15937 100644 --- a/internal/jsworker/remote.go +++ b/internal/jsworker/remote.go @@ -25,7 +25,7 @@ func NewRemote(name, url string) (_ *Remote, err error) { val := jsWorker.New(url, map[string]interface{}{ "name": name, }) - port, err := wrapMessagePort(val) + port, err := WrapMessagePort(val) return &Remote{ port: port, worker: val, diff --git a/internal/worker/dom.go b/internal/worker/dom.go index 63a0f85..4cb11c4 100644 --- a/internal/worker/dom.go +++ b/internal/worker/dom.go @@ -5,6 +5,7 @@ import ( "syscall/js" "github.com/hack-pad/hackpad/internal/jsworker" + "github.com/hack-pad/hackpad/internal/log" ) type DOM struct { @@ -13,17 +14,27 @@ type DOM struct { } func ExecDOM(ctx context.Context, localJS *jsworker.Local, command string, args []string, workingDirectory string, env map[string]string) (*DOM, error) { - err := localJS.PostMessage(makeInitMessage("dom", command, append([]string{command}, args...), workingDirectory, env), nil) + msg, transfers := makeInitMessage( + "dom", + command, append([]string{command}, args...), + workingDirectory, + env, + nil, + ) + err := localJS.PostMessage(msg, transfers) if err != nil { return nil, err } + log.Print("NewLocal start") local, err := NewLocal(ctx, localJS) if err != nil { return nil, err } + log.Print("local start") if err := local.Start(); err != nil { return nil, err } + log.Print("local ready") return &DOM{ local: local, port: localJS, diff --git a/internal/worker/local.go b/internal/worker/local.go index 007f4b6..38968d5 100644 --- a/internal/worker/local.go +++ b/internal/worker/local.go @@ -2,6 +2,7 @@ package worker import ( "context" + "io" "syscall/js" "github.com/hack-pad/hackpad/internal/common" @@ -32,30 +33,30 @@ func NewLocal(ctx context.Context, localJS *jsworker.Local) (_ *Local, err error if err != nil { return nil, err } - defer common.CatchException(&err) + + global.Set("workerName", init.Get("workerName")) + log.Debug("Setting process details...") local.process, err = process.New( kernel.ReservePID(), init.Get("command").String(), interop.StringsFromJSValue(init.Get("argv")), init.Get("workingDirectory").String(), - nil, // TODO open files + parseOpenFiles(init.Get("openFiles")), interop.StringMapFromJSObject(init.Get("env")), ) if err != nil { return nil, err } + log.Debug("Initializing process") jsprocess.Init(local.process, local, local) + log.Debug("Initializing fs") jsfs.Init(local.process) - global.Set("workerName", init.Get("workerName")) - global.Set("ready", true) - log.Debug("before ready post") - localJS.PostMessage(js.ValueOf("ready"), nil) - log.Debug("after ready post") return local, nil } func (l *Local) awaitInit(ctx context.Context) (js.Value, error) { + log.Debug("NewLocal 1") ctx, cancel := context.WithCancel(ctx) defer cancel() l.processStartCtx = ctx @@ -66,6 +67,7 @@ func (l *Local) awaitInit(ctx context.Context) (js.Value, error) { } initChan := make(chan initMessage, 1) err := l.localJS.Listen(ctx, func(me jsworker.MessageEvent, err error) { + log.Print("awaitInit listener:", err) if err != nil { initChan <- initMessage{err: err} return @@ -86,14 +88,17 @@ func (l *Local) awaitInit(ctx context.Context) (js.Value, error) { if err != nil { return js.Value{}, err } + log.Debug("NewLocal 2") message := <-initChan + log.Debug("NewLocal 3") return message.init, message.err } func (l *Local) Start() (err error) { defer common.CatchException(&err) startCtx, cancel := context.WithCancel(context.Background()) - return l.localJS.Listen(startCtx, func(me jsworker.MessageEvent, err error) { + log.Print("Listening for start...") + err = l.localJS.Listen(startCtx, func(me jsworker.MessageEvent, err error) { if err != nil { log.Error(err) cancel() @@ -103,6 +108,7 @@ func (l *Local) Start() (err error) { log.Error(err) cancel() }) + log.Print("Local's Start() received message") if me.Data.Type() != js.TypeObject { return } @@ -120,6 +126,18 @@ func (l *Local) Start() (err error) { return } }) + if err != nil { + return err + } + + global.Set("ready", true) + log.Debug("before ready post") + err = l.localJS.PostMessage(js.ValueOf("ready"), nil) + if err != nil { + return err + } + log.Debug("after ready post") + return nil } func (l *Local) Exit(exitCode int) error { @@ -166,3 +184,25 @@ func makeExitMessage(exitCode int) js.Value { "exitCode": exitCode, }) } + +func parseOpenFiles(v js.Value) []common.OpenFileAttr { + openFileJSValues := interop.SliceFromJSValue(v) + var openFiles []common.OpenFileAttr + for _, o := range openFileJSValues { + openFile := readOpenFile(o) + var pipe io.ReadWriteCloser + if openFile.pipe != nil { + var err error + pipe, err = portToReadWriteCloser(openFile.pipe) + if err != nil { + panic(err) + } + } + openFiles = append(openFiles, common.OpenFileAttr{ + FilePath: openFile.filePath, + SeekOffset: openFile.seekOffset, + RawDevice: pipe, + }) + } + return openFiles +} diff --git a/internal/worker/open_file.go b/internal/worker/open_file.go new file mode 100644 index 0000000..5c74bf9 --- /dev/null +++ b/internal/worker/open_file.go @@ -0,0 +1,51 @@ +package worker + +import ( + "syscall/js" + + "github.com/hack-pad/hackpad/internal/interop" + "github.com/hack-pad/hackpad/internal/jsworker" +) + +const ( + ofFilePath = "filePath" + ofSeekOffset = "seekOffset" + ofPipe = "pipe" +) + +type openFile struct { + filePath string + seekOffset int64 + pipe *jsworker.MessagePort +} + +func readOpenFile(v js.Value) openFile { + props := interop.Entries(v) + return openFile{ + filePath: optionalString(props[ofFilePath]), + seekOffset: optionalInt64(props[ofSeekOffset]), + pipe: optionalPipe(props[ofPipe]), + } +} + +func (o openFile) JSValue() js.Value { + return js.ValueOf(map[string]interface{}{ + ofFilePath: o.filePath, + ofSeekOffset: o.seekOffset, + ofPipe: o.pipe.JSValue(), + }) +} + +func optionalString(v js.Value) string { + if v.Type() != js.TypeString { + return "" + } + return v.String() +} + +func optionalInt64(v js.Value) int64 { + if v.Type() != js.TypeNumber { + return 0 + } + return int64(v.Int()) +} diff --git a/internal/worker/pipe.go b/internal/worker/pipe.go new file mode 100644 index 0000000..945caf7 --- /dev/null +++ b/internal/worker/pipe.go @@ -0,0 +1,83 @@ +package worker + +import ( + "context" + "io" + "syscall/js" + + "github.com/hack-pad/hackpad/internal/jsworker" + "github.com/hack-pad/hackpadfs/indexeddb/idbblob" + "github.com/hack-pad/hackpadfs/keyvalue/blob" +) + +func optionalPipe(v js.Value) *jsworker.MessagePort { + if v.Type() != js.TypeObject { + return nil + } + port, err := jsworker.WrapMessagePort(v) + if err != nil { + panic(err) + } + return port +} + +type portPipe struct { + port *jsworker.MessagePort + receivedData <-chan portPipeMessage + remainingReadData []byte + cancel context.CancelFunc +} + +type portPipeMessage struct { + Data []byte + Err error +} + +func portToReadWriteCloser(port *jsworker.MessagePort) (io.ReadWriteCloser, error) { + ctx, cancel := context.WithCancel(context.Background()) + receivedData := make(chan portPipeMessage) + err := port.Listen(ctx, func(me jsworker.MessageEvent, err error) { + var buf []byte + if err == nil { + var bl blob.Blob + bl, err = idbblob.New(me.Data) + if err == nil { + buf = bl.Bytes() + } + } + receivedData <- portPipeMessage{Data: buf, Err: err} + }) + if err != nil { + return nil, err + } + return &portPipe{ + port: port, + receivedData: receivedData, + cancel: cancel, + }, nil +} + +func (p *portPipe) Close() error { + p.cancel() + return p.port.Close() +} + +func (p *portPipe) Read(b []byte) (n int, err error) { + if len(p.remainingReadData) == 0 { + message := <-p.receivedData + p.remainingReadData = message.Data + err = message.Err + } + n = copy(b, p.remainingReadData) + p.remainingReadData = p.remainingReadData[n:] + return +} + +func (p *portPipe) Write(b []byte) (n int, err error) { + bl := idbblob.FromBlob(blob.NewBytes(b)).JSValue() + err = p.port.PostMessage(bl, []js.Value{bl}) + if err == nil { + n = len(b) + } + return +} diff --git a/internal/worker/remote.go b/internal/worker/remote.go index 954b0f4..2b98897 100644 --- a/internal/worker/remote.go +++ b/internal/worker/remote.go @@ -10,6 +10,8 @@ import ( "github.com/hack-pad/hackpad/internal/jsworker" "github.com/hack-pad/hackpad/internal/log" "github.com/hack-pad/hackpad/internal/process" + "github.com/hack-pad/hackpadfs" + "github.com/hack-pad/hackpadfs/indexeddb/idbblob" ) type Remote struct { @@ -20,33 +22,46 @@ type Remote struct { closeErr error } -type openFile struct { - filePath string - seekOffset uint -} - func NewRemote(local *Local, pid process.PID, command string, argv []string, attr *process.ProcAttr) (*Remote, error) { ctx := context.Background() + closeCtx, cancel := context.WithCancel(ctx) var openFiles []openFile for _, f := range attr.Files { - info, err := local.process.Files().Fstat(f.FID) + file, err := local.process.Files().RawFID(f.FID) if err != nil { return nil, err } - openFiles = append(openFiles, openFile{ + info, err := file.Stat() + if err != nil { + return nil, err + } + openF := openFile{ filePath: info.Name(), seekOffset: 0, // TODO expose seek offset in file descriptor - }) + } + if info.Mode()&hackpadfs.ModeNamedPipe != 0 { + log.Print("Found pipe, creating MessageChannel...") + port1, port2, err := jsworker.NewChannel() + if err != nil { + return nil, err + } + openF.pipe = port1 + log.Print("Connecting port to file...") + err = connectPortToFile(closeCtx, port2, file) + if err != nil { + return nil, err + } + log.Print("Connected port to file.") + } + openFiles = append(openFiles, openF) } - // TODO inherit file descriptors workerName := fmt.Sprintf("pid-%d", pid) port, err := jsworker.NewRemoteWasm(workerName, "/wasm/worker.wasm") if err != nil { return nil, err } - closeCtx, cancel := context.WithCancel(ctx) remote := &Remote{ pid: pid, port: port, @@ -66,29 +81,40 @@ func NewRemote(local *Local, pid process.PID, command string, argv []string, att if jsExitCode, ok := data["exitCode"]; ok && jsExitCode.Type() == js.TypeNumber { exitCode := jsExitCode.Int() remote.closeExitCode = &exitCode + cancel() + log.Warn("Remote exited with code:", exitCode) } - cancel() }) if err != nil { return nil, err } go func() { + log.Print("Worker ", workerName, " awaiting pending_init...") err := awaitMessage(ctx, port, "pending_init") if err != nil { log.Error("Failed awaiting pending_init:", workerName, err) return } - err = port.PostMessage(makeInitMessage(workerName, command, argv, attr.Dir, attr.Env), nil) + log.Print("Worker ", workerName, " waiting to init. Sending init...") + msg, transfers := makeInitMessage(workerName, command, argv, attr.Dir, attr.Env, openFiles) + err = port.PostMessage(msg, transfers) if err != nil { - log.Error("Failed sending init to worker:", workerName, err) + log.Error("Failed sending init to worker: ", workerName, " ", err) return } + log.Print("Sent init message to worker ", workerName, ". Awaiting ready...") + if err := awaitMessage(ctx, remote.port, "ready"); err != nil { + log.Error("Failed awaiting ready:", workerName, err) + return + } + log.Print("Worker ", workerName, " is ready. Sending start message.") err = remote.port.PostMessage(makeStartMessage(), nil) if err != nil { - log.Error("Failed sending start to worker:", workerName, err) + log.Error("Failed sending start to worker: ", workerName, " ", err) return } + log.Print("Sent start message.") }() return remote, nil @@ -122,7 +148,21 @@ func awaitMessage(ctx context.Context, port *jsworker.Remote, messageStr string) } } -func makeInitMessage(workerName, command string, argv []string, workingDirectory string, env map[string]string) js.Value { +func makeInitMessage( + workerName, + command string, argv []string, + workingDirectory string, + env map[string]string, + openFiles []openFile, +) (msg js.Value, transfers []js.Value) { + var openFileJSValues []interface{} + var ports []js.Value + for _, o := range openFiles { + openFileJSValues = append(openFileJSValues, o) + if o.pipe != nil { + ports = append(ports, o.pipe.JSValue()) + } + } return js.ValueOf(map[string]interface{}{ "init": map[string]interface{}{ "workerName": workerName, @@ -130,8 +170,9 @@ func makeInitMessage(workerName, command string, argv []string, workingDirectory "argv": interop.SliceFromStrings(argv), "workingDirectory": workingDirectory, "env": interop.StringMap(env), + "openFiles": openFileJSValues, }, - }) + }), ports } func (r *Remote) Wait() (exitCode int, err error) { @@ -146,3 +187,23 @@ func (r *Remote) Wait() (exitCode int, err error) { } return *r.closeExitCode, r.closeErr } + +func connectPortToFile(ctx context.Context, port *jsworker.MessagePort, file hackpadfs.File) error { + return port.Listen(ctx, func(me jsworker.MessageEvent, err error) { + if err != nil { + log.Error(err) + return + } + bl, err := idbblob.New(me.Data) + if err != nil { + log.Error(err) + return + } + log.Print("Received message! ", bl.Len()) + _, err = hackpadfs.WriteFile(file, bl.Bytes()) + if err != nil { + log.Error(err) + return + } + }) +} diff --git a/main.go b/main.go index f6c92ad..52ce63d 100644 --- a/main.go +++ b/main.go @@ -69,6 +69,7 @@ func main() { if err := dom.Start(); err != nil { panic(err) } + log.Print("DOM started") select {} } From e62332dace1c588813196047fbc6db270826ae8a Mon Sep 17 00:00:00 2001 From: John Starich Date: Thu, 16 Jun 2022 23:41:54 -0500 Subject: [PATCH 23/37] WIP: Successfully sent stdout to dom process, print in log --- internal/fs/device_file.go | 9 ++++++++- internal/fs/fs.go | 3 ++- internal/jsworker/message_port.go | 9 +++++++-- internal/worker/pipe.go | 2 +- internal/worker/remote.go | 2 +- 5 files changed, 19 insertions(+), 6 deletions(-) diff --git a/internal/fs/device_file.go b/internal/fs/device_file.go index c11e096..ee95594 100644 --- a/internal/fs/device_file.go +++ b/internal/fs/device_file.go @@ -4,6 +4,7 @@ import ( "io" "github.com/hack-pad/hackpadfs" + "github.com/pkg/errors" ) type deviceFile struct { @@ -21,7 +22,13 @@ func newDeviceFile(name string, rawDevice io.ReadWriteCloser) *deviceFile { } func (d *deviceFile) Read(p []byte) (n int, err error) { - return d.Read(p) + n, err = d.rawDevice.Read(p) + return n, errors.WithStack(err) +} + +func (d *deviceFile) Write(p []byte) (n int, err error) { + n, err = d.rawDevice.Write(p) + return n, errors.WithStack(err) } func (d *deviceFile) Close() error { diff --git a/internal/fs/fs.go b/internal/fs/fs.go index 2decb1c..6a5665e 100644 --- a/internal/fs/fs.go +++ b/internal/fs/fs.go @@ -61,6 +61,7 @@ func OverlayTarGzip(mountPath string, gzipReader io.ReadCloser, persist bool, sh return err } + originalMountPath := mountPath mountPath = common.ResolvePath(".", mountPath) if !persist { underlyingFS, err := mem.NewFS() @@ -78,7 +79,7 @@ func OverlayTarGzip(mountPath string, gzipReader io.ReadCloser, persist bool, sh const tarfsDoneMarker = ".tarfs-complete" - underlyingFS, err := newPersistDB(mountPath, true, shouldCache) + underlyingFS, err := newPersistDB(originalMountPath, true, shouldCache) if err != nil { return err } diff --git a/internal/jsworker/message_port.go b/internal/jsworker/message_port.go index 96bbf84..0b8c7c4 100644 --- a/internal/jsworker/message_port.go +++ b/internal/jsworker/message_port.go @@ -2,12 +2,14 @@ package jsworker import ( "context" - "errors" + "runtime/debug" "syscall/js" "github.com/hack-pad/hackpad/internal/common" "github.com/hack-pad/hackpad/internal/interop" "github.com/hack-pad/hackpad/internal/jsfunc" + "github.com/hack-pad/hackpad/internal/log" + "github.com/pkg/errors" ) type MessagePort struct { @@ -35,7 +37,10 @@ func WrapMessagePort(v js.Value) (*MessagePort, error) { } func (p *MessagePort) PostMessage(message js.Value, transfers []js.Value) (err error) { - defer common.CatchException(&err) + defer common.CatchExceptionHandler(func(e error) { + err = e + log.Error(err, ": ", string(debug.Stack())) + }) args := append([]interface{}{message}, interop.SliceFromJSValues(transfers)) p.jsMessagePort.Call("postMessage", args...) return nil diff --git a/internal/worker/pipe.go b/internal/worker/pipe.go index 945caf7..c84e42c 100644 --- a/internal/worker/pipe.go +++ b/internal/worker/pipe.go @@ -75,7 +75,7 @@ func (p *portPipe) Read(b []byte) (n int, err error) { func (p *portPipe) Write(b []byte) (n int, err error) { bl := idbblob.FromBlob(blob.NewBytes(b)).JSValue() - err = p.port.PostMessage(bl, []js.Value{bl}) + err = p.port.PostMessage(bl, []js.Value{bl.Get("buffer")}) if err == nil { n = len(b) } diff --git a/internal/worker/remote.go b/internal/worker/remote.go index 2b98897..1fa6149 100644 --- a/internal/worker/remote.go +++ b/internal/worker/remote.go @@ -199,7 +199,7 @@ func connectPortToFile(ctx context.Context, port *jsworker.MessagePort, file hac log.Error(err) return } - log.Print("Received message! ", bl.Len()) + log.Print("Received message! ", string(bl.Bytes())) _, err = hackpadfs.WriteFile(file, bl.Bytes()) if err != nil { log.Error(err) From e5e0b6f320bf8aaf28049bf2f0cb20e1d7ecb7dd Mon Sep 17 00:00:00 2001 From: John Starich Date: Thu, 16 Jun 2022 23:54:20 -0500 Subject: [PATCH 24/37] Increment open count for piped files in new process --- internal/fs/file_descriptors.go | 8 +++++--- internal/worker/remote.go | 2 +- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/internal/fs/file_descriptors.go b/internal/fs/file_descriptors.go index 75c5da2..012ba36 100644 --- a/internal/fs/file_descriptors.go +++ b/internal/fs/file_descriptors.go @@ -345,11 +345,13 @@ func (f *FileDescriptors) Flock(fd FID, action LockAction) error { return nil } -func (f *FileDescriptors) RawFID(fid FID) (hackpadfs.File, error) { - if _, ok := f.files[fid]; !ok { +func (f *FileDescriptors) OpenRawFID(pid common.PID, fid FID) (hackpadfs.File, error) { + fd, ok := f.files[fid] + if !ok { return nil, interop.BadFileNumber(fid) } - return f.files[fid].file, nil + fd.Open(pid) + return fd.file, nil } func (f *FileDescriptors) RawFIDs() []io.Reader { diff --git a/internal/worker/remote.go b/internal/worker/remote.go index 1fa6149..fda2da2 100644 --- a/internal/worker/remote.go +++ b/internal/worker/remote.go @@ -28,7 +28,7 @@ func NewRemote(local *Local, pid process.PID, command string, argv []string, att var openFiles []openFile for _, f := range attr.Files { - file, err := local.process.Files().RawFID(f.FID) + file, err := local.process.Files().OpenRawFID(pid, f.FID) if err != nil { return nil, err } From 28bff3cd65a02e9f57ee4e40c8d92637c78cd8a2 Mon Sep 17 00:00:00 2001 From: John Starich Date: Fri, 17 Jun 2022 00:14:48 -0500 Subject: [PATCH 25/37] Fix inconsistent blob type --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 57315e8..ad4e7e8 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.16 require ( github.com/avct/uasurfer v0.0.0-20191028135549-26b5daa857f1 github.com/hack-pad/go-indexeddb v0.1.0 - github.com/hack-pad/hackpadfs v0.1.2 + github.com/hack-pad/hackpadfs v0.1.4 github.com/hack-pad/hush v0.1.0 github.com/johnstarich/go/datasize v0.0.1 github.com/machinebox/progress v0.2.0 diff --git a/go.sum b/go.sum index 954e55c..851be9e 100644 --- a/go.sum +++ b/go.sum @@ -16,6 +16,8 @@ github.com/hack-pad/hackpadfs v0.1.1 h1:DhzS50ln5XAOxZ0Xlnb/o3P/+MWUqlcbGdOQ5m+F github.com/hack-pad/hackpadfs v0.1.1/go.mod h1:8bsINHOQhQUioUUiCzCyZZNLfEXjs0RwBIf3lTG+CEg= github.com/hack-pad/hackpadfs v0.1.2 h1:ZsHfvrNAMNNBVLMKprOiN2rLD37x+YGj3QPJrhUdRF4= github.com/hack-pad/hackpadfs v0.1.2/go.mod h1:8bsINHOQhQUioUUiCzCyZZNLfEXjs0RwBIf3lTG+CEg= +github.com/hack-pad/hackpadfs v0.1.4 h1:vwLyuaVPFDqiy6YjLzvQ5fBTt0upzCaCkTok9aoKOdY= +github.com/hack-pad/hackpadfs v0.1.4/go.mod h1:8bsINHOQhQUioUUiCzCyZZNLfEXjs0RwBIf3lTG+CEg= github.com/hack-pad/hush v0.0.0-20210730065049-bd589dbef3a3 h1:0WBvEONkD8zXBRe7+5+mp34L2Upmok0yPKvOqOzpksw= github.com/hack-pad/hush v0.0.0-20210730065049-bd589dbef3a3/go.mod h1:NqjEIfyA2YtlnEPlI/1K3tNuyXGByWFadPxPlGrDPms= github.com/hack-pad/hush v0.1.0 h1:lm/iUaRpVsKkpbN6U9wf45arVnCXzTqsMG1jyihIgkI= From 030b6a5a8cfab40b876e5189272a4bb3bfba6bea Mon Sep 17 00:00:00 2001 From: John Starich Date: Sat, 18 Jun 2022 14:23:21 -0500 Subject: [PATCH 26/37] Close remote worker files on exit --- cmd/editor/main.go | 2 +- internal/worker/remote.go | 20 +++++++++++++++----- 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/cmd/editor/main.go b/cmd/editor/main.go index 4527443..7570870 100644 --- a/cmd/editor/main.go +++ b/cmd/editor/main.go @@ -61,7 +61,7 @@ func main() { return } - select {} // TODO: remove + select {} // TODO: remove when go version output appears on task console if err := os.MkdirAll("playground", 0700); err != nil { log.Error("Failed to make playground dir", err) diff --git a/internal/worker/remote.go b/internal/worker/remote.go index fda2da2..4532204 100644 --- a/internal/worker/remote.go +++ b/internal/worker/remote.go @@ -47,12 +47,12 @@ func NewRemote(local *Local, pid process.PID, command string, argv []string, att return nil, err } openF.pipe = port1 - log.Print("Connecting port to file...") - err = connectPortToFile(closeCtx, port2, file) + log.Print("Binding port to file...") + err = bindPortToFile(closeCtx, port2, file) if err != nil { return nil, err } - log.Print("Connected port to file.") + log.Print("Bound port to file.") } openFiles = append(openFiles, openF) } @@ -188,8 +188,8 @@ func (r *Remote) Wait() (exitCode int, err error) { return *r.closeExitCode, r.closeErr } -func connectPortToFile(ctx context.Context, port *jsworker.MessagePort, file hackpadfs.File) error { - return port.Listen(ctx, func(me jsworker.MessageEvent, err error) { +func bindPortToFile(ctx context.Context, port *jsworker.MessagePort, file hackpadfs.File) error { + err := port.Listen(ctx, func(me jsworker.MessageEvent, err error) { if err != nil { log.Error(err) return @@ -205,5 +205,15 @@ func connectPortToFile(ctx context.Context, port *jsworker.MessagePort, file hac log.Error(err) return } + log.Print("Successfully wrote message") }) + if err != nil { + return err + } + + go func() { + <-ctx.Done() + file.Close() + }() + return nil } From 195cc849741f0724b5e07e34f536a5551538eab3 Mon Sep 17 00:00:00 2001 From: John Starich Date: Sat, 18 Jun 2022 14:38:21 -0500 Subject: [PATCH 27/37] Fix remote worker working directory --- internal/worker/remote.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/internal/worker/remote.go b/internal/worker/remote.go index 4532204..cdb17a6 100644 --- a/internal/worker/remote.go +++ b/internal/worker/remote.go @@ -26,6 +26,11 @@ func NewRemote(local *Local, pid process.PID, command string, argv []string, att ctx := context.Background() closeCtx, cancel := context.WithCancel(ctx) + if attr.Dir == "" { + // collect default current working directory, remote workers won't inherit it + attr.Dir = local.process.WorkingDirectory() + } + var openFiles []openFile for _, f := range attr.Files { file, err := local.process.Files().OpenRawFID(pid, f.FID) @@ -199,13 +204,11 @@ func bindPortToFile(ctx context.Context, port *jsworker.MessagePort, file hackpa log.Error(err) return } - log.Print("Received message! ", string(bl.Bytes())) _, err = hackpadfs.WriteFile(file, bl.Bytes()) if err != nil { log.Error(err) return } - log.Print("Successfully wrote message") }) if err != nil { return err From 66569c2e5c9f656f67a569c61bfc3f5076637bf9 Mon Sep 17 00:00:00 2001 From: John Starich Date: Sat, 18 Jun 2022 14:38:38 -0500 Subject: [PATCH 28/37] Continue setup after go version --- cmd/editor/main.go | 2 -- cmd/worker/main.go | 2 -- 2 files changed, 4 deletions(-) diff --git a/cmd/editor/main.go b/cmd/editor/main.go index 7570870..86979d2 100644 --- a/cmd/editor/main.go +++ b/cmd/editor/main.go @@ -61,8 +61,6 @@ func main() { return } - select {} // TODO: remove when go version output appears on task console - if err := os.MkdirAll("playground", 0700); err != nil { log.Error("Failed to make playground dir", err) return diff --git a/cmd/worker/main.go b/cmd/worker/main.go index ef2a4fa..310045d 100644 --- a/cmd/worker/main.go +++ b/cmd/worker/main.go @@ -23,8 +23,6 @@ func main() { log.SetLevel(log.LevelDebug) bootCtx := context.Background() - //bootCtx, bootCancel := context.WithTimeout(context.Background(), 30*time.Second) - //defer bootCancel() log.Debug("booting worker") local, err := worker.NewLocal(bootCtx, jsworker.GetLocal()) if err != nil { From d8f7c7d280b6887c2fec53c2a7b504a5a08d3677 Mon Sep 17 00:00:00 2001 From: John Starich Date: Sat, 18 Jun 2022 15:41:20 -0500 Subject: [PATCH 29/37] Disable debug logs for worker --- cmd/worker/main.go | 1 - 1 file changed, 1 deletion(-) diff --git a/cmd/worker/main.go b/cmd/worker/main.go index 310045d..f0c1839 100644 --- a/cmd/worker/main.go +++ b/cmd/worker/main.go @@ -21,7 +21,6 @@ func main() { os.Exit(1) }) - log.SetLevel(log.LevelDebug) bootCtx := context.Background() log.Debug("booting worker") local, err := worker.NewLocal(bootCtx, jsworker.GetLocal()) From 3f7c394d90b67d398eeadb001df0c5dd8513636f Mon Sep 17 00:00:00 2001 From: John Starich Date: Sat, 18 Jun 2022 15:41:37 -0500 Subject: [PATCH 30/37] Copy current env to remote worker --- internal/process/process.go | 8 ++++++++ internal/worker/remote.go | 5 ++++- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/internal/process/process.go b/internal/process/process.go index 9b966e8..26cd90a 100644 --- a/internal/process/process.go +++ b/internal/process/process.go @@ -164,3 +164,11 @@ func Dump() interface{} { } return s.String() } + +func (p *Process) Env() map[string]string { + envCopy := make(map[string]string, len(p.env)) + for k, v := range p.env { + envCopy[k] = v + } + return envCopy +} diff --git a/internal/worker/remote.go b/internal/worker/remote.go index cdb17a6..f638f00 100644 --- a/internal/worker/remote.go +++ b/internal/worker/remote.go @@ -26,10 +26,13 @@ func NewRemote(local *Local, pid process.PID, command string, argv []string, att ctx := context.Background() closeCtx, cancel := context.WithCancel(ctx) + // collect current process details and use as defaults, remote workers won't inherit them if attr.Dir == "" { - // collect default current working directory, remote workers won't inherit it attr.Dir = local.process.WorkingDirectory() } + if len(attr.Env) == 0 { + attr.Env = local.process.Env() + } var openFiles []openFile for _, f := range attr.Files { From aa0ff3e2f61bf41d1205f2957c1da3a7ddd6a063 Mon Sep 17 00:00:00 2001 From: John Starich Date: Sun, 26 Jun 2022 17:05:00 -0500 Subject: [PATCH 31/37] Add idb-based global file locker --- internal/fs/file_descriptors.go | 33 +++-- internal/fs/file_locker.go | 209 ++++++++++++++++++++++++++++++++ 2 files changed, 224 insertions(+), 18 deletions(-) create mode 100644 internal/fs/file_locker.go diff --git a/internal/fs/file_descriptors.go b/internal/fs/file_descriptors.go index 012ba36..68dbdd0 100644 --- a/internal/fs/file_descriptors.go +++ b/internal/fs/file_descriptors.go @@ -1,6 +1,7 @@ package fs import ( + "context" "io" "os" "path" @@ -29,17 +30,23 @@ type FileDescriptors struct { files map[FID]*fileDescriptor mu sync.Mutex workingDirectory *workingDirectory + locks *fileLocker } func NewStdFileDescriptors(parentPID common.PID, workingDirectory string) (*FileDescriptors, error) { + locker, err := newFileLocker(context.Background()) + if err != nil { + return nil, err + } f := &FileDescriptors{ parentPID: parentPID, previousFID: 0, files: make(map[FID]*fileDescriptor), workingDirectory: newWorkingDirectory(workingDirectory), + locks: locker, } // order matters - _, err := f.Open("/dev/stdin", syscall.O_RDONLY, 0) + _, err = f.Open("/dev/stdin", syscall.O_RDONLY, 0) if err != nil { return nil, err } @@ -52,11 +59,16 @@ func NewStdFileDescriptors(parentPID common.PID, workingDirectory string) (*File } func NewFileDescriptors(parentPID common.PID, workingDirectory string, openFiles []common.OpenFileAttr) (_ *FileDescriptors, _ func(wd string) error, returnedErr error) { + locker, err := newFileLocker(context.Background()) + if err != nil { + return nil, nil, err + } f := &FileDescriptors{ parentPID: parentPID, previousFID: 0, files: make(map[FID]*fileDescriptor), workingDirectory: newWorkingDirectory(workingDirectory), + locks: locker, } type openFile struct { attr common.OpenFileAttr @@ -314,35 +326,20 @@ const ( Unlock ) -var ( - processFileLocks = make(map[string]*sync.RWMutex) - newFileLockMu sync.Mutex -) - func (f *FileDescriptors) Flock(fd FID, action LockAction) error { fileDescriptor := f.files[fd] if fileDescriptor == nil { return interop.BadFileNumber(fd) } absPath := fileDescriptor.FileName() - if _, ok := processFileLocks[absPath]; !ok { - newFileLockMu.Lock() - if _, ok := processFileLocks[absPath]; !ok { - processFileLocks[absPath] = new(sync.RWMutex) - } - newFileLockMu.Unlock() - } - lock := processFileLocks[absPath] switch action { case LockShared, LockExclusive: - // TODO support shared locks - lock.Lock() + return f.locks.Lock(context.Background(), absPath, action == LockShared) case Unlock: - lock.Unlock() + return f.locks.Unlock(context.Background(), absPath) default: return interop.ErrNotImplemented } - return nil } func (f *FileDescriptors) OpenRawFID(pid common.PID, fid FID) (hackpadfs.File, error) { diff --git a/internal/fs/file_locker.go b/internal/fs/file_locker.go new file mode 100644 index 0000000..2496ebc --- /dev/null +++ b/internal/fs/file_locker.go @@ -0,0 +1,209 @@ +package fs + +import ( + "context" + "errors" + "syscall/js" + "time" + + "github.com/hack-pad/go-indexeddb/idb" +) + +type fileLocker struct { + db *idb.Database +} + +const ( + locksObjectStore = "locks" + + sharedCountField = "sharedCount" +) + +func newFileLocker(ctx context.Context) (*fileLocker, error) { + openRequest, err := idb.Global().Open(ctx, "file-locks", 1, func(db *idb.Database, oldVersion, newVersion uint) error { + _, err := db.CreateObjectStore(locksObjectStore, idb.ObjectStoreOptions{}) + return err + }) + if err != nil { + return nil, err + } + db, err := openRequest.Await(ctx) + if err != nil { + return nil, err + } + return &fileLocker{ + db: db, + }, nil +} + +func (f *fileLocker) Lock(ctx context.Context, filePath string, shared bool) error { + locked, err := f.tryLock(ctx, filePath, shared) + if locked || err != nil { + return err + } + + // lock is either exclusive or does not match the current lock type. must wait its turn. + const pollInterval = 10 * time.Millisecond + ticker := time.NewTicker(pollInterval) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return ctx.Err() + case <-ticker.C: + drainTicker(ticker) + locked, err := f.tryLock(ctx, filePath, shared) + if locked || err != nil { + return err + } + } + } +} + +func (f *fileLocker) tryLock(ctx context.Context, filePath string, shared bool) (locked bool, err error) { + txn, err := f.db.Transaction(idb.TransactionReadWrite, locksObjectStore) + if err != nil { + return false, err + } + locks, err := txn.ObjectStore(locksObjectStore) + if err != nil { + return false, err + } + jsKey := js.ValueOf(filePath) + req, err := locks.Get(jsKey) + if err != nil { + return false, err + } + tryLock := func() (locked bool, err error) { + lock, err := req.Result() + if err != nil { + return false, err + } + if !lock.Truthy() { + // lock not yet held + sharedCount := 0 + if shared { + sharedCount++ + } + err := putLock(locks, jsKey, sharedCount) + if err != nil { + return false, err + } + return true, txn.Commit() + } + + // lock is held + sharedCount, err := getSharedCount(lock) + if err != nil { + return false, err + } + isShared := sharedCount > 0 + if shared { + sharedCount++ + } + if shared && isShared { // lock already held by shared: add 1 and return + err := putLock(locks, jsKey, sharedCount+1) + if err != nil { + return false, err + } + return true, txn.Commit() + } + return false, nil + } + var listenErr error + req.ListenSuccess(ctx, func() { + locked, listenErr = tryLock() + if listenErr != nil { + txn.Abort() + return + } + }) + err = txn.Await(ctx) + if listenErr != nil { + return false, listenErr + } + if err != nil { + return false, err + } + return locked, nil +} + +func putLock(locks *idb.ObjectStore, key js.Value, sharedCount int) error { + _, err := locks.PutKey(key, js.ValueOf(map[string]interface{}{ + sharedCountField: sharedCount, + })) + return err +} + +func getSharedCount(lock js.Value) (int, error) { + jsSharedCount := lock.Get(sharedCountField) + if jsSharedCount.Type() != js.TypeNumber { + return 0, errors.New("malformed shared count") + } + return jsSharedCount.Int(), nil +} + +func drainTicker(ticker *time.Ticker) { + for { + select { + case _, ok := <-ticker.C: + if !ok { + return + } + default: + return + } + } +} + +func (f *fileLocker) Unlock(ctx context.Context, filePath string) error { + txn, err := f.db.Transaction(idb.TransactionReadWrite, locksObjectStore) + if err != nil { + return err + } + locks, err := txn.ObjectStore(locksObjectStore) + if err != nil { + return err + } + jsKey := js.ValueOf(filePath) + req, err := locks.Get(jsKey) + if err != nil { + return err + } + tryUnlock := func() error { + lock, err := req.Result() + if err != nil { + return err + } + sharedCount, err := getSharedCount(lock) + if err != nil { + return err + } + if sharedCount <= 1 { // is exclusive lock or last shared lock + _, err := locks.Delete(jsKey) + if err != nil { + return err + } + return txn.Commit() + } + sharedCount-- + err = putLock(locks, jsKey, sharedCount) + if err != nil { + return err + } + return txn.Commit() + } + var listenErr error + req.ListenSuccess(ctx, func() { + listenErr = tryUnlock() + if listenErr != nil { + txn.Abort() + return + } + }) + err = txn.Await(ctx) + if listenErr != nil { + return listenErr + } + return err +} From 76bc53ca62ce5e74bf58b8e0c0cdba1d458bc039 Mon Sep 17 00:00:00 2001 From: John Starich Date: Sun, 26 Jun 2022 18:57:20 -0500 Subject: [PATCH 32/37] Mount /tmp in worker too --- cmd/worker/main.go | 27 ++++++++++++++------------- main.go | 20 ++++++++++++++------ 2 files changed, 28 insertions(+), 19 deletions(-) diff --git a/cmd/worker/main.go b/cmd/worker/main.go index f0c1839..dca9708 100644 --- a/cmd/worker/main.go +++ b/cmd/worker/main.go @@ -51,28 +51,29 @@ func main() { func setUpFS() error { const dirPerm = 0700 - if err := os.MkdirAll("/bin", dirPerm); err != nil { - return err - } - if err := overlayIndexedDB("/bin", idb.DurabilityRelaxed); err != nil { - return err + mkdirMount := func(mountPath string, durability idb.TransactionDurability) error { + if err := os.MkdirAll(mountPath, dirPerm); err != nil { + return err + } + if err := overlayIndexedDB(mountPath, durability); err != nil { + return err + } + return nil } - if err := os.MkdirAll("/home/me", dirPerm); err != nil { - return err - } - if err := overlayIndexedDB("/home/me", idb.DurabilityDefault); err != nil { + + if err := mkdirMount("/bin", idb.DurabilityRelaxed); err != nil { return err } - if err := os.MkdirAll("/home/me/.cache", dirPerm); err != nil { + if err := mkdirMount("/home/me", idb.DurabilityDefault); err != nil { return err } - if err := overlayIndexedDB("/home/me/.cache", idb.DurabilityRelaxed); err != nil { + if err := mkdirMount("/home/me/.cache", idb.DurabilityRelaxed); err != nil { return err } - if err := os.MkdirAll("/usr/local/go", dirPerm); err != nil { + if err := mkdirMount("/tmp", idb.DurabilityRelaxed); err != nil { return err } - if err := overlayIndexedDB("/usr/local/go", idb.DurabilityRelaxed); err != nil { + if err := mkdirMount("/usr/local/go", idb.DurabilityRelaxed); err != nil { return err } return nil diff --git a/main.go b/main.go index 52ce63d..8ef422b 100644 --- a/main.go +++ b/main.go @@ -76,21 +76,29 @@ func main() { func setUpFS(shim domShim) error { const dirPerm = 0700 - if err := os.MkdirAll("/bin", dirPerm); err != nil { - return err + mkdirMount := func(mountPath string, durability idb.TransactionDurability) error { + if err := os.MkdirAll(mountPath, dirPerm); err != nil { + return err + } + if err := overlayIndexedDB(mountPath, durability); err != nil { + return err + } + return nil } - if err := overlayIndexedDB("/bin", idb.DurabilityRelaxed); err != nil { + + if err := mkdirMount("/bin", idb.DurabilityRelaxed); err != nil { return err } - if err := overlayIndexedDB("/home/me", idb.DurabilityDefault); err != nil { + if err := mkdirMount("/home/me", idb.DurabilityDefault); err != nil { return err } - if err := os.MkdirAll("/home/me/.cache", dirPerm); err != nil { + if err := mkdirMount("/home/me/.cache", idb.DurabilityRelaxed); err != nil { return err } - if err := overlayIndexedDB("/home/me/.cache", idb.DurabilityRelaxed); err != nil { + if err := mkdirMount("/tmp", idb.DurabilityRelaxed); err != nil { return err } + if err := os.MkdirAll("/usr/local/go", dirPerm); err != nil { return err } From a55e39ce248ed29e65d26ef0d61001a21cafc1fd Mon Sep 17 00:00:00 2001 From: John Starich Date: Sun, 26 Jun 2022 19:10:02 -0500 Subject: [PATCH 33/37] Fix exit status error missing newline --- cmd/editor/taskconsole/console.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/editor/taskconsole/console.go b/cmd/editor/taskconsole/console.go index f8908b8..c343a55 100644 --- a/cmd/editor/taskconsole/console.go +++ b/cmd/editor/taskconsole/console.go @@ -78,7 +78,7 @@ func (c *console) runLoopIter() { defer cancel(commandErr) elapsed := time.Since(startTime) if commandErr != nil { - _, _ = c.stderr.Write([]byte(commandErr.Error())) + _, _ = c.stderr.Write([]byte(commandErr.Error() + "\n")) } exitCode := 0 From 47400a80525141298b01e9d23cf09ea96540ff3c Mon Sep 17 00:00:00 2001 From: John Starich Date: Sun, 26 Jun 2022 20:14:55 -0500 Subject: [PATCH 34/37] Reduce logging --- cmd/worker/main.go | 2 +- internal/worker/dom.go | 4 ---- internal/worker/local.go | 8 ++------ internal/worker/remote.go | 15 ++++++--------- server/public/wasmWorker.js | 1 - 5 files changed, 9 insertions(+), 21 deletions(-) diff --git a/cmd/worker/main.go b/cmd/worker/main.go index dca9708..9496449 100644 --- a/cmd/worker/main.go +++ b/cmd/worker/main.go @@ -44,7 +44,7 @@ func main() { log.Error("Failed to wait for PID ", pid, ":", err) exitCode = 1 } - log.Warn("worker stopped for PID ", pid, "; exit code = ", exitCode) + log.Debug("worker stopped for PID ", pid, "; exit code = ", exitCode) local.Exit(exitCode) os.Exit(exitCode) } diff --git a/internal/worker/dom.go b/internal/worker/dom.go index 4cb11c4..61bb58a 100644 --- a/internal/worker/dom.go +++ b/internal/worker/dom.go @@ -5,7 +5,6 @@ import ( "syscall/js" "github.com/hack-pad/hackpad/internal/jsworker" - "github.com/hack-pad/hackpad/internal/log" ) type DOM struct { @@ -25,16 +24,13 @@ func ExecDOM(ctx context.Context, localJS *jsworker.Local, command string, args if err != nil { return nil, err } - log.Print("NewLocal start") local, err := NewLocal(ctx, localJS) if err != nil { return nil, err } - log.Print("local start") if err := local.Start(); err != nil { return nil, err } - log.Print("local ready") return &DOM{ local: local, port: localJS, diff --git a/internal/worker/local.go b/internal/worker/local.go index 38968d5..2d6e3aa 100644 --- a/internal/worker/local.go +++ b/internal/worker/local.go @@ -67,7 +67,6 @@ func (l *Local) awaitInit(ctx context.Context) (js.Value, error) { } initChan := make(chan initMessage, 1) err := l.localJS.Listen(ctx, func(me jsworker.MessageEvent, err error) { - log.Print("awaitInit listener:", err) if err != nil { initChan <- initMessage{err: err} return @@ -97,7 +96,6 @@ func (l *Local) awaitInit(ctx context.Context) (js.Value, error) { func (l *Local) Start() (err error) { defer common.CatchException(&err) startCtx, cancel := context.WithCancel(context.Background()) - log.Print("Listening for start...") err = l.localJS.Listen(startCtx, func(me jsworker.MessageEvent, err error) { if err != nil { log.Error(err) @@ -108,7 +106,6 @@ func (l *Local) Start() (err error) { log.Error(err) cancel() }) - log.Print("Local's Start() received message") if me.Data.Type() != js.TypeObject { return } @@ -119,7 +116,6 @@ func (l *Local) Start() (err error) { } cancel() - log.Print("Starting process: ", l.process.PID()) err = l.process.Start() if err != nil { log.Error(err) @@ -150,7 +146,7 @@ func (l *Local) Exit(exitCode int) error { func (l *Local) Spawn(command string, argv []string, attr *process.ProcAttr) (jsprocess.PIDer, error) { pid := kernel.ReservePID() - log.Print("Spawning pid ", pid, " for command: ", command, argv) + log.Debug("Spawning pid ", pid, " for command: ", command, argv) remote, err := NewRemote(l, pid, command, argv, attr) if err != nil { return nil, err @@ -160,7 +156,7 @@ func (l *Local) Spawn(command string, argv []string, attr *process.ProcAttr) (js } func (l *Local) Wait(pid common.PID) (exitCode int, err error) { - log.Print("Waiting on pid ", pid) + log.Debug("Waiting on pid ", pid) if pid == l.process.PID() { return l.process.Wait() } diff --git a/internal/worker/remote.go b/internal/worker/remote.go index f638f00..5d70a0b 100644 --- a/internal/worker/remote.go +++ b/internal/worker/remote.go @@ -49,18 +49,15 @@ func NewRemote(local *Local, pid process.PID, command string, argv []string, att seekOffset: 0, // TODO expose seek offset in file descriptor } if info.Mode()&hackpadfs.ModeNamedPipe != 0 { - log.Print("Found pipe, creating MessageChannel...") port1, port2, err := jsworker.NewChannel() if err != nil { return nil, err } openF.pipe = port1 - log.Print("Binding port to file...") err = bindPortToFile(closeCtx, port2, file) if err != nil { return nil, err } - log.Print("Bound port to file.") } openFiles = append(openFiles, openF) } @@ -90,7 +87,7 @@ func NewRemote(local *Local, pid process.PID, command string, argv []string, att exitCode := jsExitCode.Int() remote.closeExitCode = &exitCode cancel() - log.Warn("Remote exited with code:", exitCode) + log.Debug("Remote exited with code:", exitCode) } }) if err != nil { @@ -98,31 +95,31 @@ func NewRemote(local *Local, pid process.PID, command string, argv []string, att } go func() { - log.Print("Worker ", workerName, " awaiting pending_init...") + log.Debug("Worker ", workerName, " awaiting pending_init...") err := awaitMessage(ctx, port, "pending_init") if err != nil { log.Error("Failed awaiting pending_init:", workerName, err) return } - log.Print("Worker ", workerName, " waiting to init. Sending init...") + log.Debug("Worker ", workerName, " waiting to init. Sending init...") msg, transfers := makeInitMessage(workerName, command, argv, attr.Dir, attr.Env, openFiles) err = port.PostMessage(msg, transfers) if err != nil { log.Error("Failed sending init to worker: ", workerName, " ", err) return } - log.Print("Sent init message to worker ", workerName, ". Awaiting ready...") + log.Debug("Sent init message to worker ", workerName, ". Awaiting ready...") if err := awaitMessage(ctx, remote.port, "ready"); err != nil { log.Error("Failed awaiting ready:", workerName, err) return } - log.Print("Worker ", workerName, " is ready. Sending start message.") + log.Debug("Worker ", workerName, " is ready. Sending start message.") err = remote.port.PostMessage(makeStartMessage(), nil) if err != nil { log.Error("Failed sending start to worker: ", workerName, " ", err) return } - log.Print("Sent start message.") + log.Debug("Sent start message.") }() return remote, nil diff --git a/server/public/wasmWorker.js b/server/public/wasmWorker.js index 32feb36..1f1c47b 100644 --- a/server/public/wasmWorker.js +++ b/server/public/wasmWorker.js @@ -1,7 +1,6 @@ "use strict"; async function runWasm(params) { - console.log("Loading Wasm:", params) self.importScripts("wasm/wasm_exec.js") const go = new Go() const result = await WebAssembly.instantiateStreaming(fetch(params.wasm), go.importObject) From b7eaa9f7211f1a4129f3c7195bcb5e7ba4252123 Mon Sep 17 00:00:00 2001 From: John Starich Date: Sun, 26 Jun 2022 20:26:55 -0500 Subject: [PATCH 35/37] Add env info inside worker's hackpad.process --- internal/worker/local.go | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/internal/worker/local.go b/internal/worker/local.go index 2d6e3aa..85eea7b 100644 --- a/internal/worker/local.go +++ b/internal/worker/local.go @@ -37,13 +37,20 @@ func NewLocal(ctx context.Context, localJS *jsworker.Local) (_ *Local, err error global.Set("workerName", init.Get("workerName")) log.Debug("Setting process details...") + var ( + command = init.Get("command") + argv = init.Get("argv") + workingDirectory = init.Get("workingDirectory") + openFiles = init.Get("openFiles") + env = init.Get("env") + ) local.process, err = process.New( kernel.ReservePID(), - init.Get("command").String(), - interop.StringsFromJSValue(init.Get("argv")), - init.Get("workingDirectory").String(), - parseOpenFiles(init.Get("openFiles")), - interop.StringMapFromJSObject(init.Get("env")), + command.String(), + interop.StringsFromJSValue(argv), + workingDirectory.String(), + parseOpenFiles(openFiles), + interop.StringMapFromJSObject(env), ) if err != nil { return nil, err @@ -52,6 +59,12 @@ func NewLocal(ctx context.Context, localJS *jsworker.Local) (_ *Local, err error jsprocess.Init(local.process, local, local) log.Debug("Initializing fs") jsfs.Init(local.process) + global.Set("process", map[string]interface{}{ + "command": command, + "argv": argv, + "workingDirectory": workingDirectory, + "env": env, + }) return local, nil } From 5358139e2533383950ebca17ae568e3e71f5ce41 Mon Sep 17 00:00:00 2001 From: John Starich Date: Sun, 26 Jun 2022 23:19:34 -0500 Subject: [PATCH 36/37] Fix missing read loop on local port side --- internal/worker/remote.go | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/internal/worker/remote.go b/internal/worker/remote.go index 5d70a0b..1e5c110 100644 --- a/internal/worker/remote.go +++ b/internal/worker/remote.go @@ -12,6 +12,7 @@ import ( "github.com/hack-pad/hackpad/internal/process" "github.com/hack-pad/hackpadfs" "github.com/hack-pad/hackpadfs/indexeddb/idbblob" + "github.com/hack-pad/hackpadfs/keyvalue/blob" ) type Remote struct { @@ -213,10 +214,30 @@ func bindPortToFile(ctx context.Context, port *jsworker.MessagePort, file hackpa if err != nil { return err } - go func() { <-ctx.Done() file.Close() }() + go func() { + const maxReadSize = 1 << 10 + buf := make([]byte, maxReadSize) + for { + n, err := file.Read(buf) + if err != nil { + if err.Error() != "operation not supported" { + log.Error(err) + } + return + } + if n > 0 { + bl := idbblob.FromBlob(blob.NewBytes(buf[:n])).JSValue() + err := port.PostMessage(bl, []js.Value{bl.Get("buffer")}) + if err != nil { + log.Error(err) + return + } + } + } + }() return nil } From 460a392aaf4604c8575b9204d2c6e66eb74cea19 Mon Sep 17 00:00:00 2001 From: John Starich Date: Sun, 26 Jun 2022 23:20:31 -0500 Subject: [PATCH 37/37] Allow pipe chan to read without blocking if it already has 1 byte --- internal/fs/pipe.go | 28 +++++++++++++++++++++------- 1 file changed, 21 insertions(+), 7 deletions(-) diff --git a/internal/fs/pipe.go b/internal/fs/pipe.go index e4ddb39..c10aa27 100644 --- a/internal/fs/pipe.go +++ b/internal/fs/pipe.go @@ -88,16 +88,30 @@ func (p *pipeChan) Sync() error { } func (p *pipeChan) Read(buf []byte) (n int, err error) { + // Read should always block if the pipe is not closed + b, ok := <-p.buf + if !ok { + err = io.EOF + return + } + buf[n] = b + n++ + for n < len(buf) { - // Read should always block if the pipe is not closed - b, ok := <-p.buf - if !ok { - err = io.EOF - return + // attempt to read anything else if we still have room + select { + case b, ok := <-p.buf: + if !ok { + err = io.EOF + return + } + buf[n] = b + n++ + default: + goto doneReading } - buf[n] = b - n++ } +doneReading: if n == 0 { err = io.EOF }