Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
28 changes: 19 additions & 9 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -244,7 +244,8 @@ the creation, termination and expiration extensions), so tus clients such
as [tus-js-client](https://github.com/tus/tus-js-client) or
[tuspy](https://github.com/tus/tus-py-client) can upload to a folder
(`https://host/dir/` as the endpoint, the file's name as `filename` in its
metadata, and `replace` set to `1` to replace an existing file). With curl:
metadata, `replace` set to `1` to replace an existing file, and `mtime`, in
seconds since 1970, for the modification time it keeps). With curl:

```sh
auth="Authorization: Bearer $TOKEN"
Expand Down Expand Up @@ -413,16 +414,26 @@ curl -H "Authorization: Bearer $TOKEN" -d action=create -d name=laptop https://h
curl -H "Authorization: Bearer $TOKEN" -d action=revoke -d id=1 https://host/.sync/
```

With [rclone](https://rclone.org), which keeps modification times with
`vendor=owncloud`, a folder syncs both ways like this; the first run needs
`--resync`, and the second line can then run from cron:
With [rclone](https://rclone.org), a folder syncs both ways like this; the
first run needs `--resync`, and the second line can then run from cron:

```sh
rclone config create goftp webdav url=https://host/.dav/home/bob/ vendor=owncloud user=bob pass=APP-PASSWORD --obscure
rclone config create goftp webdav url=https://host/.dav/home/bob/ vendor=infinitescale user=bob pass=APP-PASSWORD --obscure
rclone bisync ~/goftp goftp: --exclude ".*" --exclude ".*/**" --resync
rclone bisync ~/goftp goftp: --exclude ".*" --exclude ".*/**"
```

With `vendor=infinitescale`, rclone sends files as ownCloud Infinite Scale
takes them: as resumable uploads ([tus](https://tus.io), as the browser
does) that a POST to the folder starts, in 10 MiB pieces, so that files of
any size get through proxies that limit the size of requests. Those
pieces sign in with the same app password, at the absolute `Location`
the POST answers; an upload replaces a file of its name unless
`If-None-Match: *` says not to, and `mtime` in `Upload-Metadata` (in
seconds since 1970) becomes the file's modification time, as it does for
the browser's uploads. `vendor=owncloud` works too, sending each file in
one request.

FolderSync (Android), GoodSync, Mountain Duck and Cyberduck take the same
address, username and app password as a WebDAV account; Finder opens it
with Go → Connect to Server. goftp keeps the modification times that
Expand All @@ -441,10 +452,9 @@ much room a home has left (RFC 4331). Limits:
folder that a MOVE or COPY replaces goes to the trash, once the move or
copy worked. `If-Match` and `If-Unmodified-Since` make changes depend on
the ETag or time a client saw.
- WebDAV sends each file in one request, where the browser sends large
ones in pieces: behind a proxy that limits the size of requests
(Cloudflare: 100 MB on its Free and Pro plans), larger files cannot be
uploaded over WebDAV.
- Apps other than rclone with `vendor=infinitescale` send each file in one
request: behind a proxy that limits the size of requests (Cloudflare:
100 MB on its Free and Pro plans), they cannot upload larger files.

Wrong app passwords count against the client's sign-in budget
(`limiter.*`).
Expand Down
22 changes: 16 additions & 6 deletions internal/auth/postgres.go
Original file line number Diff line number Diff line change
Expand Up @@ -324,21 +324,31 @@ func (s *PgStore) DeleteExpiredShares(ctx context.Context) error {
return err
}

const uploadColumns = "id, user_id, dir, name, length, replace, temp, share_id, created_at, expires_at"
const uploadColumns = "id, user_id, dir, name, length, replace, temp, share_id, mtime, created_at, expires_at"

func scanUpload(row pgx.Row) (*Upload, error) {
var up Upload
err := row.Scan(&up.ID, &up.UserID, &up.Dir, &up.Name, &up.Length, &up.Replace, &up.Temp, &up.ShareID, &up.CreatedAt, &up.ExpiresAt)
var (
up Upload
mtime *time.Time
)
err := row.Scan(&up.ID, &up.UserID, &up.Dir, &up.Name, &up.Length, &up.Replace, &up.Temp, &up.ShareID, &mtime, &up.CreatedAt, &up.ExpiresAt)
if errors.Is(err, pgx.ErrNoRows) {
return nil, ErrNotFound
}
if mtime != nil {
up.Mtime = *mtime
}
return &up, err
}

func (s *PgStore) CreateUpload(ctx context.Context, tokenHash []byte, up *Upload, keep time.Duration) error {
return s.db.QueryRow(ctx, `INSERT INTO uploads (token_hash, user_id, dir, name, length, replace, temp, share_id, expires_at)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, now() + $9::interval) RETURNING id, created_at, expires_at`,
tokenHash, up.UserID, up.Dir, up.Name, up.Length, up.Replace, up.Temp, up.ShareID, keep).Scan(&up.ID, &up.CreatedAt, &up.ExpiresAt)
var mtime *time.Time
if !up.Mtime.IsZero() {
mtime = &up.Mtime
}
return s.db.QueryRow(ctx, `INSERT INTO uploads (token_hash, user_id, dir, name, length, replace, temp, share_id, mtime, expires_at)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, now() + $10::interval) RETURNING id, created_at, expires_at`,
tokenHash, up.UserID, up.Dir, up.Name, up.Length, up.Replace, up.Temp, up.ShareID, mtime, keep).Scan(&up.ID, &up.CreatedAt, &up.ExpiresAt)
}

func (s *PgStore) UploadByToken(ctx context.Context, tokenHash []byte) (*Upload, error) {
Expand Down
3 changes: 2 additions & 1 deletion internal/auth/store.go
Original file line number Diff line number Diff line change
Expand Up @@ -62,7 +62,8 @@ type Upload struct {
Length int64
Replace bool // may replace a file of that name
Temp string
ShareID int64 // the upload link it came through, if any
ShareID int64 // the upload link it came through, if any
Mtime time.Time // the modification time the file gets; zero for when it is stored
CreatedAt time.Time
ExpiresAt time.Time
}
Expand Down
12 changes: 12 additions & 0 deletions internal/auth/upload_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,18 @@ func TestUploads(t *testing.T) {
t.Errorf("UploadByToken(%q): %v", bad, err)
}
}
if !got.Mtime.IsZero() {
t.Errorf("a time without one: %v", got.Mtime)
}
// The modification time the file gets.
timed := &auth.Upload{UserID: 7, Dir: "/d", Name: "b.bin", Length: 1, Temp: ".goftp-B.upload", Mtime: time.Unix(1700000000, 0)}
timedToken, err := svc.CreateUpload(ctx, timed, time.Hour)
if err != nil {
t.Fatal(err)
}
if got, err := svc.UploadByToken(ctx, timedToken); err != nil || !got.Mtime.Equal(time.Unix(1700000000, 0)) {
t.Errorf("with a time: %+v %v", got, err)
}

// One writer at a time; a claim that ran out can be taken over.
claim := func(writer string, lease time.Duration) bool {
Expand Down
2 changes: 1 addition & 1 deletion internal/db/db_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,7 @@ func TestConcurrentMigrations(t *testing.T) {
if err := pools[0].QueryRow(ctx, "SELECT count(*) FROM casbin_rule").Scan(&rules); err != nil {
t.Fatal(err)
}
if versions != 7 || rules != 9 {
if versions != 8 || rules != 9 {
t.Errorf("%d migrations recorded, %d rules seeded", versions, rules)
}
}
Expand Down
3 changes: 3 additions & 0 deletions internal/db/migrations/008_upload_mtime.sql
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
-- The modification time a resumable upload's file gets once complete, as
-- its sender gave it (e.g. a sync app); NULL for when it is stored.
ALTER TABLE uploads ADD COLUMN mtime TIMESTAMPTZ;
22 changes: 19 additions & 3 deletions internal/server/dav.go
Original file line number Diff line number Diff line change
Expand Up @@ -28,22 +28,26 @@ import (
// WebDAV (RFC 4918) at /.dav/, for sync apps and file managers: the same
// files as the site, with the same rules, homes, quotas and trash, signed
// in to with HTTP Basic authentication and an app password (see
// auth.AppPassword). /.dav/PATH is PATH. Locks are advisory: they keep other
// auth.AppPassword). /.dav/PATH is PATH. Files may also be sent in pieces,
// as resumable uploads (see createUpload) that a POST to their folder
// starts, as ownCloud Infinite Scale takes them: proxies that limit the
// size of requests let any size through then. Locks are advisory: they keep other
// WebDAV clients that ask from taking the same lock, and nothing else, as
// goftp's own locks guard every change.
const davPrefix = "/.dav"

// davMethods are the methods WebDAV adds to HTTP's.
var davMethods = []string{"PROPFIND", "PROPPATCH", "MKCOL", "COPY", "MOVE", "LOCK", "UNLOCK"}

const davAllow = "OPTIONS, GET, HEAD, PUT, DELETE, PROPFIND, PROPPATCH, MKCOL, COPY, MOVE, LOCK, UNLOCK"
const davAllow = "OPTIONS, GET, HEAD, PUT, POST, DELETE, PROPFIND, PROPPATCH, MKCOL, COPY, MOVE, LOCK, UNLOCK"

// davBodyLimit bounds the XML bodies of WebDAV requests.
const davBodyLimit = 64 << 10

// davRoutes serves WebDAV under davPrefix.
func (s *Server) davRoutes() {
methods := append([]string{fiber.MethodGet, fiber.MethodHead, fiber.MethodPut, fiber.MethodDelete, fiber.MethodOptions}, davMethods...)
methods := append([]string{fiber.MethodGet, fiber.MethodHead, fiber.MethodPut, fiber.MethodPost, fiber.MethodDelete, fiber.MethodOptions},
davMethods...)
s.app.Add(methods, davPrefix, s.dav)
s.app.Add(methods, davPrefix+"/*", s.dav)
}
Expand All @@ -54,6 +58,9 @@ func (s *Server) dav(c fiber.Ctx) error {
c.Set(fiber.HeaderCacheControl, "no-store")
if c.Method() == fiber.MethodOptions {
c.Set(fiber.HeaderAllow, davAllow)
c.Set("Tus-Resumable", tusVersion)
c.Set("Tus-Version", tusVersion)
c.Set("Tus-Extension", tusExtensions)
return c.SendStatus(fiber.StatusOK)
}
if err := s.davSignIn(c); err != nil {
Expand Down Expand Up @@ -82,6 +89,15 @@ func (s *Server) dav(c fiber.Ctx) error {
return s.davGet(c, urlPath)
case fiber.MethodPut:
return s.davPut(c, urlPath)
case fiber.MethodPost:
// A resumable upload into the folder.
switch {
case c.Get("Tus-Resumable") == "":
return fiber.ErrMethodNotAllowed
case hidden(urlPath):
return fiber.ErrForbidden
}
return s.createUpload(c, urlPath, nil, true)
case fiber.MethodDelete:
return s.davDelete(c, urlPath)
}
Expand Down
94 changes: 93 additions & 1 deletion internal/server/dav_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -764,7 +764,7 @@ func TestSyncPage(t *testing.T) {
for _, want := range []string{
`<code id="dav-url">` + home + `</code>`,
"The rest of the site is at http://example.com/.dav/",
"rclone config create goftp webdav url=" + home + " vendor=owncloud user=" + name + " pass=APP-PASSWORD --obscure",
"rclone config create goftp webdav url=" + home + " vendor=infinitescale user=" + name + " pass=APP-PASSWORD --obscure",
"<code>dav://example.com/.dav/home/" + name + "/</code>",
"You have no app passwords",
"not served over HTTPS",
Expand Down Expand Up @@ -1125,3 +1125,95 @@ func TestDavAppleDoubleBody(t *testing.T) {
t.Errorf("the next request: %v, %v", resp, err)
}
}

// Sync apps may send files in pieces, as resumable uploads that a POST to
// their folder starts: proxies that limit the size of requests let files
// of any size through then.
func TestDavResumable(t *testing.T) {
ta := newTestAuth(t)
f := newFixtureWith(t, ta, withHomes, func(cfg *config.Config) { cfg.Homes.Quota = 100 })
ta.addUser(t, "dora", "user")
ta.addUser(t, "erin", "user")
dora, erin := davAs(t, f, "dora"), davAs(t, f, "erin")
home := filepath.Join(f.dir, "home", "dora")

resp, _ := f.as("").do(t, "OPTIONS", "/.dav/")
if resp.Header.Get("Tus-Version") != tusVersion || !strings.Contains(resp.Header.Get("Allow"), "POST") {
t.Errorf("OPTIONS: %v", resp.Header)
}
start := func(d *davClient, dir, name string, length int, headers ...string) (*http.Response, string) {
t.Helper()
resp, _ := d.send(t, "POST", dir, nil, append([]string{"Tus-Resumable", tusVersion, "Upload-Length", strconv.Itoa(length),
"Upload-Metadata", metadata(name, "mtime=1700000000")}, headers...)...)
return resp, strings.TrimPrefix(resp.Header.Get("Location"), "http://example.com")
}
patch := func(d *davClient, loc string, offset int, data string) *http.Response {
t.Helper()
resp, _ := d.send(t, "PATCH", loc, strings.NewReader(data), "Tus-Resumable", tusVersion, "Upload-Offset", strconv.Itoa(offset),
"Content-Type", "application/offset+octet-stream")
return resp
}

resp, loc := start(dora, "/.dav/home/dora/", "big.bin", 10)
expectStatus(t, resp, http.StatusCreated)
if !strings.HasPrefix(resp.Header.Get("Location"), "http://example.com"+uploadsPrefix) {
t.Fatalf("Location %q", resp.Header.Get("Location"))
}
// Its pieces come with the app password it was started with, and only
// its user's.
if resp := patch(erin, loc, 0, "01234"); resp.StatusCode != http.StatusNotFound {
t.Errorf("another user's piece: %d", resp.StatusCode)
}
if resp := patch(&davClient{f: f, auth: basicAuthHeader("dora", strings.Repeat("a", 24))}, loc, 0, "01234"); resp.StatusCode != http.StatusUnauthorized {
t.Errorf("a wrong app password: %d", resp.StatusCode)
}
resp = patch(dora, loc, 0, "01234")
if resp.StatusCode != http.StatusNoContent || resp.Header.Get("Upload-Offset") != "5" {
t.Fatalf("a piece: %d %v", resp.StatusCode, resp.Header)
}
if resp, _ := dora.send(t, "HEAD", loc, nil, "Tus-Resumable", tusVersion); resp.Header.Get("Upload-Offset") != "5" {
t.Errorf("HEAD: %d %v", resp.StatusCode, resp.Header)
}
resp = patch(dora, loc, 5, "56789")
expectStatus(t, resp, http.StatusNoContent)
if readFile(t, f.dir, "home/dora/big.bin") != "0123456789" || !mtimeOf(t, filepath.Join(home, "big.bin")).Equal(time.Unix(1700000000, 0)) {
t.Errorf("stored: %q at %v", readFile(t, f.dir, "home/dora/big.bin"), mtimeOf(t, filepath.Join(home, "big.bin")))
}

// They replace what is there, as PUTs do, unless If-None-Match: *.
resp, loc = start(dora, "/.dav/home/dora/", "big.bin", 3)
expectStatus(t, resp, http.StatusCreated)
expectStatus(t, patch(dora, loc, 0, "abc"), http.StatusNoContent)
if readFile(t, f.dir, "home/dora/big.bin") != "abc" {
t.Error("not replaced")
}
for _, tc := range []struct {
d *davClient
dir string
name string
length int
headers []string
want int
}{
{dora, "/.dav/home/dora/", "big.bin", 3, []string{"If-None-Match", "*"}, http.StatusConflict},
{dora, "/.dav/home/dora/", "huge.bin", 1000, nil, http.StatusInsufficientStorage}, // over the quota
{dora, "/.dav/home/dora/.x/", "a.txt", 1, nil, http.StatusForbidden},
{erin, "/.dav/home/dora/", "a.txt", 1, nil, http.StatusNotFound},
} {
if resp, _ := start(tc.d, tc.dir, tc.name, tc.length, tc.headers...); resp.StatusCode != tc.want &&
!(tc.want == http.StatusNotFound && resp.StatusCode == http.StatusForbidden) {
t.Errorf("POST %s%s: %d, want %d", tc.dir, tc.name, resp.StatusCode, tc.want)
}
}
if resp, _ := dora.send(t, "POST", "/.dav/home/dora/", nil); resp.StatusCode != http.StatusMethodNotAllowed {
t.Errorf("POST without tus: %d", resp.StatusCode)
}

// The site's resumable uploads keep the time too.
admin := f.as("superadmin")
loc = startResumable(t, admin, "/", "m.txt", 1, "mtime=1600000000")
expectStatus(t, patchAt(t, admin, loc, 0, "m"), http.StatusNoContent)
if !mtimeOf(t, filepath.Join(f.dir, "m.txt")).Equal(time.Unix(1600000000, 0)) {
t.Error("the site's upload lost its time")
}
}
35 changes: 28 additions & 7 deletions internal/server/resumable.go
Original file line number Diff line number Diff line change
Expand Up @@ -90,11 +90,14 @@ func setExpires(c fiber.Ctx, up *auth.Upload) {

// createUpload starts a resumable upload into the folder at dirPath (tus
// creation). Upload-Length gives the file's size; Upload-Metadata its name
// as filename and, with replace set to 1, that it may replace a file of
// that name. Everything is checked as for other uploads, so a doomed
// upload is refused before its data is sent. Through the upload link sh,
// it has the link's rights, and a taken name gets a number instead.
func (s *Server) createUpload(c fiber.Ctx, dirPath string, sh *auth.Share) error {
// as filename, with replace set to 1 that it may replace a file of that
// name, and its modification time as mtime, in seconds since 1970. With
// overwrite, as WebDAV clients expect of uploads, it replaces a file of
// that name unless If-None-Match: * says not to. Everything is checked as
// for other uploads, so a doomed upload is refused before its data is
// sent. Through the upload link sh, it has the link's rights, and a taken
// name gets a number instead.
func (s *Server) createUpload(c fiber.Ctx, dirPath string, sh *auth.Share, overwrite bool) error {
if err := tusRequest(c); err != nil {
return err
}
Expand All @@ -111,6 +114,9 @@ func (s *Server) createUpload(c fiber.Ctx, dirPath string, sh *auth.Share) error
return explain(fiber.ErrBadRequest, badName)
}
replace := meta["replace"] == "1" && sh == nil
if overwrite {
replace = sh == nil && c.Get(fiber.HeaderIfNoneMatch) != "*"
}
if limit := int64(s.cfg.Upload.MaxSize); limit > 0 && length > limit {
return fiber.ErrRequestEntityTooLarge
}
Expand Down Expand Up @@ -158,6 +164,9 @@ func (s *Server) createUpload(c fiber.Ctx, dirPath string, sh *auth.Share) error
}

up := &auth.Upload{Dir: dirPath, Name: name, Length: length, Replace: replace, Temp: tempPrefix + rand.Text() + partialSuffix}
if mtime, ok := unixTime(meta["mtime"]); ok {
up.Mtime = mtime
}
if u := userOf(c); u != nil {
up.UserID = u.ID
}
Expand Down Expand Up @@ -190,7 +199,8 @@ func (s *Server) createUpload(c fiber.Ctx, dirPath string, sh *auth.Share) error
}
s.log.Info("upload started", zap.String("ip", c.IP()), zap.String("user", visitor(c)), zap.String("path", urlPath),
zap.Int64("length", length), zap.Int64("upload", up.ID))
c.Location(uploadsPrefix + token)
// Absolute, as some clients take it (rclone).
c.Location(c.BaseURL() + uploadsPrefix + token)
setExpires(c, up)
if length == 0 {
// Nothing will follow: the file is complete already.
Expand All @@ -202,11 +212,17 @@ func (s *Server) createUpload(c fiber.Ctx, dirPath string, sh *auth.Share) error
}

// upload finds the upload of a request to /.uploads/TOKEN: one the visitor
// started, signed in as now or anonymously. Others look missing.
// started, signed in as now or anonymously. Others look missing. Sync apps
// that started theirs over WebDAV sign in as they do there.
func (s *Server) upload(c fiber.Ctx) (*auth.Upload, error) {
if err := tusRequest(c); err != nil {
return nil, err
}
if userOf(c) == nil && strings.HasPrefix(strings.ToLower(c.Get(fiber.HeaderAuthorization)), "basic ") {
if err := s.davSignIn(c); err != nil {
return nil, err
}
}
ctx, cancel := dbContext(c)
defer cancel()
up, err := s.auth.UploadByToken(ctx, strings.TrimPrefix(c.Path(), uploadsPrefix))
Expand Down Expand Up @@ -448,6 +464,11 @@ func (s *Server) completeUpload(c fiber.Ctx, dir *os.Root, up *auth.Upload, righ
}
// Linking leaves the temp file behind.
_ = dir.Remove(up.Temp)
if !up.Mtime.IsZero() {
if err := dir.Chtimes(name, time.Now(), up.Mtime); err != nil {
s.log.Warn("keep modification time", zap.String("path", path.Join(up.Dir, name)), zap.Error(err))
}
}
ctx, cancel := context.WithTimeout(context.Background(), dbTimeout)
defer cancel()
if err := s.auth.DeleteUpload(ctx, up.ID); err != nil {
Expand Down
4 changes: 2 additions & 2 deletions internal/server/resumable_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -47,8 +47,8 @@ func startResumable(t *testing.T, f *fixture, dir, name string, length int, meta
if resp.StatusCode != http.StatusCreated {
t.Fatalf("POST %s: %d %s", dir, resp.StatusCode, body)
}
loc := resp.Header.Get("Location")
if !strings.HasPrefix(loc, uploadsPrefix) || !auth.ValidToken(strings.TrimPrefix(loc, uploadsPrefix)) ||
loc, absolute := strings.CutPrefix(resp.Header.Get("Location"), "http://example.com")
if !absolute || !strings.HasPrefix(loc, uploadsPrefix) || !auth.ValidToken(strings.TrimPrefix(loc, uploadsPrefix)) ||
resp.Header.Get("Tus-Resumable") != tusVersion || resp.Header.Get("Upload-Expires") == "" {
t.Fatalf("creation answer: %v", resp.Header)
}
Expand Down
4 changes: 3 additions & 1 deletion internal/server/templates/app.js
Original file line number Diff line number Diff line change
Expand Up @@ -851,7 +851,9 @@
let res;
const patching = url && offset !== null;
if (!url) {
const meta = "filename " + base64(file.name) + (replacing ? ",replace " + base64("1") : "");
let meta = "filename " + base64(file.name) + (replacing ? ",replace " + base64("1") : "");
// The file keeps when it was last changed.
if (file.lastModified > 0) meta += ",mtime " + base64(String(Math.floor(file.lastModified / 1000)));
res = await request(t, "POST", dir, { "Upload-Length": String(file.size), "Upload-Metadata": meta });
if (res.status === 201) {
url = new URL(res.getResponseHeader("Location"), location.href).pathname;
Expand Down
Loading
Loading