diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 62889e8ad..650c1e874 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -4,7 +4,7 @@ on: push: branches: [main] pull_request: - branches: "*" + branches: '*' jobs: check_format: @@ -54,6 +54,12 @@ jobs: - uses: crystal-lang/install-crystal@v1 with: crystal: ${{matrix.crystal_version}} + - name: Install ImageMagick (Ubuntu) + if: runner.os == 'Linux' + run: sudo apt-get install -y imagemagick + - name: Install ImageMagick (macOS) + if: runner.os == 'macOS' + run: brew install imagemagick - name: Install shards run: shards install --skip-postinstall --skip-executables env: diff --git a/Dockerfile b/Dockerfile index 33404cf33..ce8c88df5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,7 +2,7 @@ FROM crystallang/crystal:latest WORKDIR /data RUN apt-get update && \ - apt-get install -y curl libreadline-dev unzip && \ + apt-get install -y curl libreadline-dev unzip file imagemagick && \ curl -fsSL https://bun.sh/install | bash && \ ln -s /root/.bun/bin/bun /usr/local/bin/bun && \ # Cleanup leftovers diff --git a/shard.yml b/shard.yml index 4969a95ac..81fc227fe 100644 --- a/shard.yml +++ b/shard.yml @@ -1,7 +1,7 @@ name: lucky version: 1.4.0 -crystal: ">= 1.16.3" +crystal: '>= 1.16.3' authors: - Paul Smith @@ -56,5 +56,11 @@ development_dependencies: ameba: github: crystal-ameba/ameba version: ~> 1.6.4 + awscr-s3: + github: taylorfinnell/awscr-s3 + version: ~> 0.10.0 + webmock: + github: manastech/webmock.cr + branch: master license: MIT diff --git a/spec/fixtures/lucky_logo_tiny.png b/spec/fixtures/lucky_logo_tiny.png new file mode 100644 index 000000000..8772c812a Binary files /dev/null and b/spec/fixtures/lucky_logo_tiny.png differ diff --git a/spec/lucky/attachment/extractor/dimensions_from_magick_spec.cr b/spec/lucky/attachment/extractor/dimensions_from_magick_spec.cr new file mode 100644 index 000000000..84cecc065 --- /dev/null +++ b/spec/lucky/attachment/extractor/dimensions_from_magick_spec.cr @@ -0,0 +1,55 @@ +require "../../../spec_helper" + +describe Lucky::Attachment::Extractor::DimensionsFromMagick do + describe "#extract" do + subject = Lucky::Attachment::Extractor::DimensionsFromMagick.new + png_path = "spec/fixtures/lucky_logo_tiny.png" + + context "when magick is not installed" do + it "raises Lucky::Attachment::Error" do + original_path = ENV["PATH"] + ENV["PATH"] = "" + io = IO::Memory.new + + begin + expect_raises( + Lucky::Attachment::Error, + /The `magick|identify` command-line tool is not installed/ + ) do + subject.extract(io, metadata: Lucky::Attachment::MetadataHash.new) + end + ensure + ENV["PATH"] = original_path + end + end + end + + context "when magick is installed" do + it "extracts width and height from a PNG file" do + file = File.open(png_path) + metadata = Lucky::Attachment::MetadataHash.new + result = subject.extract(file, metadata: metadata) + + result.should be_nil + metadata["width"].should eq(69) + metadata["height"].should eq(16) + end + + it "does not modify metadata when magick returns no output" do + io = IO::Memory.new + metadata = Lucky::Attachment::MetadataHash.new + subject.extract(io, metadata: metadata) + + metadata.should be_empty + end + + it "rewinds the IO after reading" do + file = File.open(png_path) + metadata = Lucky::Attachment::MetadataHash.new + subject.extract(file, metadata: metadata) + + file.pos.should eq(0) + end + end + end +end diff --git a/spec/lucky/attachment/extractor/filename_from_io_spec.cr b/spec/lucky/attachment/extractor/filename_from_io_spec.cr new file mode 100644 index 000000000..bfca8d424 --- /dev/null +++ b/spec/lucky/attachment/extractor/filename_from_io_spec.cr @@ -0,0 +1,142 @@ +require "../../../spec_helper" + +describe Lucky::Attachment::Extractor::FilenameFromIO do + describe "#extract" do + subject = Lucky::Attachment::Extractor::FilenameFromIO.new + + context "when a filename is provided in options" do + it "returns the filename from options" do + io = PlainIO.new + result = subject.extract(io, metadata: nil, filename: "override.txt") + + result.should eq("override.txt") + end + + it "prefers options filename over #original_filename" do + io = IOWithOriginalFilename.new("ignored.txt") + result = subject.extract(io, metadata: nil, filename: "override.txt") + + result.should eq("override.txt") + end + end + + context "when the IO responds to #original_filename" do + it "returns the original filename" do + io = IOWithOriginalFilename.new("photo.jpg") + result = subject.extract(io, metadata: nil) + + result.should eq("photo.jpg") + end + + it "returns nil when original_filename is nil" do + io = IOWithOriginalFilename.new(nil) + result = subject.extract(io, metadata: nil) + + result.should be_nil + end + end + + context "when the IO responds to #filename but not #original_filename" do + it "returns the filename" do + io = IOWithFilename.new("document.pdf") + result = subject.extract(io, metadata: nil) + + result.should eq("document.pdf") + end + + it "returns nil when filename is blank" do + io = IOWithFilename.new("") + result = subject.extract(io, metadata: nil) + + result.should be_nil + end + + it "returns nil when filename is nil" do + io = IOWithFilename.new(nil) + result = subject.extract(io, metadata: nil) + + result.should be_nil + end + end + + context "when the IO responds to #path but not #original_filename or #filename" do + it "returns the basename of the path" do + io = IOWithPath.new("/uploads/tmp/archive.zip") + result = subject.extract(io, metadata: nil) + + result.should eq("archive.zip") + end + + it "returns just the filename when path has no directory component" do + io = IOWithPath.new("archive.zip") + result = subject.extract(io, metadata: nil) + + result.should eq("archive.zip") + end + end + + context "when the IO responds to none of the known methods" do + it "returns nil" do + io = PlainIO.new + result = subject.extract(io, metadata: nil) + + result.should be_nil + end + end + end +end + +private class PlainIO < IO + def read(slice : Bytes) + raise "not implemented" + end + + def write(slice : Bytes) : Nil + raise "not implemented" + end +end + +private class IOWithOriginalFilename < IO + getter original_filename : String? + + def initialize(@original_filename : String?) + end + + def read(slice : Bytes) + raise "not implemented" + end + + def write(slice : Bytes) : Nil + raise "not implemented" + end +end + +private class IOWithFilename < IO + getter filename : String? + + def initialize(@filename : String?) + end + + def read(slice : Bytes) + raise "not implemented" + end + + def write(slice : Bytes) : Nil + raise "not implemented" + end +end + +private class IOWithPath < IO + getter path : String + + def initialize(@path : String) + end + + def read(slice : Bytes) + raise "not implemented" + end + + def write(slice : Bytes) : Nil + raise "not implemented" + end +end diff --git a/spec/lucky/attachment/extractor/mime_from_extension_spec.cr b/spec/lucky/attachment/extractor/mime_from_extension_spec.cr new file mode 100644 index 000000000..c817ec660 --- /dev/null +++ b/spec/lucky/attachment/extractor/mime_from_extension_spec.cr @@ -0,0 +1,149 @@ +require "../../../spec_helper" + +describe Lucky::Attachment::Extractor::MimeFromExtension do + describe "#extract" do + subject = Lucky::Attachment::Extractor::MimeFromExtension.new + + context "when a filename is passed in options" do + it "uses the filename from options over the IO filename" do + io = IOWithFilename.new("ignored.png") + result = subject.extract(io, metadata: nil, filename: "overridden.pdf") + + result.should eq("application/pdf") + end + end + + context "when the IO responds to #original_filename" do + it "returns the MIME type for a known extension" do + io = IOWithOriginalFilename.new("photo.png") + result = subject.extract(io, metadata: nil) + + result.should eq("image/png") + end + + it "returns nil when original_filename is nil" do + io = IOWithOriginalFilename.new(nil) + result = subject.extract(io, metadata: nil) + + result.should be_nil + end + end + + context "when the IO responds to #filename" do + it "returns the MIME type for a known extension" do + io = IOWithFilename.new("document.pdf") + result = subject.extract(io, metadata: nil) + + result.should eq("application/pdf") + end + + it "returns nil when filename is nil" do + io = IOWithFilename.new(nil) + result = subject.extract(io, metadata: nil) + + result.should be_nil + end + + it "returns nil when filename is blank" do + io = IOWithFilename.new("") + result = subject.extract(io, metadata: nil) + + result.should be_nil + end + end + + context "when the IO responds to #path" do + it "returns the MIME type using the basename of the path" do + io = IOWithPath.new("/tmp/uploads/photo.png") + result = subject.extract(io, metadata: nil) + + result.should eq("image/png") + end + + it "handles a path with no directory component" do + io = IOWithPath.new("photo.jpg") + result = subject.extract(io, metadata: nil) + + result.should eq("image/jpeg") + end + end + + context "when the IO has no filename-related methods" do + it "returns nil" do + io = IO::Memory.new + result = subject.extract(io, metadata: nil) + + result.should be_nil + end + end + + context "when the filename has an unknown extension" do + it "returns nil" do + io = IOWithFilename.new("file.unknownextension") + result = subject.extract(io, metadata: nil) + + result.should be_nil + end + end + + context "when the filename has no extension" do + it "returns nil" do + io = IOWithFilename.new("Makefile") + result = subject.extract(io, metadata: nil) + + result.should be_nil + end + end + + context "when the filename has multiple dots" do + it "uses only the last extension" do + io = IOWithFilename.new("my.profile.photo.jpg") + result = subject.extract(io, metadata: nil) + + result.should eq("image/jpeg") + end + end + end +end + +private class IOWithOriginalFilename < IO + getter original_filename : String? + + def initialize(@original_filename : String?) + @io = IO::Memory.new + end + + delegate read, to: @io + + def write(slice : Bytes) : Nil + @io.write(slice) + end +end + +private class IOWithFilename < IO + getter filename : String? + + def initialize(@filename : String?) + @io = IO::Memory.new + end + + delegate read, to: @io + + def write(slice : Bytes) : Nil + @io.write(slice) + end +end + +private class IOWithPath < IO + getter path : String + + def initialize(@path : String) + @io = IO::Memory.new + end + + delegate read, to: @io + + def write(slice : Bytes) : Nil + @io.write(slice) + end +end diff --git a/spec/lucky/attachment/extractor/mime_from_file_spec.cr b/spec/lucky/attachment/extractor/mime_from_file_spec.cr new file mode 100644 index 000000000..9a33af4b6 --- /dev/null +++ b/spec/lucky/attachment/extractor/mime_from_file_spec.cr @@ -0,0 +1,94 @@ +require "../../../spec_helper" + +describe Lucky::Attachment::Extractor::MimeFromFile do + describe "#extract" do + subject = Lucky::Attachment::Extractor::MimeFromFile.new + + context "when the IO is empty" do + it "returns nil without invoking the file utility" do + io = IOWithSize.new("", size: 0_i64) + result = subject.extract(io, metadata: nil) + + result.should be_nil + end + end + + context "when the file utility is not installed" do + it "raises Lucky::Attachment::Error" do + original_path = ENV["PATH"] + ENV["PATH"] = "" + io = IOWithSize.new("Hello, world!") + + begin + expect_raises( + Lucky::Attachment::Error, + "The `file` command-line tool is not installed" + ) do + subject.extract(io, metadata: nil) + end + ensure + ENV["PATH"] = original_path + end + end + end + + context "when the file utility is installed" do + it "returns the MIME type for a PNG file" do + png_base64 = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8BQDwADhQGAWjR9awAAAABJRU5ErkJggg==" + png_bytes = Base64.decode(png_base64) + path = File.tempfile("test", ".png", &.write(png_bytes)).path + file = File.open(path) + result = subject.extract(file, metadata: nil) + + result.should eq("image/png") + ensure + File.delete(path) if path + end + + it "returns the MIME type for plain text" do + io = IOWithSize.new("Hello, world!") + result = subject.extract(io, metadata: nil) + + result.should eq("text/plain") + end + + it "strips surrounding whitespace from the output" do + io = IOWithSize.new("Hello, world!") + result = subject.extract(io, metadata: nil) + + result.should eq(result.try &.strip) + end + + it "rewinds the IO after reading" do + io = IOWithSize.new("Hello, world!") + subject.extract(io, metadata: nil) + + io.pos.should eq(0) + end + + it "returns nil for a nil-size IO" do + io = IOWithSize.new("hello", size: nil) + result = subject.extract(io, metadata: nil) + + result.should_not be_nil + end + end + end +end + +private class IOWithSize < IO + getter size : Int64? + + def initialize(content : String, @size : Int64? = nil) + @io = IO::Memory.new(content) + @size ||= content.bytesize.to_i64 + end + + delegate read, to: @io + delegate rewind, to: @io + delegate pos, to: @io + + def write(slice : Bytes) : Nil + @io.write(slice) + end +end diff --git a/spec/lucky/attachment/extractor/mime_from_io_spec.cr b/spec/lucky/attachment/extractor/mime_from_io_spec.cr new file mode 100644 index 000000000..2e4fef534 --- /dev/null +++ b/spec/lucky/attachment/extractor/mime_from_io_spec.cr @@ -0,0 +1,77 @@ +require "../../../spec_helper" + +describe Lucky::Attachment::Extractor::MimeFromIO do + describe "#extract" do + subject = Lucky::Attachment::Extractor::MimeFromIO.new + + context "when the IO responds to #content_type" do + context "and content_type is a plain MIME type" do + it "returns the MIME type" do + io = IOWithContentType.new("image/png") + result = subject.extract(io, metadata: nil) + + result.should eq("image/png") + end + end + + context "and content_type includes parameters (e.g. charset)" do + it "strips parameters and returns only the MIME type" do + io = IOWithContentType.new("text/plain; charset=utf-8") + result = subject.extract(io, metadata: nil) + + result.should eq("text/plain") + end + end + + context "and content_type includes multiple parameters" do + it "strips all parameters and returns only the MIME type" do + io = IOWithContentType.new("multipart/form-data; boundary=something; charset=utf-8") + result = subject.extract(io, metadata: nil) + + result.should eq("multipart/form-data") + end + end + + context "and content_type has surrounding whitespace" do + it "strips whitespace from the MIME type" do + io = IOWithContentType.new(" image/jpeg ; quality=80") + result = subject.extract(io, metadata: nil) + + result.should eq("image/jpeg") + end + end + + context "and content_type is nil" do + it "returns nil" do + io = IOWithContentType.new(nil) + result = subject.extract(io, metadata: nil) + + result.should be_nil + end + end + end + + context "when the IO does not respond to #content_type" do + it "returns nil" do + io = IO::Memory.new + result = subject.extract(io, metadata: nil) + + result.should be_nil + end + end + end +end + +private class IOWithContentType < IO + getter content_type : String? + + def initialize(@content_type : String?) + @io = IO::Memory.new + end + + delegate read, to: @io + + def write(slice : Bytes) : Nil + @io.write(slice) + end +end diff --git a/spec/lucky/attachment/extractor/size_from_io_spec.cr b/spec/lucky/attachment/extractor/size_from_io_spec.cr new file mode 100644 index 000000000..68d3de8a6 --- /dev/null +++ b/spec/lucky/attachment/extractor/size_from_io_spec.cr @@ -0,0 +1,100 @@ +require "../../../spec_helper" + +describe Lucky::Attachment::Extractor::SizeFromIO do + describe "#extract" do + subject = Lucky::Attachment::Extractor::SizeFromIO.new + + context "when the IO responds to #tempfile" do + it "returns the size of the tempfile" do + tempfile = File.tempfile("test") + tempfile.print("hello lucky") + tempfile.flush + + begin + io = IOWithTempfile.new(tempfile) + result = subject.extract(io, metadata: nil) + result.should eq(11_i64) + ensure + tempfile.delete + end + end + + it "prefers #tempfile over #size when both are present" do + tempfile = File.tempfile("test") + tempfile.print("hello") + tempfile.flush + + begin + io = IOWithTempfile.new(tempfile) + result = subject.extract(io, metadata: nil) + result.should eq(5_i64) + ensure + tempfile.delete + end + end + end + + context "when the IO responds to #size but not #tempfile" do + it "returns the size" do + io = IOWithSize.new(42_i64) + result = subject.extract(io, metadata: nil) + + result.should eq(42_i64) + end + + it "returns 0 for an empty IO" do + io = IOWithSize.new(0_i64) + result = subject.extract(io, metadata: nil) + + result.should eq(0_i64) + end + end + + context "when the IO responds to neither #tempfile nor #size" do + it "returns nil" do + io = PlainIO.new + result = subject.extract(io, metadata: nil) + + result.should be_nil + end + end + end +end + +private class IOWithTempfile < IO + getter tempfile : File + + def initialize(@tempfile : File) + end + + delegate read, to: @tempfile + + def write(slice : Bytes) : Nil + @tempfile.write(slice) + end +end + +private class IOWithSize < IO + getter size : Int64 + + def initialize(@size : Int64) + end + + def read(slice : Bytes) + raise "not implemented" + end + + def write(slice : Bytes) : Nil + raise "not implemented" + end +end + +private class PlainIO < IO + def read(slice : Bytes) + raise "not implemented" + end + + def write(slice : Bytes) : Nil + raise "not implemented" + end +end diff --git a/spec/lucky/attachment/storage/file_system_spec.cr b/spec/lucky/attachment/storage/file_system_spec.cr new file mode 100644 index 000000000..e7bf8b937 --- /dev/null +++ b/spec/lucky/attachment/storage/file_system_spec.cr @@ -0,0 +1,131 @@ +require "../../../spec_helper" + +describe Lucky::Attachment::Storage::FileSystem do + temp_dir = File.tempname("lucky_attachment_spec") + + before_each do + Dir.mkdir_p(temp_dir) + end + + after_each do + FileUtils.rm_rf(temp_dir) + end + + describe "#upload and #open" do + it "writes and reads file content" do + storage = Lucky::Attachment::Storage::FileSystem.new(temp_dir) + storage.upload(IO::Memory.new("file content"), "test.txt") + + storage.open("test.txt").gets_to_end.should eq("file content") + end + + it "creates intermediate directories" do + storage = Lucky::Attachment::Storage::FileSystem.new(temp_dir) + storage.upload(IO::Memory.new("data"), "a/b/c/test.txt") + + Dir.exists?(File.join(temp_dir, "a/b/c")).should be_true + end + + it "respects the prefix" do + storage = Lucky::Attachment::Storage::FileSystem.new(temp_dir, prefix: "cache") + storage.upload(IO::Memory.new("data"), "test.txt") + + File.exists?(File.join(temp_dir, "cache", "test.txt")).should be_true + end + end + + describe "#open" do + it "raises FileNotFound for missing files" do + storage = Lucky::Attachment::Storage::FileSystem.new(temp_dir) + + expect_raises(Lucky::Attachment::FileNotFound, /missing\.txt/) do + storage.open("missing.txt") + end + end + end + + describe "#exists?" do + it "returns true for existing files" do + storage = Lucky::Attachment::Storage::FileSystem.new(temp_dir) + storage.upload(IO::Memory.new("data"), "test.txt") + + storage.exists?("test.txt").should be_true + end + + it "returns false for missing files" do + storage = Lucky::Attachment::Storage::FileSystem.new(temp_dir) + + storage.exists?("missing.txt").should be_false + end + end + + describe "#delete" do + it "removes the file" do + storage = Lucky::Attachment::Storage::FileSystem.new(temp_dir) + storage.upload(IO::Memory.new("data"), "test.txt") + storage.delete("test.txt") + + storage.exists?("test.txt").should be_false + end + + it "cleans up empty parent directories by default" do + storage = Lucky::Attachment::Storage::FileSystem.new(temp_dir) + storage.upload(IO::Memory.new("data"), "a/b/test.txt") + storage.delete("a/b/test.txt") + + Dir.exists?(File.join(temp_dir, "a/b")).should be_false + Dir.exists?(File.join(temp_dir, "a")).should be_false + end + + it "does not clean non-empty parent directories" do + storage = Lucky::Attachment::Storage::FileSystem.new(temp_dir) + storage.upload(IO::Memory.new("data"), "a/b/test1.txt") + storage.upload(IO::Memory.new("data"), "a/b/test2.txt") + storage.delete("a/b/test1.txt") + + Dir.exists?(File.join(temp_dir, "a/b")).should be_true + end + + it "skips cleanup when clean is false" do + storage = Lucky::Attachment::Storage::FileSystem.new(temp_dir, clean: false) + storage.upload(IO::Memory.new("data"), "a/b/test.txt") + storage.delete("a/b/test.txt") + + Dir.exists?(File.join(temp_dir, "a/b")).should be_true + end + + it "does not raise when deleting a missing file" do + storage = Lucky::Attachment::Storage::FileSystem.new(temp_dir) + + storage.delete("nonexistent.txt") + end + end + + describe "#url" do + it "returns a path from the root" do + storage = Lucky::Attachment::Storage::FileSystem.new(temp_dir) + + storage.url("test.txt").should eq("/test.txt") + end + + it "includes the prefix in the URL" do + storage = Lucky::Attachment::Storage::FileSystem.new(temp_dir, prefix: "uploads/cache") + + storage.url("test.txt").should eq("/uploads/cache/test.txt") + end + + it "prepends host when provided" do + storage = Lucky::Attachment::Storage::FileSystem.new(temp_dir) + + storage.url("test.txt", host: "https://example.com").should eq("https://example.com/test.txt") + end + end + + describe "#path_for" do + it "returns the full filesystem path" do + storage = Lucky::Attachment::Storage::FileSystem.new(temp_dir) + + storage.path_for("test.txt").should eq(File.join(File.expand_path(temp_dir), "test.txt")) + end + end +end diff --git a/spec/lucky/attachment/storage/memory_spec.cr b/spec/lucky/attachment/storage/memory_spec.cr new file mode 100644 index 000000000..451e1bf68 --- /dev/null +++ b/spec/lucky/attachment/storage/memory_spec.cr @@ -0,0 +1,94 @@ +require "../../../spec_helper" + +describe Lucky::Attachment::Storage::Memory do + describe "#upload and #open" do + it "stores and retrieves file content" do + storage = Lucky::Attachment::Storage::Memory.new + io = IO::Memory.new("hello world") + storage.upload(io, "test.txt") + + result = storage.open("test.txt") + result.gets_to_end.should eq("hello world") + end + + it "overwrites existing content" do + storage = Lucky::Attachment::Storage::Memory.new + storage.upload(IO::Memory.new("original"), "test.txt") + storage.upload(IO::Memory.new("updated"), "test.txt") + + storage.open("test.txt").gets_to_end.should eq("updated") + end + end + + describe "#open" do + it "raises FileNotFound for missing files" do + storage = Lucky::Attachment::Storage::Memory.new + + expect_raises(Lucky::Attachment::FileNotFound, /missing\.txt/) do + storage.open("missing.txt") + end + end + end + + describe "#exists?" do + it "returns true for existing files" do + storage = Lucky::Attachment::Storage::Memory.new + storage.upload(IO::Memory.new("test"), "test.txt") + + storage.exists?("test.txt").should be_true + end + + it "returns false for non-existing files" do + storage = Lucky::Attachment::Storage::Memory.new + + storage.exists?("missing.txt").should be_false + end + end + + describe "#delete" do + it "removes the file" do + storage = Lucky::Attachment::Storage::Memory.new + storage.upload(IO::Memory.new("test"), "test.txt") + storage.delete("test.txt") + + storage.exists?("test.txt").should be_false + end + + it "does not raise when deleting a missing file" do + storage = Lucky::Attachment::Storage::Memory.new + + storage.delete("nonexistent.txt") + end + end + + describe "#url" do + it "returns path without base_url" do + storage = Lucky::Attachment::Storage::Memory.new + + storage.url("path/to/file.jpg").should eq("/path/to/file.jpg") + end + + it "prepends base_url when configured" do + storage = Lucky::Attachment::Storage::Memory.new(base_url: "https://cdn.example.com") + + storage.url("path/to/file.jpg").should eq("https://cdn.example.com/path/to/file.jpg") + end + + it "handles base_url with trailing slash" do + storage = Lucky::Attachment::Storage::Memory.new(base_url: "https://cdn.example.com/") + + storage.url("path/to/file.jpg").should eq("https://cdn.example.com/path/to/file.jpg") + end + end + + describe "#clear!" do + it "removes all files" do + storage = Lucky::Attachment::Storage::Memory.new + storage.upload(IO::Memory.new("a"), "a.txt") + storage.upload(IO::Memory.new("b"), "b.txt") + storage.clear! + + storage.size.should eq(0) + end + end +end diff --git a/spec/lucky/attachment/storage/s3_spec.cr b/spec/lucky/attachment/storage/s3_spec.cr new file mode 100644 index 000000000..e302d9005 --- /dev/null +++ b/spec/lucky/attachment/storage/s3_spec.cr @@ -0,0 +1,436 @@ +require "../../../spec_helper" +require "webmock" + +describe Lucky::Attachment::Storage::S3 do + after_each do + WebMock.reset + end + + describe "#object_key" do + it "returns the id when no prefix is configured" do + storage = build_storage + + storage.object_key("photo.jpg").should eq("photo.jpg") + end + + it "prepends the prefix" do + storage = build_storage(prefix: "cache") + + storage.object_key("photo.jpg").should eq("cache/photo.jpg") + end + + it "strips extra slashes from prefix and id" do + storage = build_storage(prefix: "/cache/") + + storage.object_key("/photo.jpg").should eq("cache/photo.jpg") + end + end + + describe "#upload and #open" do + it "writes and reads file content" do + storage = build_storage + WebMock.stub(:put, "#{base_url}/test.txt") + .to_return(status: 200, headers: test_headers) + WebMock.stub(:get, "#{base_url}/test.txt") + .to_return(status: 200, body_io: test_file_io, headers: test_headers) + + storage.upload(test_file_io, "test.txt") + + storage.open("test.txt").gets_to_end.should eq("file content") + end + + it "respects the prefix when uploading" do + storage = build_storage(prefix: "cache") + WebMock.stub(:put, "#{base_url}/cache/test.txt") + .to_return(status: 200, headers: test_headers) + + storage.upload(IO::Memory.new("data"), "test.txt") + end + end + + describe "#open" do + it "raises FileNotFound when the object does not exist" do + storage = build_storage + WebMock.stub(:get, "#{base_url}/missing.txt") + .to_return(status: 404, body: s3_error_xml("NoSuchKey", "Missing.")) + + expect_raises(Lucky::Attachment::FileNotFound, /missing\.txt/) do + storage.open("missing.txt") + end + end + + it "returns a rewindable IO" do + storage = build_storage + WebMock.stub(:get, "#{base_url}/test.txt") + .to_return(status: 200, body_io: test_file_io("rewindable")) + + io = storage.open("test.txt") + io.gets_to_end.should eq("rewindable") + io.rewind + io.gets_to_end.should eq("rewindable") + end + end + + describe "#exists?" do + it "returns true when the object exists" do + storage = build_storage + WebMock.stub(:head, "#{base_url}/photo.jpg").to_return(status: 200) + + storage.exists?("photo.jpg").should be_true + end + + it "returns false when the object does not exist" do + storage = build_storage + WebMock.stub(:head, "#{base_url}/missing.txt").to_return(status: 404) + + storage.exists?("missing.txt").should be_false + end + end + + describe "#delete" do + it "sends a DELETE request for the object" do + storage = build_storage + WebMock.stub(:delete, "#{base_url}/photo.jpg").to_return(status: 204) + + storage.delete("photo.jpg") + end + + it "uses the prefix in the DELETE path" do + storage = build_storage(prefix: "cache") + WebMock.stub(:delete, "#{base_url}/cache/photo.jpg").to_return(status: 204) + + storage.delete("photo.jpg") + end + end + + describe "#url" do + describe "public URL (no expires_in)" do + it "returns a standard AWS S3 URL" do + storage = build_storage + + storage.url("photo.jpg").should eq( + "https://s3-eu-west-1.amazonaws.com/lucky-bucket/photo.jpg" + ) + end + + it "includes the prefix in the key" do + storage = build_storage(prefix: "store") + + storage.url("photo.jpg").should eq( + "https://s3-eu-west-1.amazonaws.com/lucky-bucket/store/photo.jpg" + ) + end + + it "returns a custom-emdpoint URL for S3-compatible services" do + storage = build_rustfs_storage + + storage.url("photo.jpg").should eq( + "http://localhost:9000/lucky-bucket/photo.jpg" + ) + end + + it "includes non-standard ports in the URL" do + storage = Lucky::Attachment::Storage::S3.new( + bucket: "lucky-bucket", + region: "eu-west-1", + access_key_id: "key", + secret_access_key: "secret", + endpoint: "http://localhost:9000" + ) + + storage.url("photo.jpg").should contain("http://localhost:9000") + end + + it "omits standard ports from the URL" do + storage = Lucky::Attachment::Storage::S3.new( + bucket: "lucky-bucket", + region: "eu-west-1", + access_key_id: "key", + secret_access_key: "secret", + endpoint: "https://s3.example.com:443" + ) + + storage.url("photo.jpg").should eq( + "https://s3.example.com/lucky-bucket/photo.jpg" + ) + end + end + + describe "#move" do + it "falls back to upload for a plain IO" do + storage = build_storage + WebMock.stub(:put, "#{base_url}/dest.txt") + .to_return(status: 200, headers: test_headers) + + storage.move(IO::Memory.new("data"), "dest.txt") + end + + it "uses server-side copy and deletes the source for a same-bucket StoredFile" do + storage = build_storage + WebMock.stub(:put, "#{base_url}/store/photo.jpg") + .to_return(status: 200, headers: test_headers, body: copy_object_xml) + WebMock.stub(:delete, "#{base_url}/cache/photo.jpg") + .to_return(status: 204) + Lucky::Attachment.settings.storages["cache"] = storage + + source = TestUploader::StoredFile.new( + id: "cache/photo.jpg", + storage_key: "cache", + metadata: Lucky::Attachment::MetadataHash.new + ) + + storage.move(source, "store/photo.jpg") + end + + it "falls back to upload for a StoredFile from a different bucket" do + storage = build_storage + other_storage = build_storage(bucket: "other-bucket") + Lucky::Attachment.settings.storages["other"] = other_storage + WebMock.stub(:get, "https://s3-eu-west-1.amazonaws.com/other-bucket/photo.jpg") + .to_return(status: 200, body_io: test_file_io("data"), headers: test_headers) + WebMock.stub(:put, "#{base_url}/photo.jpg") + .to_return(status: 200, headers: test_headers) + source = TestUploader::StoredFile.new( + id: "photo.jpg", + storage_key: "other", + metadata: Lucky::Attachment::MetadataHash.new + ) + + storage.move(source, "photo.jpg") + end + + it "uses only the object key (not double-prefixed) when copying a same-bucket StoredFile" do + storage = build_storage(prefix: "store") + Lucky::Attachment.settings.storages["store"] = storage + WebMock.stub(:put, "#{base_url}/store/photo.jpg?") + .to_return(status: 200, body: copy_object_xml) + WebMock.stub(:delete, "#{base_url}/store/photo.jpg?") + .to_return(status: 204) + + source = TestUploader::StoredFile.new( + id: "photo.jpg", + storage_key: "store", + metadata: Lucky::Attachment::MetadataHash.new + ) + + storage.move(source, "photo.jpg") + end + end + + describe "presigned URL (with expires_in)" do + it "returns a URL containing signature query parameters" do + storage = build_storage + url = storage.url("photo.jpg", expires_in: 3600) + + url.should contain("X-Amz-Signature") + url.should contain("X-Amz-Expires=3600") + url.should contain("photo.jpg") + end + + it "includes the prefix in the presigned key" do + storage = build_storage(prefix: "cache") + url = storage.url("photo.jpg", expires_in: 3600) + + url.should contain("cache/photo.jpg") + end + + it "uses the custom endpoint host for presigned URLs" do + storage = build_rustfs_storage + url = storage.url("photo.jpg", expires_in: 3600) + + url.should contain("localhost:9000") + end + end + end + + describe "upload headers" do + it "sets Content-Disposition from metadata filename" do + client = TestAwss3Client.new + build_storage(client: client).upload( + IO::Memory.new, "test-id", + metadata: Lucky::Attachment::MetadataHash{"filename" => "photo.jpg"}, + ) + + client.headers["Content-Disposition"] + .should eq(%(inline; filename="photo.jpg")) + end + + it "sets Content-Type from metadata mime_type" do + client = TestAwss3Client.new + build_storage(client: client).upload( + IO::Memory.new, "test-id", + metadata: Lucky::Attachment::MetadataHash{"mime_type" => "image/jpeg"}, + ) + + client.headers["Content-Type"].should eq("image/jpeg") + end + + it "allows content_type to override metadata mime_type" do + client = TestAwss3Client.new + build_storage(client: client).upload( + IO::Memory.new, "test-id", + metadata: Lucky::Attachment::MetadataHash{"mime_type" => "image/jpeg"}, + content_type: "application/octet-stream" + ) + + client.headers["Content-Type"].should eq("application/octet-stream") + end + + it "allows content_disposition to override metadata filename" do + client = TestAwss3Client.new + build_storage(client: client).upload( + IO::Memory.new, "test-id", + metadata: Lucky::Attachment::MetadataHash{"filename" => "photo.jpg"}, + content_disposition: "attachment" + ) + + client.headers["Content-Disposition"].should eq("attachment") + end + + it "sets x-amz-acl: public-read when storage is public" do + client = TestAwss3Client.new + build_storage(client: client, public: true) + .upload(IO::Memory.new, "test-id") + + client.headers["x-amz-acl"].should eq("public-read") + end + + it "merges upload_options" do + client = TestAwss3Client.new + build_storage( + client: client, + upload_options: {"Cache-Control" => "max-age=31536000"} + ).upload(IO::Memory.new, "test-id") + + client.headers["Cache-Control"].should eq("max-age=31536000") + end + + it "per-call options take precedence over upload_options" do + client = TestAwss3Client.new + build_storage( + client: client, + upload_options: {"Content-Type" => "application/octet-stream"} + ).upload(IO::Memory.new, "test-id", content_type: "image/jpeg") + + client.headers["Content-Type"].should eq("image/jpeg") + end + + it "upload_options do not override metadata Content-Disposition" do + client = TestAwss3Client.new + build_storage( + client: client, + upload_options: {"Content-Disposition" => "attachment"} + ).upload( + IO::Memory.new, "test-id", + metadata: Lucky::Attachment::MetadataHash{"filename" => "photo.jpg"} + ) + + client.headers["Content-Disposition"] + .should eq(%(inline; filename="photo.jpg")) + end + end +end + +private class TestAwss3Client < Awscr::S3::Client + SIGNER = Awscr::Signer::Signers::V4.new("blah", "blah", "blah", "blah") + + getter headers = {} of String => String + + def initialize( + @region = "eu-west-1", + @aws_access_key = "test-key", + @aws_secret_key = "test-secret", + @endpoint = URI.new, + @signer = SIGNER, + @http = Awscr::S3::Http.new(SIGNER, URI.new), + ) + end + + def put_object(_bucket, _id, _io, @headers) + end +end + +private struct TestUploader < Lucky::Attachment::Uploader; end + +private def bucket + "lucky-bucket" +end + +private def region + "eu-west-1" +end + +private def base_url + "https://s3-#{region}.amazonaws.com/#{bucket}" +end + +private def test_headers(headers = {} of String => String) + {"ETag" => %("abc123")}.merge(headers) +end + +private def test_file_io(content = "file content") + IO::Memory.new(content) +end + +private def copy_object_xml + <<-XML + + + "abc123" + 2026-03-01T10:08:56.000Z + + XML +end + +private def s3_error_xml(code : String, message : String) : String + <<-XML + + + #{code} + #{message} + + XML +end + +private def build_storage( + bucket = "lucky-bucket", + region = "eu-west-1", + access_key_id = "test-key", + secret_access_key = "test-secret", + prefix = nil, + public = false, + upload_options = Hash(String, String).new, + endpoint = nil, + client = nil, +) + if s3_client = client + Lucky::Attachment::Storage::S3.new( + bucket: bucket, + client: s3_client, + prefix: prefix, + public: public, + upload_options: upload_options + ) + else + Lucky::Attachment::Storage::S3.new( + bucket: bucket, + region: region, + access_key_id: "test-key", + secret_access_key: "test-secret", + prefix: prefix, + endpoint: endpoint, + public: public, + upload_options: upload_options + ) + end +end + +private def build_rustfs_storage(bucket = "lucky-bucket", prefix = nil) + build_storage( + bucket: bucket, + access_key_id: "rustfsadmin", + secret_access_key: "rustfsadmin", + prefix: prefix, + endpoint: "http://localhost:9000" + ) +end diff --git a/spec/lucky/attachment/stored_file_spec.cr b/spec/lucky/attachment/stored_file_spec.cr new file mode 100644 index 000000000..433c3ab6c --- /dev/null +++ b/spec/lucky/attachment/stored_file_spec.cr @@ -0,0 +1,241 @@ +require "../../spec_helper" + +describe Lucky::Attachment::StoredFile do + memory_store = Lucky::Attachment::Storage::Memory.new(base_url: "https://example.com") + + before_each do + memory_store.clear! + + Lucky::Attachment.configure do |settings| + settings.storages["store"] = memory_store + end + end + + describe ".from_json" do + it "deserializes from JSON" do + file = TestUploader::StoredFile.from_json( + { + id: "test.jpg", + storage: "store", + metadata: { + filename: "original.jpg", + size: 1024_i64, + mime_type: "image/jpeg", + }, + }.to_json + ) + + file.id.should eq("test.jpg") + file.storage_key.should eq("store") + file.filename.should eq("original.jpg") + file.size.should eq(1024) + file.mime_type.should eq("image/jpeg") + end + end + + describe "#to_json" do + it "serializes to JSON" do + file = TestUploader::StoredFile.new( + id: "test.jpg", + storage_key: "store", + metadata: Lucky::Attachment::MetadataHash{ + "filename" => "original.jpg", + "size" => 1024_i64, + "mime_type" => "image/jpeg", + } + ) + parsed = JSON.parse(file.to_json) + + parsed["id"].should eq("test.jpg") + parsed["storage"].should eq("store") + parsed["metadata"]["size"].should eq(1024_i64) + parsed["metadata"]["filename"].should eq("original.jpg") + parsed["metadata"]["mime_type"].should eq("image/jpeg") + end + end + + describe "#extension" do + it "extracts from id" do + file = TestUploader::StoredFile.new( + id: "path/to/file.jpg", + storage_key: "store" + ) + + file.extension.should eq("jpg") + end + + it "falls back to filename metadata" do + file = TestUploader::StoredFile.new( + id: "abc123", + storage_key: "store", + metadata: Lucky::Attachment::MetadataHash{"filename" => "photo.png"} + ) + + file.extension.should eq("png") + end + + it "returns nil when no extension can be determined" do + file = TestUploader::StoredFile.new( + id: "abc123", + storage_key: "store" + ) + + file.extension?.should be_nil + end + end + + describe "#size" do + it "returns Int64 from integer metadata" do + file = TestUploader::StoredFile.new( + id: "file.jpg", + storage_key: "store", + metadata: Lucky::Attachment::MetadataHash{"size" => 1024_i64} + ) + + file.size.should eq(1024_i64) + file.size.should be_a(Int64) + end + + it "coerces Int32 to Int64" do + file = TestUploader::StoredFile.new( + id: "file.jpg", + storage_key: "store", + metadata: Lucky::Attachment::MetadataHash{"size" => 512_i32} + ) + + file.size.should eq(512_i64) + file.size.should be_a(Int64) + end + + it "returns nil when size is absent" do + file = TestUploader::StoredFile.new( + id: "file.jpg", + storage_key: "store" + ) + + file.size?.should be_nil + end + end + + describe "#url" do + it "delegates to storage" do + file = TestUploader::StoredFile.new( + id: "uploads/photo.jpg", + storage_key: "store" + ) + + file.url.should eq("https://example.com/uploads/photo.jpg") + end + end + + describe "#exists?" do + it "returns true when file is in storage" do + memory_store.upload(IO::Memory.new("data"), "photo.jpg") + file = TestUploader::StoredFile.new( + id: "photo.jpg", + storage_key: "store" + ) + + file.exists?.should be_true + end + + it "returns false when file is not in storage" do + file = TestUploader::StoredFile.new( + id: "missing.jpg", + storage_key: "store" + ) + file.exists?.should be_false + end + end + + describe "#open" do + it "yields the file IO" do + memory_store.upload(IO::Memory.new("file content"), "test.txt") + file = TestUploader::StoredFile.new( + id: "test.txt", + storage_key: "store" + ) + + file.open(&.gets_to_end.should(eq("file content"))) + end + + it "closes the IO after the block" do + memory_store.upload(IO::Memory.new("data"), "test.txt") + file = TestUploader::StoredFile.new(id: "test.txt", storage_key: "store") + captured_io = nil + file.open { |io| captured_io = io } + + captured_io.as(IO).closed?.should be_true + end + + it "closes the IO even if the block raises" do + memory_store.upload(IO::Memory.new("data"), "test.txt") + file = TestUploader::StoredFile.new(id: "test.txt", storage_key: "store") + captured_io = nil + + expect_raises(Exception) do + file.open do |io| + captured_io = io + raise "oops" + end + end + captured_io.as(IO).closed?.should be_true + end + + describe "#download" do + it "returns a tempfile with file content" do + memory_store.upload(IO::Memory.new("downloaded content"), "test.txt") + file = TestUploader::StoredFile.new(id: "test.txt", storage_key: "store") + tempfile = file.download + + tempfile.gets_to_end.should eq("downloaded content") + tempfile.close + tempfile.delete + end + + it "cleans up the tempfile after the block" do + memory_store.upload(IO::Memory.new("data"), "test.txt") + file = TestUploader::StoredFile.new(id: "test.txt", storage_key: "store") + tempfile_path = "" + file.download { |tempfile| tempfile_path = tempfile.path } + + File.exists?(tempfile_path).should be_false + end + end + end + + describe "#delete" do + it "removes the file from storage" do + memory_store.upload(IO::Memory.new("data"), "test.txt") + file = TestUploader::StoredFile.new(id: "test.txt", storage_key: "store") + file.delete + + memory_store.exists?("test.txt").should be_false + end + end + + describe "#==" do + it "is equal when id and storage_key match" do + a = TestUploader::StoredFile.new(id: "file.jpg", storage_key: "store") + b = TestUploader::StoredFile.new(id: "file.jpg", storage_key: "store") + + (a == b).should be_true + end + + it "is not equal when id differs" do + a = TestUploader::StoredFile.new(id: "a.jpg", storage_key: "store") + b = TestUploader::StoredFile.new(id: "b.jpg", storage_key: "store") + + (a == b).should be_false + end + + it "is not equal when storage_key differs" do + a = TestUploader::StoredFile.new(id: "file.jpg", storage_key: "cache") + b = TestUploader::StoredFile.new(id: "file.jpg", storage_key: "store") + + (a == b).should be_false + end + end +end + +private struct TestUploader < Lucky::Attachment::Uploader; end diff --git a/spec/lucky/attachment/uploader_spec.cr b/spec/lucky/attachment/uploader_spec.cr new file mode 100644 index 000000000..7d10f0d93 --- /dev/null +++ b/spec/lucky/attachment/uploader_spec.cr @@ -0,0 +1,285 @@ +require "../../spec_helper" + +describe Lucky::Attachment::Uploader do + memory_cache = Lucky::Attachment::Storage::Memory.new + memory_store = Lucky::Attachment::Storage::Memory.new + + before_each do + memory_cache.clear! + memory_store.clear! + + Lucky::Attachment.configure do |settings| + settings.storages["cache"] = memory_cache + settings.storages["store"] = memory_store + end + end + + describe "#upload" do + context "with a basic IO" do + it "uploads and returns a stored file" do + io = IO::Memory.new("hello") + file = TestUploader.new("store").upload(io) + + file.should be_a(TestUploader::StoredFile) + file.storage_key.should eq("store") + file.exists?.should be_true + end + + it "generates a unique location each time" do + file_a = TestUploader.new("store").upload(IO::Memory.new("a")) + file_b = TestUploader.new("store").upload(IO::Memory.new("b")) + + file_a.id.should_not eq(file_b.id) + end + + it "extracts size metadata" do + io = IO::Memory.new("hello world") + file = TestUploader.new("store").upload(io) + + file.size.should eq(11) + end + + it "preserves extension in the location" do + io = IO::Memory.new("data") + file = TestUploader.new("store").upload( + io, + metadata: Lucky::Attachment::MetadataHash{"filename" => "photo.jpg"} + ) + + file.id.should end_with(".jpg") + end + + it "accepts a custom location" do + io = IO::Memory.new("data") + file = TestUploader.new("store").upload(io, location: "my/custom/path.jpg") + + file.id.should eq("my/custom/path.jpg") + end + + it "merges provided metadata with extracted metadata" do + io = IO::Memory.new("data") + file = TestUploader.new("store").upload( + io, + metadata: Lucky::Attachment::MetadataHash{ + "filename" => "override.png", + "custom" => "value", + } + ) + + file.filename.should eq("override.png") + file["custom"]?.should eq("value") + end + end + + context "with a File IO" do + it "extracts filename from path" do + file = File.tempfile("myfile", ".txt", &.print("content")) + uploaded = TestUploader.new("store").upload(File.open(file.path)) + + uploaded.filename.should eq(File.basename(file.path)) + ensure + file.try(&.delete) + end + + it "extracts size" do + file = File.tempfile("myfile", ".txt", &.print("content")) + uploaded = TestUploader.new("store").upload(File.open(file.path)) + + uploaded.size.should eq(7) + ensure + file.try(&.delete) + end + end + + context "error handling" do + it "raises Error when no storages are not configured" do + Lucky::Attachment.configure do |settings| + settings.storages = {} of String => Lucky::Attachment::Storage + end + + expect_raises( + Lucky::Attachment::Error, + "There are no storages registered yet" + ) do + TestUploader.new("store").upload(IO::Memory.new("data")) + end + end + + it "raises Error when a storage is not configured" do + expect_raises( + Lucky::Attachment::Error, + %(Storage "missing" is not registered. The available storages are: "cache", "store") + ) do + TestUploader.new("missing").upload(IO::Memory.new("data")) + end + end + end + end + + describe "custom uploader behaviour" do + it "uses overridden generate_location" do + file = CustomLocationUploader.new("store").upload(IO::Memory.new("data")) + + file.id.should start_with("custom/") + end + + it "uses overridden extract_metadata" do + file = CustomMetadataUploader.new("store").upload(IO::Memory.new("data")) + + file["custom_key"]?.should eq("custom_value") + end + end + + describe ".cache" do + it "uploads to the cache storage" do + file = TestUploader.cache(IO::Memory.new("data")) + + file.storage_key.should eq("cache") + memory_cache.exists?(file.id).should be_true + end + end + + describe ".store" do + it "uploads to the store storage" do + file = TestUploader.store(IO::Memory.new("data")) + + file.storage_key.should eq("store") + memory_store.exists?(file.id).should be_true + end + end + + describe ".promote" do + it "moves a cached file to the store" do + cached = TestUploader.cache(IO::Memory.new("data")) + stored = TestUploader.promote(cached) + + stored.storage_key.should eq("store") + memory_store.exists?(stored.id).should be_true + end + + it "deletes the source file by default" do + cached = TestUploader.cache(IO::Memory.new("data")) + cached_id = cached.id + TestUploader.promote(cached) + + memory_cache.exists?(cached_id).should be_false + end + + it "preserves the source when delete_source is false" do + cached = TestUploader.cache(IO::Memory.new("data")) + cached_id = cached.id + TestUploader.promote(cached, delete_source: false) + + memory_cache.exists?(cached_id).should be_true + end + + it "preserves the file id across storages" do + cached = TestUploader.cache(IO::Memory.new("data")) + stored = TestUploader.promote(cached) + + stored.id.should eq(cached.id) + end + + it "preserves metadata" do + cached = TestUploader.cache( + IO::Memory.new("data"), + metadata: Lucky::Attachment::MetadataHash{"filename" => "test.jpg"} + ) + stored = TestUploader.promote(cached) + + stored.filename.should eq("test.jpg") + end + + it "can promote to a custom storage key" do + Lucky::Attachment.configure do |settings| + settings.storages["cache"] = memory_cache + settings.storages["store"] = memory_store + settings.storages["offsite"] = Lucky::Attachment::Storage::Memory.new + end + cached = TestUploader.cache(IO::Memory.new("data")) + offsite = TestUploader.promote(cached, to: "offsite") + + offsite.storage_key.should eq("offsite") + end + end +end + +describe "Lucky::UploadedFile integration" do + memory_store = Lucky::Attachment::Storage::Memory.new + + before_each do + memory_store.clear! + Lucky::Attachment.configure do |settings| + settings.storages["store"] = memory_store + end + end + + it "extracts filename from Lucky::UploadedFile" do + part = build_form_data_part("avatar", "photo.jpg", "image/jpeg", "data") + lucky_file = Lucky::UploadedFile.new(part) + uploaded = AvatarUploader.new("store").upload(lucky_file.tempfile) + + uploaded.filename.should eq(File.basename(lucky_file.tempfile.path)) + end + + it "extracts content_type when Lucky::UploadedFile exposes it" do + part = build_form_data_part("avatar", "photo.jpg", "image/jpeg", "data") + lucky_file = Lucky::UploadedFile.new(part) + uploaded = AvatarUploader.new("store").upload(lucky_file.tempfile) + + uploaded.mime_type?.should be_nil + end + + it "handles blank files gracefully" do + part = build_form_data_part("avatar", "", "application/octet-stream", "") + lucky_file = Lucky::UploadedFile.new(part) + + lucky_file.blank?.should be_true + end +end + +private struct TestUploader < Lucky::Attachment::Uploader +end + +private struct AvatarUploader < Lucky::Attachment::Uploader +end + +private struct CustomLocationUploader < Lucky::Attachment::Uploader + def generate_location( + io : IO, + metadata : Lucky::Attachment::MetadataHash, + ) : String + "custom/#{super}" + end +end + +private struct CustomMetadataUploader < Lucky::Attachment::Uploader + def extract_metadata( + io : IO, + metadata : Lucky::Attachment::MetadataHash? = nil, + **options, + ) : Lucky::Attachment::MetadataHash + data = super + data["custom_key"] = "custom_value" + data + end +end + +private def build_form_data_part( + name : String, + filename : String, + content_type : String, + body : String, +) : HTTP::FormData::Part + disposition = if filename.empty? + %(form-data; name="#{name}") + else + %(form-data; name="#{name}"; filename="#{filename}") + end + headers = HTTP::Headers{ + "Content-Disposition" => disposition, + "Content-Type" => content_type, + } + + HTTP::FormData::Part.new(headers: headers, body: IO::Memory.new(body)) +end diff --git a/src/lucky.cr b/src/lucky.cr index a587e51fc..8042a1f2a 100644 --- a/src/lucky.cr +++ b/src/lucky.cr @@ -10,6 +10,8 @@ require "./lucky/quick_def" require "./charms/*" require "http/server" require "lucky_router" +require "./lucky/attachment" +require "./lucky/attachment/**" require "./bun/*" require "./lucky/events/*" require "./lucky/support/*" diff --git a/src/lucky/attachment.cr b/src/lucky/attachment.cr new file mode 100644 index 000000000..d0a93d509 --- /dev/null +++ b/src/lucky/attachment.cr @@ -0,0 +1,17 @@ +require "./attachment/storage" + +module Lucky::Attachment + alias MetadataValue = String | Int64 | Int32 | Float64 | Bool | Nil + alias MetadataHash = Hash(String, MetadataValue) + + annotation MetadataMethods + end + + class Error < Exception; end + + class FileNotFound < Error; end + + class InvalidFile < Error; end + + class CliToolNotFound < Error; end +end diff --git a/src/lucky/attachment/config.cr b/src/lucky/attachment/config.cr new file mode 100644 index 000000000..4e0f9c2ca --- /dev/null +++ b/src/lucky/attachment/config.cr @@ -0,0 +1,44 @@ +require "habitat" +require "./storage" + +module Lucky::Attachment + Habitat.create do + # Storage configurations keyed by name. The default storages are typically: + # - "cache" (temporary storage between requests to avoid re-uploads) + # - "store" (where uploads are moved from the cache after a commit) + # + # NOTE: Additional stores are not supported yet. Please reach out if that + # is something you need. + # + setting storages : Hash(String, Storage) = {} of String => Storage + + # Path prefix for uploads. Possible keywords are: + # - `:model` (an underscored string of the model name) + # - `:id` (the record's primary key value) + # - `:attachment` (the name of the attachment; e.g. "avatar") + # + setting path_prefix : String = ":model/:id/:attachment" + end + + # Retrieves a storage by name, raising if not found. + # + # ``` + # Lucky::Attachment.find_storage("store") # => Storage::FileSystem + # Lucky::Attachment.find_storage("missing") # raises Lucky::Attachment::Error + # ``` + # + def self.find_storage(name : String) : Storage + settings.storages[name]? || + raise Error.new( + String.build do |io| + if settings.storages.keys.empty? + io << "There are no storages registered yet" + else + io << %(Storage ) << name.inspect + io << %( is not registered. The available storages are: ) + io << settings.storages.keys.map(&.inspect).join(", ") + end + end + ) + end +end diff --git a/src/lucky/attachment/extractor.cr b/src/lucky/attachment/extractor.cr new file mode 100644 index 000000000..e485f3c80 --- /dev/null +++ b/src/lucky/attachment/extractor.cr @@ -0,0 +1,15 @@ +# Extractors try to extract metadata from the context they're given: the `IO` +# object, the current state of the resulting metadata hash, or arbitrary +# options passed to the `upload` method of the uploader. +# +module Lucky::Attachment::Extractor + # Extracts metadata and returns a `MetadataValue`. Alternatively, the + # metadata hash may be modified directly if multiple values need to be added + # (e.g. the dimensions of an image). + # + abstract def extract( + io : IO, + metadata : MetadataHash, + **options, + ) : MetadataValue? +end diff --git a/src/lucky/attachment/extractor/dimensions_from_magick.cr b/src/lucky/attachment/extractor/dimensions_from_magick.cr new file mode 100644 index 000000000..2af23298b --- /dev/null +++ b/src/lucky/attachment/extractor/dimensions_from_magick.cr @@ -0,0 +1,23 @@ +require "./run_command" + +@[Lucky::Attachment::MetadataMethods(width : Int32, height : Int32)] +struct Lucky::Attachment::Extractor::DimensionsFromMagick + include Lucky::Attachment::Extractor + include Lucky::Attachment::Extractor::RunCommand + + # Extracts the dimensions of a file using ImageMagick's `magick` command. + def extract(io, metadata, **options) : Nil + if result = run_magick_command(io, ["identify", "-format", "%[fx:w] %[fx:h]"]) + dimensions = result.split.map(&.to_i) + + metadata["width"] = dimensions[0] + metadata["height"] = dimensions[1] + end + end + + private def run_magick_command(io, args) + run_command("magick", args, io) + rescue CliToolNotFound + run_command("identify", args[1..], io) + end +end diff --git a/src/lucky/attachment/extractor/filename_from_io.cr b/src/lucky/attachment/extractor/filename_from_io.cr new file mode 100644 index 000000000..7ea201631 --- /dev/null +++ b/src/lucky/attachment/extractor/filename_from_io.cr @@ -0,0 +1,15 @@ +struct Lucky::Attachment::Extractor::FilenameFromIO + include Lucky::Attachment::Extractor + + # Returns the filename from the options or tries to extract the filename from + # the IO object. + def extract(io, metadata, **options) : String? + options[:filename]? || if io.responds_to?(:original_filename) + io.original_filename + elsif io.responds_to?(:filename) + io.filename.presence + elsif io.responds_to?(:path) + File.basename(io.path) + end + end +end diff --git a/src/lucky/attachment/extractor/mime_from_extension.cr b/src/lucky/attachment/extractor/mime_from_extension.cr new file mode 100644 index 000000000..40a1f7c2c --- /dev/null +++ b/src/lucky/attachment/extractor/mime_from_extension.cr @@ -0,0 +1,12 @@ +struct Lucky::Attachment::Extractor::MimeFromExtension + include Lucky::Attachment::Extractor + + # Extracts the MIME type from the extension of the filename. + def extract(io, metadata, **options) : String? + return unless filename = FilenameFromIO.new.extract(io, metadata, **options) + + MIME.from_filename(filename) + rescue KeyError + nil + end +end diff --git a/src/lucky/attachment/extractor/mime_from_file.cr b/src/lucky/attachment/extractor/mime_from_file.cr new file mode 100644 index 000000000..0af8b0e53 --- /dev/null +++ b/src/lucky/attachment/extractor/mime_from_file.cr @@ -0,0 +1,14 @@ +require "./run_command" + +struct Lucky::Attachment::Extractor::MimeFromFile + include Lucky::Attachment::Extractor + include Lucky::Attachment::Extractor::RunCommand + + # Extracts the MIME type using the `file` utility. + def extract(io, metadata, **options) : String? + # Avoids returning "application/x-empty" for empty files + return nil if io.size.try &.zero? + + run_command("file", %w[--mime-type --brief], io) + end +end diff --git a/src/lucky/attachment/extractor/mime_from_io.cr b/src/lucky/attachment/extractor/mime_from_io.cr new file mode 100644 index 000000000..a4e2d859d --- /dev/null +++ b/src/lucky/attachment/extractor/mime_from_io.cr @@ -0,0 +1,13 @@ +struct Lucky::Attachment::Extractor::MimeFromIO + include Lucky::Attachment::Extractor + + # Extracts the MIME type from the IO. + def extract(io, metadata, **options) : String? + return unless io.responds_to?(:content_type) + return unless type = io.content_type + return unless mime = type.split(';').first?.try(&.strip) + return if mime.empty? + + mime if mime.matches?(/\A\w+\/[\w\.\+\-]+\z/) + end +end diff --git a/src/lucky/attachment/extractor/run_command.cr b/src/lucky/attachment/extractor/run_command.cr new file mode 100644 index 000000000..5956a1025 --- /dev/null +++ b/src/lucky/attachment/extractor/run_command.cr @@ -0,0 +1,41 @@ +# Some extractors may need to call on command-line tools to extract certain +# data. This module will provide a helper to simplify running commands. +# +# ``` +# struct ColourspaceFromIdentify +# include Lucky::Attachment::Extractor +# include Lucky::Attachment::Extractor::RunCommand +# +# def extract(io, metadata, **options) : String? +# run_command("magick", ["identify", "-format", "%[colorspace]"], io) +# end +# end +# ``` +# +module Lucky::Attachment::Extractor::RunCommand + # Runs th given command on the given IO object and returns the resulting + # string if the command was successful. + private def run_command( + command : String, + args : Array(String), + input : IO, + ) : String? + stdout, stderr = IO::Memory.new, IO::Memory.new + result = Process.run( + command, + args: args + ["-"], + output: stdout, + error: stderr, + input: input + ) + input.rewind + + return stdout.to_s.strip if result.success? + + Log.debug do + "Unable to extract data with `#{command} #{args.join(' ')}` (#{stderr})" + end + rescue File::NotFoundError + raise CliToolNotFound.new("The `#{command}` command-line tool is not installed") + end +end diff --git a/src/lucky/attachment/extractor/size_from_io.cr b/src/lucky/attachment/extractor/size_from_io.cr new file mode 100644 index 000000000..e32ce09c0 --- /dev/null +++ b/src/lucky/attachment/extractor/size_from_io.cr @@ -0,0 +1,12 @@ +struct Lucky::Attachment::Extractor::SizeFromIO + include Lucky::Attachment::Extractor + + # Tries to extract the file size from the IO. + def extract(io, metadata, **options) : Int64? + if io.responds_to?(:tempfile) + io.tempfile.size + elsif io.responds_to?(:size) + io.size.to_i64 + end + end +end diff --git a/src/lucky/attachment/storage.cr b/src/lucky/attachment/storage.cr new file mode 100644 index 000000000..8126e20db --- /dev/null +++ b/src/lucky/attachment/storage.cr @@ -0,0 +1,61 @@ +# Storage backends handle the actual persistence of uploaded files. +# Implementations must provide methods for uploading, retreiving, checking +# existence, and deleting files. +# +abstract class Lucky::Attachment::Storage + # Uploads an IO to the given location (id) in the storage. + # + # ``` + # storage.upload(io, "uploads/photo.jpg") + # storage.upload(io, "uploads/photo.jpg", metadata: {"filename" => "original.jpg"}) + # ``` + # + abstract def upload(io : IO, id : String, **options) : Nil + + # Opens the file at the given location and returns an IO for reading. + # + # ``` + # io = storage.open("uploads/photo.jpg") + # content = io.gets_to_end + # io.close + # ``` + # + # Raises `Lucky::Attachment::FileNotFound` if the file doesn't exist. + # + abstract def open(id : String, **options) : IO + + # Returns whether a file exists at the given location. + # + # ``` + # storage.exists?("uploads/photo.jpg") + # # => true + # ``` + # + abstract def exists?(id : String) : Bool + + # Returns the URL for accessing the file at the given location. + # + # ``` + # storage.url("uploads/photo.jpg") + # # => "/uploads/photo.jpg" + # storage.url("uploads/photo.jpg", host: "https://example.com") + # # => "https://example.com/uploads/photo.jpg" + # ``` + # + abstract def url(id : String, **options) : String + + # Deletes the file at the given location. + # + # ``` + # storage.delete("uploads/photo.jpg") + # ``` + # + # Does not raise if the file doesn't exist. + # + abstract def delete(id : String) : Nil + + # Moves a file from another location. + def move(io : IO, id : String, **options) : Nil + upload(io, id, **options) + end +end diff --git a/src/lucky/attachment/storage/file_system.cr b/src/lucky/attachment/storage/file_system.cr new file mode 100644 index 000000000..7eae43e2f --- /dev/null +++ b/src/lucky/attachment/storage/file_system.cr @@ -0,0 +1,126 @@ +require "../storage" + +# Local filesystem storage backend. Files are stored in a directory on the +# local filesystem. Supports an optional prefix for organizing files. +# +# ``` +# Lucky::Attachment.configure do |settings| +# settings.storages["cache"] = Lucky::Attachment::Storage::FileSystem.new( +# directory: "uploads", +# prefix: "cache" +# ) +# settings.storages["store"] = Lucky::Attachment::Storage::FileSystem.new( +# directory: "uploads" +# ) +# end +# ``` +# +class Lucky::Attachment::Storage::FileSystem < Lucky::Attachment::Storage + DEFAULT_PERMISSIONS = File::Permissions.new(0o644) + DEFAULT_DIRECTORY_PERMISSIONS = File::Permissions.new(0o755) + + getter directory : String + getter prefix : String? + getter? clean : Bool + getter permissions : File::Permissions + getter directory_permissions : File::Permissions + + def initialize( + @directory : String, + @prefix : String? = nil, + @clean : Bool = true, + @permissions : File::Permissions = DEFAULT_PERMISSIONS, + @directory_permissions : File::Permissions = DEFAULT_DIRECTORY_PERMISSIONS, + ) + Dir.mkdir_p(expanded_directory, mode: directory_permissions.value) + end + + # Returns the full expanded path including prefix. + # + # ``` + # storage.expanded_directory + # # => "/app/uploads/cache" + # ``` + # + def expanded_directory : String + return File.expand_path(directory) unless p = prefix + + File.expand_path(File.join(directory, p)) + end + + # Uploads an IO to the given location (id) in the storage. + def upload(io : IO, id : String, move : Bool = false, **options) : Nil + path = path_for(id) + Dir.mkdir_p(File.dirname(path), mode: directory_permissions.value) + + if move && io.is_a?(File) + File.rename(io.path, path) + File.chmod(path, permissions) + else + File.open(path, "wb", perm: permissions) do |file| + IO.copy(io, file) + end + end + end + + # Opens the file at the given location and returns an IO for reading. + def open(id : String, **options) : IO + File.open(path_for(id), "rb") + rescue ex : File::NotFoundError + raise FileNotFound.new("File not found: #{id}") + end + + # Returns whether a file exists at the given location. + def exists?(id : String) : Bool + File.exists?(path_for(id)) + end + + # Returns the full filesystem path for the given id. + # + # ``` + # storage.path_for("abc123.jpg") + # # => "/app/uploads/abc123.jpg" + # ``` + # + def path_for(id : String) : String + File.join(expanded_directory, id.gsub('/', File::SEPARATOR)) + end + + def url(id : String, host : String? = nil, **options) : String + String.build do |url| + url << host.rstrip('/') if host + url << '/' + if p = prefix + url << p.lstrip('/') << '/' + end + url << id + end + end + + # Deletes the file at the given location. + def delete(id : String) : Nil + path = path_for(id) + File.delete?(path) + clean_directories(path) if clean? + rescue ex : File::Error + # Ignore errors here + end + + # Override move for efficient file system rename + def move(io : IO, id : String, **options) : Nil + upload(io, id, **options, move: io.is_a?(File)) + end + + # Cleans empty parent directories up to the expanded_directory. + private def clean_directories(path : String) : Nil + current = File.dirname(path) + + while current != expanded_directory && current.starts_with?(expanded_directory) + break unless Dir.empty?(current) + Dir.delete(current) + current = File.dirname(current) + end + rescue ex : File::Error + # Ignore errors here + end +end diff --git a/src/lucky/attachment/storage/memory.cr b/src/lucky/attachment/storage/memory.cr new file mode 100644 index 000000000..e8f504991 --- /dev/null +++ b/src/lucky/attachment/storage/memory.cr @@ -0,0 +1,65 @@ +require "../storage" + +# In-memory storage backend for testing purposes. Files are stored in a hash +# and are lost when the process exits. This is useful for testing without +# hitting the filesystem or network. +# +# ``` +# Lucky::Attachment.configure do |settings| +# settings.storages["cache"] = Lucky::Attachment::Storage::Memory.new +# settings.storages["store"] = Lucky::Attachment::Storage::Memory.new +# end +# ``` +# +class Lucky::Attachment::Storage::Memory < Lucky::Attachment::Storage + getter store : Hash(String, Bytes) + getter base_url : String? + + def initialize(@base_url : String? = nil) + @store = {} of String => Bytes + end + + # Uploads an IO to the given location (id) in the storage. + def upload(io : IO, id : String, **options) : Nil + @store[id] = io.getb_to_end + end + + # Opens the file at the given location and returns an IO for reading. + def open(id : String, **options) : IO + if bytes = @store[id]? + IO::Memory.new(bytes) + else + raise FileNotFound.new("File not found: #{id}") + end + end + + # Returns whether a file exists at the given location. + def exists?(id : String) : Bool + @store.has_key?(id) + end + + # Returns the URL for accessing the file at the given location. + def url(id : String, **options) : String + String.build do |io| + if base = @base_url + io << base.rstrip('/') + end + io << '/' << id + end + end + + # Deletes the file at the given location. + def delete(id : String) : Nil + @store.delete(id) + end + + # Clears out the store. + def clear! : Nil + @store.clear + end + + # Returns the number of stored files. + def size : Int32 + @store.size + end +end diff --git a/src/lucky/attachment/storage/s3.cr b/src/lucky/attachment/storage/s3.cr new file mode 100644 index 000000000..81e9ec66f --- /dev/null +++ b/src/lucky/attachment/storage/s3.cr @@ -0,0 +1,323 @@ +require "../storage" +require "awscr-s3" + +# S3-compatible storage backend. Supports AWS S3 and any S3-compatible service +# such as RustFS, Tigris, or Cloudflare R2 via a custom endpoint. +# +# Requires the `awscr-s3` shard to be added to your `shard.yml`: +# +# ```yaml +# dependencies: +# awscr-s3: +# github: taylorfinnell/awscr-s3 +# ``` +# +# ## AWS S3 +# +# ``` +# Lucky::Attachment::Storage::S3.new( +# bucket: "lucky-bucket", +# region: "eu-west-1", +# access_key_id: ENV["KEY"], +# secret_access_key: ENV["SECRET"] +# ) +# ``` +# +# ## RustFS or other S3-compatible services +# +# ``` +# Lucky::Attachment::Storage::S3.new( +# bucket: "lucky-bucket", +# region: "eu-west-1", +# access_key_id: ENV["KEY"], +# secret_access_key: ENV["SECRET"], +# endpoint: "http://localhost:9000" +# ) +# ``` +# +# ## Bring your own client +# +# ``` +# client = Awscr::S3::Client.new("eu-west-1", ENV["KEY"], ENV["SECRET"]) +# Lucky::Attachment::Storage::S3.new(bucket: "lucky-bucket", client: client) +# ``` +# +class Lucky::Attachment::Storage::S3 < Lucky::Attachment::Storage + getter bucket : String + getter prefix : String? + getter? public : Bool + getter upload_options : Hash(String, String) + getter client : Awscr::S3::Client + + # Initialises a storage using credentials. + # + # ``` + # storage = Lucky::Attachment::Storage::S3.new( + # bucket: "lucky-bucket", + # region: "eu-west-1", + # access_key_id: "key", + # secret_access_key: "secret", + # endpoint: "http://localhost:9000" + # ) + # ``` + # + def initialize( + @bucket : String, + region : String, + @access_key_id : String, + @secret_access_key : String, + @prefix : String? = nil, + endpoint : String? = nil, + @public : Bool = false, + @upload_options : Hash(String, String) = Hash(String, String).new, + ) + @client = Awscr::S3::Client.new( + region, + @access_key_id, + @secret_access_key, + endpoint: endpoint + ) + end + + # Initialises a storage with a pre-built `Awscr::S3::Client`. Useful when you + # need full control over the client configuration, or in tests for example. + # + # ``` + # client = Awscr::S3::Client.new("eu-west-1", "key", "secret") + # storage = Lucky::Attachment::Storage::S3.new( + # bucket: "lucky-bucket", + # client: client + # ) + # ``` + # + def initialize( + @bucket : String, + @client : Awscr::S3::Client, + @prefix : String? = nil, + @public : Bool = false, + @upload_options : Hash(String, String) = Hash(String, String).new, + ) + # NOTE: These attributes are protected on the client, so this is the only + # way to get them. + @access_key_id = @client.@aws_access_key + @secret_access_key = @client.@aws_secret_key + end + + # Uploads an IO to the given key in the bucket. Any additional keys in + # `options` are forwarded to the S3 client. + # + # ``` + # storage.upload(io, "uploads/photo.jpg") + # storage.upload(io, "uploads/photo.jpg", metadata: { + # "filename" => "photo.jpg", + # "mime_type" => "image/jpeg", + # }) + # ``` + # + def upload(io : IO, id : String, **options) : Nil + @client.put_object( + bucket, + object_key(id), + io.gets_to_end, + build_upload_headers(**options) + ) + end + + # Opens the S3 object and returns an `IO::Memory` for reading. + # + # ``` + # io = storage.open("uploads/photo.jpg") + # content = io.gets_to_end + # io.close + # ``` + # + # Raises `Lucky::Attachment::FileNotFound` if the object does not exist. + # + def open(id : String, **options) : IO + buffer = IO::Memory.new + @client.get_object(bucket, object_key(id)) do |response| + IO.copy(response.body_io, buffer) + end + buffer.rewind + buffer + rescue ex : Awscr::S3::NoSuchKey + raise FileNotFound.new("File not found: #{id}") + end + + # Tests if an object exists in the bucket. + # + # ``` + # storage.exists?("uploads/photo.jpg") + # # => true + # ``` + # + def exists?(id : String) : Bool + @client.head_object(bucket, object: object_key(id)) + true + rescue Awscr::S3::Exception + false + end + + # Returns the URL for accessing the object. When `expires_in` is provided + # (in seconds), a presigned URL is returned. Otherwise a plain public URL is + # constructed without any HTTP round-trip. + # + # ``` + # storage.url("uploads/photo.jpg") + # # => "https://s3-eu-west-1.amazonaws.com/lucky-bucket/uploads/photo.jpg" + # + # storage.url("uploads/photo.jpg", expires_in: 3600) + # # => "https://s3-eu-west-1.amazonaws.com/lucky-bucket/uploads/photo.jpg?X-Amz-Signature=..." + # ``` + # + def url(id : String, **options) : String + return public_url(id) unless expires_in = options[:expires_in]? + + presigned_url(id, expires_in: expires_in.to_i) + end + + # Deletes the object for the given key. Does not raise if the object does not + # exist. + # + # ``` + # storage.delete("uploads/photo.jpg") + # ``` + # + def delete(id : String) : Nil + @client.delete_object(bucket, object_key(id)) + end + + # Promotes a file efficiently using a server-side S3 copy when the source is + # a `StoredFile` in the same bucket, avoiding the download/re-upload. Falls + # back to a regular upload for plain `IO` sources. + # + def move(io : IO, id : String, **options) : Nil + upload(io, id, **options) + end + + def move(file : Lucky::Attachment::StoredFile, id : String, **options) : Nil + if same_bucket?(file) + copy_object( + **options, + source_key: object_key(file.id), + dest_key: object_key(id) + ) + file.delete + else + move(file.io, id, **options) + end + end + + # Returns the full object key including any configured prefix. + # + # ``` + # storage.object_key("photo.jpg") + # # => "photo.jpg" + # ``` + # + def object_key(id : String) : String + return id unless p = prefix + + "#{p.strip('/')}/#{id.lstrip('/')}" + end + + # Builds a header hash. + private def build_upload_headers(**options) : Hash(String, String) + Hash(String, String).new.tap do |headers| + if metadata = options[:metadata]?.try(&.as?(MetadataHash)) + if filename = metadata["filename"]?.try(&.as?(String)) + headers["Content-Disposition"] = %(inline; filename="#{filename}") + end + if mime_type = metadata["mime_type"]?.try(&.as?(String)) + headers["Content-Type"] = mime_type + end + end + + if content_type = options[:content_type]?.try(&.to_s.presence) + headers["Content-Type"] = content_type + end + + if content_disposition = options[:content_disposition]?.try(&.to_s.presence) + headers["Content-Disposition"] = content_disposition + end + + headers["x-amz-acl"] = "public-read" if public? + + @upload_options.each do |key, value| + headers[key] ||= value + end + end + end + + # Builds a public url considering custom endpoints configured in the client. + private def public_url(id : String) : String + String.build do |io| + if uri = endpoint_uri + scheme = uri.scheme || "https" + host = endpoint_host_with_port(uri) + else + scheme = "https" + host = "s3-#{@client.region}.amazonaws.com" + end + io << scheme << "://" << host << '/' << bucket << '/' << object_key(id) + end + end + + # Builds a presigned url. + private def presigned_url(id : String, expires_in : Int32) : String + options = Awscr::S3::Presigned::Url::Options.new( + aws_access_key: @access_key_id, + aws_secret_key: @secret_access_key, + region: @client.region, + object: "/#{object_key(id)}", + bucket: bucket, + host_name: presigned_url_host_name, + expires: expires_in, + ) + Awscr::S3::Presigned::Url.new(options).for(:get) + end + + # Builds the host name for the current endpoint. + private def presigned_url_host_name : String? + return unless uri = endpoint_uri + + endpoint_host_with_port(uri) + rescue URI::Error + raw_endpoint + end + + # Copies an object to a new location. + private def copy_object(source_key : String, dest_key : String, **options) : Nil + @client.copy_object( + bucket, + source_key, + dest_key, + build_upload_headers(**options) + ) + end + + # Determines if the given file is in the same bucket as the current one. + private def same_bucket?(file : Lucky::Attachment::StoredFile) : Bool + return false unless storage = file.storage.as?(S3) + + storage.bucket == bucket + end + + # Builds a host with port from a URI object. + private def endpoint_host_with_port(uri : URI) : String + host, port = uri.host.to_s, uri.port + (port && port != 80 && port != 443) ? "#{host}:#{port}" : host + end + + # Tries to parse the S3 client's endpoint to a URI object. + private def endpoint_uri : URI? + return unless endpoint = raw_endpoint + + URI.parse(endpoint) + end + + # Tries to retrieve the endpoint from the client. + private def raw_endpoint : String? + @client.endpoint.try(&.to_s) + end +end diff --git a/src/lucky/attachment/stored_file.cr b/src/lucky/attachment/stored_file.cr new file mode 100644 index 000000000..6986f0bed --- /dev/null +++ b/src/lucky/attachment/stored_file.cr @@ -0,0 +1,238 @@ +require "json" +require "uuid" + +# Represents a file that has been uploaded to a storage backend. +# +# This class is JSON serializable and stores the file's location (`id`), +# which storage it's in (`storage`), and associated metadata. +# +# NOTE: The JSON format is compatible with Shrine.rb/Shrine.cr: +# +# ```json +# { +# "id": "uploads/abc123.jpg", +# "storage": "store", +# "metadata": { +# "filename": "photo.jpg", +# "size": 102400, +# "mime_type": "image/jpeg" +# } +# } +# ``` +# +abstract class Lucky::Attachment::StoredFile + include JSON::Serializable + + # NOTE: This mimics the behavior of Avram's `JSON::Serializable` extension. + def self.adapter + Lucky(self) + end + + getter id : String + @[JSON::Field(key: "storage")] + getter storage_key : String + getter metadata : MetadataHash + + @[JSON::Field(ignore: true)] + @io : IO? + + def initialize( + @id : String, + @storage_key : String, + @metadata : MetadataHash = MetadataHash.new, + ) + end + + # Returns the file extension based on the id or original filename. + # + # ``` + # file.extension? + # # => "jpg" + # ``` + # + def extension? : String? + ext = File.extname(id).lchop('.') + ext = File.extname(filename).lchop('.') if ext.empty? && filename? + ext.presence.try(&.downcase) + end + + # The non-nilable variant of the `extension?` method. + def extension : String + extension?.as(String) + end + + # Aliases the `[]?` method on the metadata property. + # + # ``` + # file["width"]? + # # => 800 + # file["custom"]? + # # => "value" + # ``` + # + def []?(key : String) : MetadataValue + metadata[key]? + end + + # Returns the storage instance this file is stored in. + def storage : Storage + ::Lucky::Attachment.find_storage(storage_key) + end + + # Returns the URL for accessing this file. + # + # ``` + # file.url + # # => "https://bucket.s3.amazonaws.com/uploads/abc123.jpg" + # + # # for presigned URLs + # file.url(expires_in: 1.hour) + # ``` + # + def url(**options) : String + storage.url(id, **options) + end + + # Returns whether this file exists in storage. + # + # ``` + # file.exists? # => true + # ``` + # + def exists? : Bool + storage.exists?(id) + end + + # Opens the file for reading. If a block is given, yields the IO and + # automatically closes it afterwards. Returns the block's return value. + # + # ``` + # file.open do |io| + # io.gets_to_end + # end + # ``` + # + def open(**options, &) + io = storage.open(id, **options) + begin + yield io + ensure + io.close + end + end + + # Opens the file and stores the IO handle internally for subsequent reads. + # Remember to call `close` when done. + # + # ``` + # file.open + # content = file.io.gets_to_end + # file.close + # ``` + def open(**options) : IO + close if @io + @io = storage.open(id, **options) + end + + # Returns the currently opened IO, or opens it if not already open. + def io : IO + @io || open + end + + # Closes the file if it is open. + def close : Nil + @io.try(&.close) + @io = nil + end + + # Tests whether the file has been opened or not. + def opened? : Bool + !@io.nil? + end + + # Downloads the file to a temporary file and returns it. As opposed to the + # block variant, this temporary file needs to be closed and deleted + # manually: + # + # ``` + # tempfile = file.download + # tempfile.path + # # => "/tmp/lucky-attachment123456789.jpg" + # tempfile.gets_to_end + # # => "file content" + # tempfile.close + # tempfile.delete + # ``` + # + def download(**options) : File + tempfile = File.tempfile("lucky-attachment", ".#{extension}") + stream(tempfile, **options) + tempfile.rewind + tempfile + end + + # Downloads to a tempfile, yields it to the block, then cleans up. + # + # ``` + # file.download do |tempfile| + # process(tempfile.path) + # end + # # tempfile is automatically deleted + # ``` + # + def download(**options, &) + tempfile = download(**options) + begin + yield tempfile + ensure + tempfile.close + tempfile.delete + end + end + + # Streams the file content to the given IO destination. + # + # ``` + # file.stream(response.output) + # ``` + # + def stream(destination : IO, **options) : Nil + if opened? + IO.copy(io, destination) + io.rewind if io.responds_to?(:rewind) + else + open(**options) do |io| + IO.copy(io, destination) + end + end + end + + # Deletes the file from storage. + # + # ``` + # file.delete + # ``` + # + def delete : Nil + storage.delete(id) + end + + # Returns a hash representation suitable for JSON serialization compatible + # with Shrine. + def data : Hash(String, String | MetadataHash) + { + "id" => id, + "metadata" => metadata, + "storage" => storage_key, + } + end + + # Compares two `StoredFile` by thier id and storage. + def ==(other : StoredFile) : Bool + id == other.id && storage_key == other.storage_key + end + + def ==(other) : Bool + false + end +end diff --git a/src/lucky/attachment/uploader.cr b/src/lucky/attachment/uploader.cr new file mode 100644 index 000000000..f58c61791 --- /dev/null +++ b/src/lucky/attachment/uploader.cr @@ -0,0 +1,282 @@ +require "uuid" + +# Base uploader class that handles file uploads with metadata extraction and +# location generation. +# +# ``` +# struct ImageUploader < Lucky::Attachment::Uploader +# def generate_location(io, metadata, **options) : String +# date = Time.utc.to_s("%Y/%m/%d") +# File.join("images", date, super) +# end +# end +# +# ImageUploader.new("store").upload(io) +# # => Lucky::Attachment::StoredFile with id "images/2024/01/15/abc123.jpg" +# ``` +# +abstract struct Lucky::Attachment::Uploader + alias MetadataHash = ::Lucky::Attachment::MetadataHash + + # Defines the path prefix for uploads in the storage. Overwrite this method + # in uploader subclasses to use custom path prefixes per uploader. + def self.path_prefix : String + Lucky::Attachment.settings.path_prefix + end + + # Adds shorter local aliases for built-in extractors. + # e.g. `Lucky::Attachment::Extractor::SizeFromIO` -> `SizeFromIOExtractor` + {% for extractor in %w[ + DimensionsFromMagick + FilenameFromIO + MimeFromExtension + MimeFromFile + MimeFromIO + SizeFromIO + ] %} + alias {{ extractor.id }}Extractor = Lucky::Attachment::Extractor::{{ extractor.id }} + {% end %} + + EXTRACTORS = {} of String => Extractor + + macro inherited + {% stored_file = "#{@type}::StoredFile".id %} + + class {{ stored_file }} < Lucky::Attachment::StoredFile + end + + # Register default extractors + extract filename, using: Lucky::Attachment::Extractor::FilenameFromIO + extract mime_type, using: Lucky::Attachment::Extractor::MimeFromIO + extract size, using: Lucky::Attachment::Extractor::SizeFromIO + + # Uploads a file and returns a `Lucky::Attachment::StoredFile`. This method + # accepts additional metadata and arbitrary arguments for overrides. + # + # ``` + # uploader.upload(io) + # uploader.upload(io, metadata: {"custom" => "value"}) + # uploader.upload(io, location: "custom/path.jpg") + # ``` + # + def upload(io : IO, metadata : MetadataHash? = nil, **options) : {{ stored_file }} + data = extract_metadata(io, metadata, **options) + data = data.merge(metadata) if metadata + location = options[:location]? || generate_location(io, data, **options) + + storage.upload(io, location, **options.merge(metadata: data)) + {{ stored_file }}.new(id: location, storage_key: storage_key, metadata: data) + end + + # Uploads to the "cache" storage. + # + # ``` + # cached = ImageUploader.cache(io) + # ``` + # + def self.cache(io : IO, **options) : {{ stored_file }} + new("cache").upload(io, **options) + end + + # Uploads to the "store" storage. + # + # ``` + # stored = ImageUploader.store(io) + # ``` + # + def self.store(io : IO, **options) : {{ stored_file }} + new("store").upload(io, **options) + end + + # Promotes a file from cache to store. + # + # ``` + # cached = ImageUploader.cache(io) + # stored = ImageUploader.promote(cached) + # ``` + # + def self.promote( + file : {{ stored_file }}, + to storage : String = "store", + delete_source : Bool = true, + **options, + ) : {{ stored_file }} + file.open do |io| + ::Lucky::Attachment + .find_storage(storage) + .upload(io, file.id, metadata: file.metadata) + promoted = {{ stored_file }}.new( + id: file.id, + storage_key: storage, + metadata: file.metadata + ) + file.delete if delete_source + promoted + end + end + end + + # Registers an extractor for a given key. + # + # ``` + # struct PdfUploader < Lucky::Attachment::Uploader + # # Use a different MIME type extractor than the default one + # extract mime_type, using: Lucky::Attachment::Extractor::MimeFromExtension + # + # # Or use your own custom extractor to add arbitrary data + # extract pages, using: MyNumberOfPagesExtractor + # end + # ``` + # + # The result will then be added to the attachment's metadata after uploading: + # ``` + # invoice.pdf.pages + # # => 24 + # ``` + # + macro extract(name, using) + {% + type = using.resolve.methods + .find { |method| method.name == :extract.id } + .return_type.types.first + %} + + class {{ @type }}::StoredFile < Lucky::Attachment::StoredFile + def {{ name }}? : {{ type }}? + {% if {Int32, Int64}.includes? type.resolve %} + if value = metadata["{{ name }}"]? + {{ type }}.new(value.as(Int32 | Int64)) + end + {% else %} + metadata["{{ name }}"]?.try(&.as?({{ type }})) + {% end %} + end + + def {{ name }} : {{ type }} + {% if {Int32, Int64}.includes? type.resolve %} + {{ type }}.new(metadata["{{ name }}"].as(Int32 | Int64)) + {% else %} + metadata["{{ name }}"].as({{ type }}) + {% end %} + end + + {% if methods = using.resolve.annotation(Lucky::Attachment::MetadataMethods) %} + {% for td in methods.args %} + def {{ td.var }}? : {{ td.type }}? + {% if {Int32, Int64}.includes? td.type.resolve %} + if value = metadata["{{ td.var }}"]? + {{ td.type }}.new(value.as(Int32 | Int64)) + end + {% else %} + metadata["{{ td.var }}"]?.try(&.as?({{ td.type }})) + {% end %} + end + + def {{ td.var }} : {{ td.type }} + {% if {Int32, Int64}.includes? td.type.resolve %} + {{ td.type }}.new(metadata["{{ td.var }}"].as(Int32 | Int64)) + {% else %} + metadata["{{ td.var }}"].as({{ td.type }}) + {% end %} + end + {% end %} + {% end %} + end + + EXTRACTORS["{{ name }}"] = {{ using }}.new + end + + getter storage_key : String + + def initialize(@storage_key : String) + end + + # Returns the storage instance for this uploader. + def storage : Storage + Lucky::Attachment.find_storage(storage_key) + end + + # Generates a unique location for the uploaded file. Override this in + # subclasses for custom locations. + # + # ``` + # class ImageUploader < Lucky::Attachment::Uploader + # def generate_location(io, metadata, **options) : String + # File.join("images", super) + # end + # end + # ``` + # + def generate_location(io : IO, metadata : MetadataHash, **options) : String + extension = file_extension(io, metadata) + basename = generate_uid(io, metadata, **options) + filename = extension ? "#{basename}.#{extension}" : basename + File.join([options[:path_prefix]?, filename].compact) + end + + # Generates a unique identifier for file locations. Override this in + # subclasses for custom filenames in the storage. + # + # ``` + # class ImageUploader < Lucky::Attachment::Uploader + # def generate_uid(io, metadata, **options) : String + # "#{metadata["filename"]}-#{Time.local.to_unix}" + # end + # end + # ``` + # + def generate_uid(io : IO, metadata : MetadataHash, **options) : String + UUID.random.to_s + end + + # Extracts metadata from the IO. Override in subclasses to add completely + # custom metadata extraction outside of the `extract` DSL. + # + # ``` + # class ImageUploader < Lucky::Attachment::Uploader + # def extract_metadata(io, metadata : MetadataHash? = nil, **options) : MetadataHash + # data = super + # # Add custom metadata + # data["custom"] = "value" + # data + # end + # + # # Reopen the `StoredFile` class to add a method for the custom value. + # class StoredFile + # def custom : String + # metadata["custom"].as(String) + # end + # end + # end + # ``` + # + def extract_metadata( + io : IO, + metadata : MetadataHash? = nil, + **options, + ) : MetadataHash + (metadata.try(&.dup) || MetadataHash.new).tap do |data| + EXTRACTORS.each do |name, extractor| + if value = extractor.extract(io, data, **options) + data[name] = value + end + end + end + end + + # Tries to determine the file extension from the metadata or IO. + protected def file_extension( + io : IO, + metadata : MetadataHash, + ) : String? + if filename = metadata["filename"]?.try(&.as(String)) + ext = File.extname(filename).lchop('.') + return ext.downcase unless ext.empty? + end + + if io.responds_to?(:path) + ext = File.extname(io.path).lchop('.') + return ext.downcase unless ext.empty? + end + end +end diff --git a/src/lucky/attachment/utilities.cr b/src/lucky/attachment/utilities.cr new file mode 100644 index 000000000..b933d5121 --- /dev/null +++ b/src/lucky/attachment/utilities.cr @@ -0,0 +1,28 @@ +module Lucky::Attachment + # Utility to work with a file IO. If the IO is already a File, yields it + # directly, Otherwise copies to a tempfile, yields it, then cleans up. + # + # ``` + # Lucky::Attachment.with_file(io) do |file| + # # file is guaranteed to be a File with a path + # end + # ``` + # + def self.with_file(io : IO, &) + if io.is_a?(File) + yield io + else + File.tempfile("lucky-attachment") do |tempfile| + IO.copy(io, tempfile) + tempfile.rewind + yield tempfile + end + end + end + + def self.with_file(uploaded_file : StoredFile, &) + uploaded_file.download do |tempfile| + yield tempfile + end + end +end diff --git a/src/lucky/uploaded_file.cr b/src/lucky/uploaded_file.cr index 02ad0f49b..9f9d7a424 100644 --- a/src/lucky/uploaded_file.cr +++ b/src/lucky/uploaded_file.cr @@ -39,6 +39,17 @@ class Lucky::UploadedFile tempfile.size.zero? end + # Attempts to extract the content type from the part's headers. + # + # ``` + # uploaded_file_object.content_type + # # => "image/png" + # ``` + # + def content_type : String? + @part.headers["Content-Type"]?.try(&.split(';').first.strip) + end + # Avram::Uploadable needs to be updated when this is removed @[Deprecated("`metadata` deprecated. Each method on metadata is accessible directly on Lucky::UploadedFile")] def metadata : HTTP::FormData::FileMetadata