From 1f64bd1b8c937acf87dc1648a3c2bfe6311b3588 Mon Sep 17 00:00:00 2001 From: gitadityakumar Date: Mon, 31 Aug 2026 23:33:01 +0530 Subject: [PATCH 1/3] feat(providers): add DeviantArt stock & creative artwork provider --- src/providers/deviantart.zig | 272 +++++++++++++++++++++++++++++++++++ 1 file changed, 272 insertions(+) create mode 100644 src/providers/deviantart.zig diff --git a/src/providers/deviantart.zig b/src/providers/deviantart.zig new file mode 100644 index 0000000..5284774 --- /dev/null +++ b/src/providers/deviantart.zig @@ -0,0 +1,272 @@ +//! DeviantArt provider — creative reference photos, stock imagery, and digital art. +//! Search: GET https://www.deviantart.com/tag/{tag} +//! Download: Direct high-resolution image assets from Wixmp CDN. + +const std = @import("std"); +const asset_mod = @import("../asset.zig"); +const http_client = @import("../http_client.zig"); +const download_mod = @import("../download.zig"); +const Allocator = std.mem.Allocator; +const Io = std.Io; +const Asset = asset_mod.Asset; +const SearchResult = asset_mod.SearchResult; + +pub const id = "deviantart"; +pub const name = "DeviantArt"; + +fn formatTag(allocator: Allocator, query: []const u8) ![]u8 { + const trimmed = std.mem.trim(u8, query, " \t\r\n"); + var list: std.ArrayList(u8) = .empty; + errdefer list.deinit(allocator); + + for (trimmed) |c| { + if (std.ascii.isAlphanumeric(c)) { + try list.append(allocator, std.ascii.toLower(c)); + } + } + if (list.items.len == 0) { + try list.appendSlice(allocator, "stock"); + } + return list.toOwnedSlice(allocator); +} + +fn htmlUnescape(allocator: Allocator, s: []const u8) ![]u8 { + var list: std.ArrayList(u8) = .empty; + errdefer list.deinit(allocator); + var i: usize = 0; + while (i < s.len) { + if (s[i] == '&') { + if (std.mem.startsWith(u8, s[i..], "&")) { + try list.append(allocator, '&'); + i += 5; + continue; + } + if (std.mem.startsWith(u8, s[i..], """)) { + try list.append(allocator, '"'); + i += 6; + continue; + } + if (std.mem.startsWith(u8, s[i..], "–")) { + try list.append(allocator, '-'); + i += 7; + continue; + } + if (std.mem.startsWith(u8, s[i..], "—")) { + try list.append(allocator, '-'); + i += 7; + continue; + } + if (std.mem.startsWith(u8, s[i..], "'") or std.mem.startsWith(u8, s[i..], "'")) { + try list.append(allocator, '\''); + i += if (std.mem.startsWith(u8, s[i..], "'")) 5 else 6; + continue; + } + } + try list.append(allocator, s[i]); + i += 1; + } + return list.toOwnedSlice(allocator); +} + +fn parseSearch(allocator: Allocator, html: []const u8, limit: u32) ![]Asset { + var assets: std.ArrayList(Asset) = .empty; + errdefer { + for (assets.items) |*a| a.deinit(allocator); + assets.deinit(allocator); + } + + var seen: std.StringHashMapUnmanaged(void) = .empty; + defer { + var it = seen.keyIterator(); + while (it.next()) |k| allocator.free(k.*); + seen.deinit(allocator); + } + + const art_marker = "/art/"; + var pos: usize = 0; + while (assets.items.len < limit) { + const found = std.mem.indexOfPos(u8, html, pos, art_marker) orelse break; + pos = found + art_marker.len; + + // Search backward for the enclosing tag + const tag_start = std.mem.lastIndexOfScalar(u8, html[0..found], '<') orelse found; + const window_end = @min(html.len, found + 1500); + const win = html[tag_start..window_end]; + + // Extract href + const href_key = "href=\""; + const href_i = std.mem.indexOf(u8, win, href_key) orelse continue; + const href_start = href_i + href_key.len; + const href_end = std.mem.indexOfScalarPos(u8, win, href_start, '"') orelse continue; + const href_val = win[href_start..href_end]; + if (std.mem.indexOf(u8, href_val, "/art/") == null) continue; + + // Extract img src + const src_key = "src=\""; + const src_i = std.mem.indexOf(u8, win, src_key) orelse continue; + const src_start = src_i + src_key.len; + const src_end = std.mem.indexOfScalarPos(u8, win, src_start, '"') orelse continue; + const src_val = win[src_start..src_end]; + if (std.mem.indexOf(u8, src_val, "wixmp.com") == null) continue; + + // Extract img alt + const alt_key = "alt=\""; + const alt_val: []const u8 = blk: { + if (std.mem.indexOf(u8, win, alt_key)) |ai| { + const as = ai + alt_key.len; + const ae = std.mem.indexOfScalarPos(u8, win, as, '"') orelse break :blk ""; + break :blk win[as..ae]; + } + break :blk ""; + }; + + // Parse author and slug from href: https://www.deviantart.com/{author}/art/{slug}-{id} + var u = href_val; + if (std.mem.startsWith(u8, u, "https://www.deviantart.com/")) { + u = u["https://www.deviantart.com/".len..]; + } else if (std.mem.startsWith(u8, u, "https://deviantart.com/")) { + u = u["https://deviantart.com/".len..]; + } else if (std.mem.startsWith(u8, u, "/")) { + u = u[1..]; + } + + const slash_idx = std.mem.indexOfScalar(u8, u, '/') orelse continue; + const author_str = u[0..slash_idx]; + const rest = u[slash_idx + 1 ..]; + const art_prefix = "art/"; + if (!std.mem.startsWith(u8, rest, art_prefix)) continue; + const slug_id = rest[art_prefix.len..]; + if (slug_id.len == 0) continue; + + const pid = try std.fmt.allocPrint(allocator, "{s}-{s}", .{ author_str, slug_id }); + errdefer allocator.free(pid); + + if (seen.contains(pid)) { + allocator.free(pid); + continue; + } + try seen.put(allocator, try allocator.dupe(u8, pid), {}); + + const title_unesc = try htmlUnescape(allocator, if (alt_val.len > 0) alt_val else slug_id); + defer allocator.free(title_unesc); + + const desc = if (title_unesc.len > 0) title_unesc else pid; + const desc_owned = try allocator.dupe(u8, desc); + errdefer allocator.free(desc_owned); + const prompt_owned = try allocator.dupe(u8, desc); + errdefer allocator.free(prompt_owned); + const img_owned = try allocator.dupe(u8, src_val); + errdefer allocator.free(img_owned); + const thumb_owned = try allocator.dupe(u8, src_val); + errdefer allocator.free(thumb_owned); + const prov_owned = try allocator.dupe(u8, id); + errdefer allocator.free(prov_owned); + const auth_owned = try allocator.dupe(u8, author_str); + errdefer allocator.free(auth_owned); + + var full_page_url: []u8 = undefined; + if (std.mem.startsWith(u8, href_val, "http")) { + full_page_url = try allocator.dupe(u8, href_val); + } else { + full_page_url = try std.fmt.allocPrint(allocator, "https://www.deviantart.com/{s}", .{u}); + } + defer allocator.free(full_page_url); + + var meta_aw: Io.Writer.Allocating = .init(allocator); + defer meta_aw.deinit(); + try meta_aw.writer.print("{{\"source\":\"deviantart\",\"page\":\"{s}\"}}", .{full_page_url}); + const meta = try meta_aw.toOwnedSlice(); + + try assets.append(allocator, .{ + .id = pid, + .description = desc_owned, + .prompt = prompt_owned, + .image_url = img_owned, + .thumbnail_url = thumb_owned, + .provider = prov_owned, + .author = auth_owned, + .width = null, + .height = null, + .metadata_json = meta, + }); + } + + return try assets.toOwnedSlice(allocator); +} + +pub fn search( + client: *std.http.Client, + allocator: Allocator, + query: []const u8, + limit: u32, +) !SearchResult { + const tag = try formatTag(allocator, query); + defer allocator.free(tag); + + const search_url = try std.fmt.allocPrint(allocator, "https://www.deviantart.com/tag/{s}", .{tag}); + defer allocator.free(search_url); + + var resp = try http_client.get(client, allocator, search_url, .{ + .accept = "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", + .referer = "https://www.deviantart.com/", + .max_redirects = 5, + }); + defer resp.deinit(); + if (resp.status != 200) return error.HttpStatus; + + const lim = @min(@max(limit, 1), 50); + const assets = try parseSearch(allocator, resp.body, lim); + + return .{ + .assets = assets, + .total = null, + .provider = try allocator.dupe(u8, id), + .query = try allocator.dupe(u8, query), + .allocator = allocator, + }; +} + +pub fn download( + client: *std.http.Client, + allocator: Allocator, + io: Io, + a: Asset, + output_dir: []const u8, +) !download_mod.Saved { + const url = a.image_url orelse return error.NoUrl; + const body = http_client.getBody(client, allocator, url, .{ + .referer = "https://www.deviantart.com/", + .accept = "image/avif,image/webp,image/apng,image/*,*/*;q=0.8", + .max_redirects = 5, + }) catch |err| switch (err) { + error.RateLimited => return error.RateLimited, + error.HttpStatus => return error.HttpStatus, + error.OutOfMemory => return error.OutOfMemory, + else => return error.Network, + }; + defer allocator.free(body); + if (body.len == 0) return error.HttpStatus; + + try download_mod.ensureDir(io, output_dir); + const ext = download_mod.guessExtension(url); + const path = try download_mod.uniquePath(allocator, io, output_dir, a.id, ext); + errdefer allocator.free(path); + const cwd = Io.Dir.cwd(); + cwd.writeFile(io, .{ .sub_path = path, .data = body }) catch return error.Io; + return .{ + .path = path, + .filename = std.fs.path.basename(path), + .bytes = body.len, + .allocator = allocator, + }; +} + +pub fn getPrompt(a: Asset) ?[]const u8 { + if (a.prompt) |p| if (p.len > 0) return p; + if (a.description.len > 0) return a.description; + return null; +} + +pub fn getUrl(a: Asset) ?[]const u8 { + return a.image_url; +} From d1718ff8dca88cb5603e4b72203a05d280078ec8 Mon Sep 17 00:00:00 2001 From: gitadityakumar Date: Mon, 31 Aug 2026 23:33:03 +0530 Subject: [PATCH 2/3] feat(registry): register deviantart provider and wire search and interactive CLI --- src/commands/search.zig | 9 +++++++++ src/interactive.zig | 9 +++++++++ src/providers/registry.zig | 4 +++- 3 files changed, 21 insertions(+), 1 deletion(-) diff --git a/src/commands/search.zig b/src/commands/search.zig index c101377..75a1762 100644 --- a/src/commands/search.zig +++ b/src/commands/search.zig @@ -16,6 +16,7 @@ const jaymantri = @import("../providers/jaymantri.zig"); const publicdomainarchive = @import("../providers/publicdomainarchive.zig"); const magdeleine = @import("../providers/magdeleine.zig"); const splitshire = @import("../providers/splitshire.zig"); +const deviantart = @import("../providers/deviantart.zig"); const stdio = @import("../stdio.zig"); const Allocator = std.mem.Allocator; const Io = std.Io; @@ -76,6 +77,9 @@ fn doSearch( if (std.ascii.eqlIgnoreCase(provider_id, "splitshire")) { return splitshire.search(client, allocator, query, limit); } + if (std.ascii.eqlIgnoreCase(provider_id, "deviantart")) { + return deviantart.search(client, allocator, query, limit); + } return error.UnknownProvider; } @@ -126,6 +130,9 @@ fn doDownload( if (std.ascii.eqlIgnoreCase(provider_id, "splitshire")) { return splitshire.download(client, allocator, io, a, output_dir); } + if (std.ascii.eqlIgnoreCase(provider_id, "deviantart")) { + return deviantart.download(client, allocator, io, a, output_dir); + } return error.UnknownProvider; } @@ -143,6 +150,7 @@ fn doPrompt(provider_id: []const u8, a: Asset) ?[]const u8 { if (std.ascii.eqlIgnoreCase(provider_id, "publicdomainarchive")) return publicdomainarchive.getPrompt(a); if (std.ascii.eqlIgnoreCase(provider_id, "magdeleine")) return magdeleine.getPrompt(a); if (std.ascii.eqlIgnoreCase(provider_id, "splitshire")) return splitshire.getPrompt(a); + if (std.ascii.eqlIgnoreCase(provider_id, "deviantart")) return deviantart.getPrompt(a); return a.prompt orelse a.description; } @@ -160,6 +168,7 @@ fn doUrl(provider_id: []const u8, a: Asset) ?[]const u8 { if (std.ascii.eqlIgnoreCase(provider_id, "publicdomainarchive")) return publicdomainarchive.getUrl(a); if (std.ascii.eqlIgnoreCase(provider_id, "magdeleine")) return magdeleine.getUrl(a); if (std.ascii.eqlIgnoreCase(provider_id, "splitshire")) return splitshire.getUrl(a); + if (std.ascii.eqlIgnoreCase(provider_id, "deviantart")) return deviantart.getUrl(a); return a.image_url; } diff --git a/src/interactive.zig b/src/interactive.zig index 260cdd1..2a1fcf2 100644 --- a/src/interactive.zig +++ b/src/interactive.zig @@ -16,6 +16,7 @@ const jaymantri = @import("providers/jaymantri.zig"); const publicdomainarchive = @import("providers/publicdomainarchive.zig"); const magdeleine = @import("providers/magdeleine.zig"); const splitshire = @import("providers/splitshire.zig"); +const deviantart = @import("providers/deviantart.zig"); const registry = @import("providers/registry.zig"); const stdio = @import("stdio.zig"); const Allocator = std.mem.Allocator; @@ -73,6 +74,9 @@ fn searchProvider( if (std.ascii.eqlIgnoreCase(provider_id, "splitshire")) { return splitshire.search(client, allocator, query, config.default_limit); } + if (std.ascii.eqlIgnoreCase(provider_id, "deviantart")) { + return deviantart.search(client, allocator, query, config.default_limit); + } return error.UnknownProvider; } @@ -122,6 +126,9 @@ fn downloadAsset( if (std.ascii.eqlIgnoreCase(provider_id, "splitshire")) { return splitshire.download(client, allocator, io, a, config.default_output_dir); } + if (std.ascii.eqlIgnoreCase(provider_id, "deviantart")) { + return deviantart.download(client, allocator, io, a, config.default_output_dir); + } return error.UnknownProvider; } @@ -139,6 +146,7 @@ fn promptOf(provider_id: []const u8, a: Asset) ?[]const u8 { if (std.ascii.eqlIgnoreCase(provider_id, "publicdomainarchive")) return publicdomainarchive.getPrompt(a); if (std.ascii.eqlIgnoreCase(provider_id, "magdeleine")) return magdeleine.getPrompt(a); if (std.ascii.eqlIgnoreCase(provider_id, "splitshire")) return splitshire.getPrompt(a); + if (std.ascii.eqlIgnoreCase(provider_id, "deviantart")) return deviantart.getPrompt(a); return a.prompt; } @@ -156,6 +164,7 @@ fn urlOf(provider_id: []const u8, a: Asset) ?[]const u8 { if (std.ascii.eqlIgnoreCase(provider_id, "publicdomainarchive")) return publicdomainarchive.getUrl(a); if (std.ascii.eqlIgnoreCase(provider_id, "magdeleine")) return magdeleine.getUrl(a); if (std.ascii.eqlIgnoreCase(provider_id, "splitshire")) return splitshire.getUrl(a); + if (std.ascii.eqlIgnoreCase(provider_id, "deviantart")) return deviantart.getUrl(a); return a.image_url; } diff --git a/src/providers/registry.zig b/src/providers/registry.zig index 3d0db3e..2b50c79 100644 --- a/src/providers/registry.zig +++ b/src/providers/registry.zig @@ -14,6 +14,7 @@ const jaymantri = @import("jaymantri.zig"); const publicdomainarchive = @import("publicdomainarchive.zig"); const magdeleine = @import("magdeleine.zig"); const splitshire = @import("splitshire.zig"); +const deviantart = @import("deviantart.zig"); pub const ProviderInfo = struct { id: []const u8, @@ -35,6 +36,7 @@ pub fn list() []const ProviderInfo { .{ .id = publicdomainarchive.id, .name = publicdomainarchive.name }, .{ .id = magdeleine.id, .name = magdeleine.name }, .{ .id = splitshire.id, .name = splitshire.name }, + .{ .id = deviantart.id, .name = deviantart.name }, }; } @@ -53,5 +55,5 @@ pub fn displayName(id: []const u8) ?[]const u8 { } pub fn availableIds() []const u8 { - return "aura, unsplash, isorepublic, picjumbo, foodiesfeed, picography, gratisography, startupstockphotos, burst, jaymantri, publicdomainarchive, magdeleine, splitshire"; + return "aura, unsplash, isorepublic, picjumbo, foodiesfeed, picography, gratisography, startupstockphotos, burst, jaymantri, publicdomainarchive, magdeleine, splitshire, deviantart"; } From 90953850f787d01b358fc84cb4a3e73e0602e6f6 Mon Sep 17 00:00:00 2001 From: gitadityakumar Date: Mon, 31 Aug 2026 23:33:06 +0530 Subject: [PATCH 3/3] docs: update provider reference and tracker for DeviantArt --- README.md | 2 ++ docs/providers.md | 13 +++++++++++++ docs/stock_photo_sites.md | 11 ++++++----- 3 files changed, 21 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 6383eae..4db3480 100644 --- a/README.md +++ b/README.md @@ -234,6 +234,7 @@ Provider and query may be given as positionals after `s` / `-s`, or with `-P` / | `publicdomainarchive` | [Public Domain Archive](https://publicdomainarchive.com) | 100% free public domain and vintage stock photos | Direct high-resolution photo downloads. | | `magdeleine` | [Magdeleine](https://magdeleine.co) | Hand-picked free high-resolution stock photography | Direct master resolution photo downloads. | | `splitshire` | [SplitShire](https://splitshire.com) | Free stock photos and artwork by Daniel Nanescu & community | Direct high-resolution photo downloads. | +| `deviantart` | [DeviantArt](https://deviantart.com) | Creative reference photos, stock imagery, and digital art | Direct image downloads via Wixmp CDN. | List providers at runtime via `ast help` (printed under **PROVIDERS**). @@ -296,6 +297,7 @@ src/ publicdomainarchive.zig magdeleine.zig splitshire.zig + deviantart.zig assets/ README banner and static assets docs/ Provider implementation notes install.sh curl | bash installer (Linux release binaries) diff --git a/docs/providers.md b/docs/providers.md index 228f345..ffb6117 100644 --- a/docs/providers.md +++ b/docs/providers.md @@ -157,3 +157,16 @@ This document summarizes the network endpoints, transport models, and image down * Author credited from user profile or Daniel Nanescu. * **Download**: Direct GET on high-resolution CDN assets. * **License**: Free for Personal and Commercial Use (SplitShire License / CC0). + +--- + +## 14. DeviantArt (`deviantart.com`) + +* **Search**: `GET https://www.deviantart.com/tag/{tag}` +* **Format**: SSR HTML with artwork cards, titles, author links, and direct Wixmp CDN URLs. +* **Extraction**: + * Title extracted and HTML unescaped from image `alt` attribute. + * Author parsed from artwork page link (`/{author}/art/...`). + * Image path extracted directly from signed Wixmp CDN URLs. +* **Download**: Direct GET on Wixmp CDN URL with browser headers. +* **License**: Free Creative Community Stock / Artwork Reference. diff --git a/docs/stock_photo_sites.md b/docs/stock_photo_sites.md index 9c1657c..db5f737 100644 --- a/docs/stock_photo_sites.md +++ b/docs/stock_photo_sites.md @@ -7,9 +7,9 @@ This document tracks all 24 stock photo websites listed in the [GrayGrids 21+ Be ## Summary Status - **Total Sites Listed**: 24 -- **Implemented & Working**: 12 (`unsplash`, `isorepublic`, `picjumbo`, `foodiesfeed`, `picography`, `gratisography`, `startupstockphotos`, `burst`, `jaymantri`, `publicdomainarchive`, `magdeleine`, `splitshire`) +- **Implemented & Working**: 13 (`unsplash`, `isorepublic`, `picjumbo`, `foodiesfeed`, `picography`, `gratisography`, `startupstockphotos`, `burst`, `jaymantri`, `publicdomainarchive`, `magdeleine`, `splitshire`, `deviantart`) - *(Additional Provider Implemented in `ast`)*: `aura` ([Aura.build](https://www.aura.build)) -- **Unimplemented / Blocked / Pending**: 12 +- **Unimplemented / Blocked / Pending**: 11 --- @@ -115,9 +115,10 @@ This document tracks all 24 stock photo websites listed in the [GrayGrids 21+ Be - **Status**: Unimplemented. - **Notes**: Large repository of free stock photography. -- [ ] **DeviantArt** ([deviantart.com](http://www.deviantart.com/)) - - **Status**: Unimplemented. - - **Notes**: Community art platform; requires auth/API for asset retrieval. +- [x] **DeviantArt** ([deviantart.com](http://www.deviantart.com/)) + - **CLI Provider**: `deviantart` + - **Status**: Implemented & verified. + - **Notes**: Tag search and direct image downloading via Wixmp CDN supported. - [ ] **Reshot** ([reshot.com](https://www.reshot.com/)) - **Status**: Unimplemented.