vk_native: composite window-space alpha into the atlas (#1780)

src/xrt/auxiliary/vk/vk_hud_blend.c

@@ -109,6 +109,15 @@ bool
 vk_hud_blend_init(struct vk_hud_blend *blend,
                    struct vk_bundle *vk,
                    VkFormat target_fmt)
+{
+	return vk_hud_blend_init_ex(blend, vk, target_fmt, false);
+}
+
+bool
+vk_hud_blend_init_ex(struct vk_hud_blend *blend,
+                     struct vk_bundle *vk,
+                     VkFormat target_fmt,
+                     bool write_alpha)
 {
 	VkResult ret;
 
@@ -260,9 +269,9 @@ vk_hud_blend_init(struct vk_hud_blend *blend,
 	    .rasterizationSamples = VK_SAMPLE_COUNT_1_BIT};
 	VkPipelineColorBlendAttachmentState blend_att = {
 	    .blendEnable = VK_TRUE,
-	    // Don't touch the framebuffer's alpha channel — leaves whatever the
-	    // present path expects intact (some compositors treat alpha as window
-	    // transparency even with COMPOSITE_ALPHA_OPAQUE_BIT).
+	    // Default: don't touch the framebuffer's alpha channel — leaves
+	    // whatever the present path expects intact (some compositors treat
+	    // alpha as window transparency even with COMPOSITE_ALPHA_OPAQUE_BIT).
 	    .colorWriteMask = VK_COLOR_COMPONENT_R_BIT | VK_COLOR_COMPONENT_G_BIT |
 	                      VK_COLOR_COMPONENT_B_BIT,
 	    .srcColorBlendFactor = VK_BLEND_FACTOR_SRC_ALPHA,
@@ -271,6 +280,14 @@ vk_hud_blend_init(struct vk_hud_blend *blend,
 	    .srcAlphaBlendFactor = VK_BLEND_FACTOR_ZERO,
 	    .dstAlphaBlendFactor = VK_BLEND_FACTOR_ONE,
 	    .alphaBlendOp = VK_BLEND_OP_ADD};
+	if (write_alpha) {
+		// #1780: alpha "over" as well, so a HUD over a transparent atlas
+		// region (alpha 0) is not see-through. Alpha only ever grows, so an
+		// opaque target (alpha 1) is unchanged.
+		blend_att.colorWriteMask |= VK_COLOR_COMPONENT_A_BIT;
+		blend_att.srcAlphaBlendFactor = VK_BLEND_FACTOR_ONE;
+		blend_att.dstAlphaBlendFactor = VK_BLEND_FACTOR_ONE_MINUS_SRC_ALPHA;
+	}
 	VkPipelineColorBlendStateCreateInfo cb = {
 	    .sType = VK_STRUCTURE_TYPE_PIPELINE_COLOR_BLEND_STATE_CREATE_INFO,
 	    .attachmentCount = 1,

src/xrt/auxiliary/vk/vk_hud_blend.h

@@ -83,6 +83,25 @@ vk_hud_blend_init(struct vk_hud_blend *blend,
                    struct vk_bundle *vk,
                    VkFormat target_fmt);
 
+/*!
+ * Like vk_hud_blend_init(), choosing whether the HUD's alpha is composited
+ * into the target too (#1780).
+ *
+ * With @p write_alpha false (vk_hud_blend_init) the target's alpha channel is
+ * left untouched. With it true the HUD is composited "over" in alpha as well
+ * (src ONE, dst ONE_MINUS_SRC_ALPHA, all channels written), so the stamped
+ * region gets at least the HUD's coverage. Use that when the target's alpha
+ * is the window's transparency (a transparent session's atlas); it matches
+ * the D3D11 window-space blend (#225).
+ *
+ * @ingroup aux_vk
+ */
+bool
+vk_hud_blend_init_ex(struct vk_hud_blend *blend,
+                     struct vk_bundle *vk,
+                     VkFormat target_fmt,
+                     bool write_alpha);
+
 /*!
  * Record alpha-blended HUD draw commands.
  *

src/xrt/compositor/vk_native/comp_vk_native_compositor.c

@@ -2471,7 +2471,13 @@ vk_compositor_render_window_space_into_atlas(struct comp_vk_native_compositor *c
 	if (!c->window_space_blend.initialized && !c->window_space_blend_attempted) {
 		c->window_space_blend_attempted = true;
 		VkFormat atlas_fmt = (VkFormat)comp_vk_native_renderer_get_format(c->renderer);
-		if (!vk_hud_blend_init(&c->window_space_blend, vk, atlas_fmt)) {
+		// #1780: write alpha. The atlas alpha is the window's transparency in
+		// a transparent session, so an RGB-only stamp leaves the HUD invisible
+		// wherever no 3D content sits under it. Same blend as D3D11 (#225);
+		// an opaque atlas (alpha 1) is unchanged by it. Not keyed on the
+		// session's transparency: that can switch on mid-session (lazy
+		// transparency) and this pipeline is built once.
+		if (!vk_hud_blend_init_ex(&c->window_space_blend, vk, atlas_fmt, true)) {
 			U_LOG_E("[VK native] window-space alpha-blend init failed; "
 			        "layers will be skipped");
 		}