Added per-pixel alpha blending capability to wxOverlay under wxMSW

By default, wxMSW overlay uses a constant alpha for the window's opacity which
can be changed by the added wxOverlay::SetOpacity() function. For some applications,
that's all they need. But for others, per-pixel alpha blending is a requirement.
This capability can be enabled by calling SetOpacity(-1) before initializing the
overlay. But the drawing must be done via a graphics context because the standard DC
under MSW does not support alpha drawing.

Also request that the overlay window be positioned below any floating windows
on the target (if any).

include/wx/overlay.h

@@ -34,6 +34,8 @@ public:
 
     bool IsNative() const;
 
+    void SetOpacity(int alpha);
+
 private:
     friend class WXDLLIMPEXP_FWD_CORE wxDCOverlay;
 

include/wx/private/overlay.h

@@ -38,6 +38,7 @@ public:
     virtual void EndDrawing(wxDC* dc) = 0;
     virtual void Clear(wxDC* dc) = 0;
     virtual void Reset() = 0;
+    virtual void SetOpacity(int WXUNUSED(alpha)) { }
 };
 
 #endif // _WX_PRIVATE_OVERLAY_H_

interface/wx/overlay.h

@@ -27,6 +27,25 @@ public:
        example, when the window content has been changed and repainted.
     */
     void Reset();
+
+    /**
+        Sets or unsets constant opacity of the overlay window.
+
+        If @a alpha is `-1`, use per-pixel alpha blending, otherwise use the
+        given alpha value for all pixels.
+
+        @note Drawing on the overlay window is opaque by default under wxMSW.
+        You have to call SetOpacity(-1) before initializing the overlay and use
+        a graphics context to do the actual drawing if you want per-pixel alpha
+        blending. Or, assuming that SetOpacity(-1) wasn't called, explicitly set
+        the alpha value (at any time) to change the overlay window's opacity.
+
+        @note This is currently only implemented for wxMSW and does nothing in
+        the other ports.
+
+        @since 3.3.0
+    */
+    void SetOpacity(int alpha);
 };
 
 

src/common/overlaycmn.cpp

@@ -74,6 +74,12 @@ void wxOverlay::Reset()
     wxASSERT_MSG(m_inDrawing==false,wxT("cannot reset overlay during drawing"));
     m_impl->Reset();
 }
+
+void wxOverlay::SetOpacity(int alpha)
+{
+    m_impl->SetOpacity(alpha);
+}
+
 // ----------------------------------------------------------------------------
 
 wxOverlay::Impl::~Impl()

src/msw/overlay.cpp

@@ -13,10 +13,40 @@
 #include "wx/msw/dc.h"
 #include "wx/msw/private.h"
 
+// A note about updating the overlay window :
+// ------------------------------------------
+// Although both SetLayeredWindowAttributes and UpdateLayeredWindow can be used
+// to update the overlay window, the latter is more flexible because it can perform
+// per-pixel alpha blending like the other ports do. But this requires the DC to
+// support alpha drawing, which can only be done via graphics contexts. That's why
+// it's not used by default and is only enabled by SetOpacity(-1) call before the
+// overlay window is initialized. Like this for example:
+//      ...
+//      m_overlay.SetOpacity(-1);
+//      wxClientDC dc(win);
+//      wxDCOverlay overlaydc(m_overlay, &dc);
+//      wxGCDC gdc(dc);
+//      "use gdc for drawing"
+//      ...
+//
+// Notice that the opacity of the overlay window is opaque by default, assuming that
+// SetOpacity(-1) wasn't called. Call SetOpacity(alpha) at any time to change that.
+// E.g.: SetOpacity(128) will make the overlay window semi-transparent.
+//
+// Notice also that SetOpacity(-1) has no effect if called after the overlay window
+// is initialized. Likewise, calling SetOpacity(alpha) has no effect if SetOpacity(-1)
+// was effectively called before the overlay window is initialized.
+//
+//    From MSDN remarks:
+//    ------------------
+//        Note that once SetLayeredWindowAttributes has been called for a
+//    layered window, subsequent UpdateLayeredWindow calls will fail until
+//    the layering style bit is cleared and set again.
+//
+
 namespace // anonymous
 {
-
-wxWindow* wxCreateOverlayWindow(const wxRect& rect)
+wxWindow* wxCreateOverlayWindow(const wxRect& rect, int alpha, HWND hWndInsertAfter)
 {
     auto overlayWin = new wxWindow();
 
@@ -34,15 +64,19 @@ wxWindow* wxCreateOverlayWindow(const wxRect& rect)
 
     overlayWin->SetBackgroundStyle(wxBG_STYLE_PAINT);
 
-    if ( !::SetLayeredWindowAttributes(GetHwndOf(overlayWin), 0, 128,
-                                       LWA_COLORKEY | LWA_ALPHA) )
+    if ( alpha >= 0 )
     {
-        wxLogLastError(wxS("SetLayeredWindowAttributes()"));
+        if ( !::SetLayeredWindowAttributes(GetHwndOf(overlayWin), 0, alpha,
+                                           LWA_COLORKEY | LWA_ALPHA) )
+        {
+            wxLogLastError(wxS("SetLayeredWindowAttributes()"));
+        }
     }
+    //else: UpdateLayeredWindow will be used to update the overlay window
 
     // We intentionally don't use WS_VISIBLE when creating this window to avoid
     // stealing activation from the parent, so show it using SWP_NOACTIVATE now.
-    ::SetWindowPos(GetHwndOf(overlayWin), nullptr, 0, 0, 0, 0,
+    ::SetWindowPos(GetHwndOf(overlayWin), hWndInsertAfter, 0, 0, 0, 0,
                    SWP_NOSIZE |
                    SWP_NOMOVE |
                    SWP_NOREDRAW |
@@ -68,6 +102,13 @@ public:
     virtual void EndDrawing(wxDC* dc) override;
     virtual void Clear(wxDC* dc) override;
 
+    // To use per-pixel alpha blending, call this function with -1 before the
+    // overlay is initialized. Or with another value to change the opacity of
+    // the overlay window if you like a constant opacity.
+    virtual void SetOpacity(int alpha) override;
+
+    bool IsUsingConstantOpacity() const { return m_alpha >= 0; }
+
 public:
     // window the overlay is associated with
     wxWindow* m_window = nullptr;
@@ -77,6 +118,15 @@ public:
 
     wxRect      m_rect;
 
+    // This variable defines how our overlay window is updated:
+    // - If set to -1, UpdateLayeredWindow will be used, which means the overlay
+    //   window will use per-pixel alpha blending with the values in the source
+    //   bitmap.
+    // - If set to a value between (0..255) a constant opacity will be applied
+    //   to the entire overlay window. SetLayeredWindowAttributes will be used
+    //   to update the overlay window in this case.
+    int m_alpha = wxALPHA_OPAQUE;
+
     // Drawing on the overlay window is achieved by hijacking the existing wxDC.
     // i.e. any drawing done through wxDC should go to the offscreen bitmap m_bitmap
     // which will eventually be drawn on the overlay window.
@@ -103,9 +153,24 @@ void wxOverlayImpl::Init(wxDC* dc, int , int , int , int )
     m_rect = m_window->GetClientRect();
     m_window->ClientToScreen(&m_rect.x, &m_rect.y);
 
-    m_bitmap.CreateWithLogicalSize(m_rect.GetSize(), m_window->GetDPIScaleFactor());
+    if ( IsUsingConstantOpacity() )
+    {
+        m_bitmap.CreateWithDIPSize(m_rect.GetSize(), m_window->GetDPIScaleFactor());
+    }
+    else
+    {
+        m_bitmap.CreateWithDIPSize(m_rect.GetSize(), m_window->GetDPIScaleFactor(), 32);
+        m_bitmap.UseAlpha();
+    }
 
-    m_overlayWindow = wxCreateOverlayWindow(m_rect);
+    // We want the overlay window precede the associated window in Z-order, but
+    // under any floating window i.e. be positioned between them. And to achieve
+    // this, we must insert it after the _previous_ window in the same order, as
+    // then it will be just before this one.
+    // GW_HWNDPREV: Returns a handle to the window above the given window. (MSDN)
+    const auto win = wxGetTopLevelParent(m_window);
+    HWND hWndInsertAfter = ::GetNextWindow(GetHwndOf(win), GW_HWNDPREV);
+    m_overlayWindow = wxCreateOverlayWindow(m_rect, m_alpha, hWndInsertAfter);
 }
 
 void wxOverlayImpl::BeginDrawing(wxDC* dc)
@@ -127,6 +192,20 @@ void wxOverlayImpl::EndDrawing(wxDC* dc)
 {
     wxCHECK_RET( IsOk(), wxS("overlay not initialized") );
 
+    if ( !IsUsingConstantOpacity() )
+    {
+        POINT ptSrc{0, 0};
+        POINT ptDst{m_rect.x, m_rect.y};
+        SIZE size{m_rect.width, m_rect.height};
+        BLENDFUNCTION blendFunc{AC_SRC_OVER, 0, 255, AC_SRC_ALPHA};
+
+        if ( !::UpdateLayeredWindow(GetHwndOf(m_overlayWindow), nullptr, &ptDst, &size,
+                                    GetHdcOf(m_memDC), &ptSrc, 0, &blendFunc, ULW_ALPHA) )
+        {
+            wxLogLastError(wxS("UpdateLayeredWindow()"));
+        }
+    }
+
     m_memDC.SelectObject(wxNullBitmap);
 
     auto impl = dc->GetImpl();
@@ -135,8 +214,11 @@ void wxOverlayImpl::EndDrawing(wxDC* dc)
     msw_impl->SetHDC(m_hdc); // restore the original hdc
     msw_impl->UpdateClipBox();
 
-    wxWindowDC winDC(m_overlayWindow);
-    winDC.DrawBitmap(m_bitmap, wxPoint(0, 0));
+    if ( IsUsingConstantOpacity() )
+    {
+        wxWindowDC winDC(m_overlayWindow);
+        winDC.DrawBitmap(m_bitmap, wxPoint(0, 0));
+    }
 }
 
 void wxOverlayImpl::Clear(wxDC* WXUNUSED(dc))
@@ -156,6 +238,28 @@ void wxOverlayImpl::Reset()
     {
         m_overlayWindow->Destroy();
         m_overlayWindow = nullptr;
+
+        m_alpha = wxALPHA_OPAQUE;
+    }
+}
+
+void wxOverlayImpl::SetOpacity(int alpha)
+{
+#if wxUSE_GRAPHICS_CONTEXT
+    if ( !IsOk() )
+    {
+        m_alpha = wxClip(alpha, -1, 255);
+    }
+    else if ( IsUsingConstantOpacity() )
+#endif // wxUSE_GRAPHICS_CONTEXT
+    {
+        m_alpha = wxClip(alpha, 0, 255);
+
+        if ( !::SetLayeredWindowAttributes(GetHwndOf(m_overlayWindow), 0, m_alpha,
+                                           LWA_COLORKEY | LWA_ALPHA) )
+        {
+            wxLogLastError(wxS("SetLayeredWindowAttributes()"));
+        }
     }
 }