Menus spawned by layershell clients are not size constrained

[Consolatis]

Once a menu gets larger than the screen it will not restrict its size to gdk_monitor_get_workarea() like GTK usually seems to do when not using layershell.

I've added a reproducer based on an implementation by @LBCrion for sfwbar below.
If changing use_layershell=True to False in the bottom, the usual GTK behavior kicks in which constrains the menu window to the workspace size and adds scroll buttons. If that boolean is kept as True instead, the window will be used as a layershell window and an additional _clamp_menu() callback will be installed. Without that callback (e.g. by removing the connect() call or stubbing out _clamp_menu()) the issue become visible.

I am not sure if this can (and should) be handled by gtk-layer-shell internally or if that is something that users of the library have to take care of. In the second case it should likely be documented somewhere that this might be required depending on the expected menu sizes.

Ref: LBCrion/sfwbar#75
Ref: LBCrion/sfwbar@760e68e

Python reproducer:

#!/usr/bin/env python3

import gi
gi.require_version("Gtk", "3.0")
gi.require_version('GtkLayerShell', '0.1')
from gi.repository import Gtk, GtkLayerShell

class Window(Gtk.Window):
	def __init__(self, use_layershell):
		super().__init__()
		if use_layershell:
			GtkLayerShell.init_for_window(self)
			GtkLayerShell.set_layer(self, GtkLayerShell.Layer.TOP)

		menu_item = self.overflow_menu()
		if use_layershell:
			menu_item.get_submenu().connect(
				'popped-up', self._clamp_menu
			)

		self.tray = Gtk.MenuBar()
		self.tray.add(Gtk.MenuItem(label="Empty"))
		self.tray.add(menu_item)
		self.add(self.tray)

		self.set_default_size(200, -1)
		self.show_all()

	def _clamp_menu(self, menu, *args):
		#win = menu.get_ancestor(Gtk.Window)
		win = menu.get_toplevel()
		win = win.get_window()

		display = win.get_display()
		monitor = display.get_monitor_at_window(win)
		workarea = monitor.get_workarea()

		menu_width = win.get_width()
		menu_height = win.get_height()
		scale = win.get_scale_factor()
		target_width = workarea.width / scale
		target_height = workarea.height / scale

		if menu_width > target_width or menu_height > target_height:
			win.resize(
				min(menu_width, target_width),
				min(menu_height, target_height)
			)

	def overflow_menu(self):
		overflow_menu = Gtk.Menu()
		for i in range(200):
			overflow_menu.add(Gtk.MenuItem(label=f"Overflow-{i:>3d}"))
		overflow_item = Gtk.MenuItem(label="Overflow")
		overflow_item.set_submenu(overflow_menu)
		return overflow_item

def main():
	win = Window(use_layershell=True)
	win.connect('destroy', Gtk.main_quit)
	try:
		Gtk.main()
	except KeyboardInterrupt:
		print()

if __name__ == '__main__':
	main()

[wmww]

I can't reproduce the "correct" behavior, even without layer shell. For me this simplified script (with all the manual overflow handling and layer shell stuff removed) overflows the screen without any scroll buttons. I'm using GTK v3.24.33 and Sway v1.7:

#!/usr/bin/env python3

import gi
gi.require_version("Gtk", "3.0")
from gi.repository import Gtk

class Window(Gtk.Window):
	def __init__(self):
		super().__init__()

		menu_item = self.overflow_menu()

		self.tray = Gtk.MenuBar()
		self.tray.add(Gtk.MenuItem(label="Empty"))
		self.tray.add(menu_item)
		self.add(self.tray)

		self.set_default_size(200, -1)
		self.show_all()

	def overflow_menu(self):
		overflow_menu = Gtk.Menu()
		for i in range(200):
			overflow_menu.add(Gtk.MenuItem(label=f"Overflow-{i:>3d}"))
		overflow_item = Gtk.MenuItem(label="Overflow")
		overflow_item.set_submenu(overflow_menu)
		return overflow_item

def main():
	win = Window()
	win.connect('destroy', Gtk.main_quit)
	try:
		Gtk.main()
	except KeyboardInterrupt:
		print()

if __name__ == '__main__':
	main()

Interestingly setting GDK_BACKEND=x11 does allow menu scrolling, so the problem is Wayland-specific.

[Consolatis]

Just tested your modified version again on labwc (also wlroots based) and it constrains the menu correctly for me on Gtk 3.24.24.
In both cases: GDK_BACKEND=x11 and GDK_BACKEND=wayland. Sway (some variant of version 1.8) works as well whereas Sway 1.7 doesn't work for me either. So it seems like an issue with sway that has been fixed in the meantime.