find a channel's top effect ID by writing above it and reading back

Which effect IDs a board has is a thing this tool measured by hand and
recorded in README, and a question `keyboard definitions` cannot answer
for a board that has no definition file: 1812 of the 2029 definitions
VIA carries name no effects at all.

QMK clamps rather than rejects. quantum/rgb_matrix/rgb_matrix.c maps a
mode at or above RGB_MATRIX_EFFECT_MAX down to EFFECT_MAX - 1, so one
write above the top returns the top itself. That makes the answer four
round trips per channel and no search, where a linear walk to the top
would be up to 47.

A firmware that ignores the write instead returns the value it already
held, which is the same answer the clamp gives when the channel already
sat on its last effect, so the two need not be told apart. The write is
not 0, because VIA's handler reads a 0 on the effect parameter as
"turn this channel off" and the probe would switch the lighting off
rather than ask a question.

The channel is left as it was found. What cannot be restored is a
channel that was off: VIA enables the channel for any nonzero effect
and exposes no enabled bit to read, which the doc comment says rather
than pretending otherwise.

internal/via/channel.go

@@ -75,3 +75,51 @@ func (p *Protocol) DetectChannels() ([]Channel, error) {
 	}
 	return present, nil
 }
+
+// effectValueID is the effect parameter's value ID. It is the same byte on every
+// lighting subsystem, like brightness.
+const effectValueID = 0x02
+
+// effectProbeValue is what EffectTop writes to find the top. It is above every
+// effect ID any firmware has, and it is not 0: QMK's VIA handler reads a 0 as
+// "turn this channel off", so a probe that wrote it would switch the lighting
+// off rather than ask a question.
+const effectProbeValue = 0xff
+
+// EffectTop returns the highest effect ID the keyboard's firmware accepts on a
+// channel, which is the number of effect slots the channel has minus one.
+//
+// It is found by writing above the top and reading back what the firmware kept.
+// QMK clamps rather than rejects: quantum/rgb_matrix/rgb_matrix.c maps a mode at
+// or above RGB_MATRIX_EFFECT_MAX down to EFFECT_MAX - 1, so a single write above
+// the top returns the top itself. A firmware that ignores the write instead
+// returns the value it already held, and that is the same answer the clamp gives
+// when the channel already sat on its last effect, so the two need not be told
+// apart. A returned value of 255 would mean the channel really does take it.
+//
+// The channel is left as it was found, because the probe otherwise leaves it
+// running the last effect. What it cannot restore is a channel that was off: VIA
+// enables the channel for any nonzero effect and exposes no enabled bit to read,
+// so a channel that was disabled comes back enabled. `disable` turns it off
+// again.
+func (p *Protocol) EffectTop(ch Channel) (int, error) {
+	previous, err := p.GetValue(ch, effectValueID)
+	if err != nil {
+		return 0, fmt.Errorf("read the effect on %s: %w", ch.Subsystem(), err)
+	}
+	if err := p.SetValue(ch, effectValueID, effectProbeValue); err != nil {
+		return 0, fmt.Errorf("probe the effect range on %s: %w", ch.Subsystem(), err)
+	}
+	top, readErr := p.GetValue(ch, effectValueID)
+	// The keyboard is holding the probe value until the original goes back, so
+	// the restore runs whether the read succeeded or not.
+	restoreErr := p.SetValue(ch, effectValueID, previous[0])
+	if readErr != nil {
+		return 0, fmt.Errorf("read back the effect range on %s: %w", ch.Subsystem(), readErr)
+	}
+	if restoreErr != nil {
+		return 0, fmt.Errorf("restore the effect on %s, which is left on %d: %w",
+			ch.Subsystem(), top[0], restoreErr)
+	}
+	return int(top[0]), nil
+}

internal/via/channel_test.go

@@ -126,6 +126,113 @@ func TestDetectChannelsReportsNoChannelsWhenNoneArePresent(t *testing.T) {
 	}
 }
 
+// effectResponse is one scripted answer to an effect get or set, carrying the
+// value the keyboard is left with.
+func effectResponse(command Message, ch Channel, value byte) []byte {
+	buf := make([]byte, 32)
+	buf[0] = byte(command)
+	buf[1] = byte(ch)
+	buf[2] = effectValueID
+	buf[3] = value
+	return buf
+}
+
+// QMK clamps rather than rejects: a mode at or above the top comes down to it.
+// A single write above the top therefore returns the top itself, which is the
+// whole reason the probe is one round trip instead of a search.
+func TestEffectTopReadsTheTopOffWhatTheFirmwareClampedTo(t *testing.T) {
+	const ch = ChannelRgbMatrix
+	transport := &fakeTransport{queue: [][]byte{
+		effectResponse(CustomGet, ch, 12),  // the effect the channel is on
+		effectResponse(CustomSet, ch, 255), // the probe, echoed
+		effectResponse(CustomGet, ch, 45),  // what the clamp left
+		effectResponse(CustomSet, ch, 12),  // the restore, echoed
+	}}
+	protocol := Protocol{handle: transport}
+
+	got, err := protocol.EffectTop(ch)
+	if err != nil {
+		t.Fatalf("EffectTop() error = %v", err)
+	}
+	if got != 45 {
+		t.Errorf("EffectTop() = %d, want 45", got)
+	}
+	if len(transport.reports) != 4 {
+		t.Fatalf("requests = %d, want 4 (read, probe, read back, restore)", len(transport.reports))
+	}
+	last := transport.reports[3]
+	if last[0] != byte(CustomSet) || last[3] != 12 {
+		t.Errorf("last request = %v, want the original effect 12 written back", last[:4])
+	}
+}
+
+// A channel already sitting on its last effect answers the probe with the value
+// it held, which is the same number the clamp would give. The two cases need not
+// be told apart, so the answer is still the top.
+func TestEffectTopReportsTheTopWhenTheChannelAlreadySatOnIt(t *testing.T) {
+	const ch = ChannelRgbMatrix
+	transport := &fakeTransport{queue: [][]byte{
+		effectResponse(CustomGet, ch, 45),
+		effectResponse(CustomSet, ch, 255),
+		effectResponse(CustomGet, ch, 45),
+		effectResponse(CustomSet, ch, 45),
+	}}
+	protocol := Protocol{handle: transport}
+
+	got, err := protocol.EffectTop(ch)
+	if err != nil {
+		t.Fatalf("EffectTop() error = %v", err)
+	}
+	if got != 45 {
+		t.Errorf("EffectTop() = %d, want 45", got)
+	}
+}
+
+// VIA reads a 0 on the effect parameter as "turn this channel off", so the probe
+// must not write one. The restore may: a channel that was on effect 0 goes back
+// to 0, which is the state it was in.
+func TestEffectTopProbesWithoutSwitchingTheChannelOff(t *testing.T) {
+	if effectProbeValue == 0 {
+		t.Fatal("effectProbeValue is 0, which switches the channel off instead of asking")
+	}
+
+	const ch = ChannelRgbMatrix
+	transport := &fakeTransport{queue: [][]byte{
+		effectResponse(CustomGet, ch, 0),
+		effectResponse(CustomSet, ch, effectProbeValue),
+		effectResponse(CustomGet, ch, 6),
+		effectResponse(CustomSet, ch, 0),
+	}}
+	protocol := Protocol{handle: transport}
+
+	if _, err := protocol.EffectTop(ch); err != nil {
+		t.Fatalf("EffectTop() error = %v", err)
+	}
+
+	probe := transport.reports[1]
+	if probe[0] != byte(CustomSet) || probe[2] != effectValueID {
+		t.Fatalf("request 1 = %v, want a set on the effect parameter", probe[:4])
+	}
+	if probe[3] != effectProbeValue {
+		t.Errorf("probe wrote effect %d, want %d", probe[3], effectProbeValue)
+	}
+}
+
+// An effect that cannot be read is a channel the probe cannot restore, so it
+// stops before it writes rather than leaving the channel on the last effect.
+func TestEffectTopStopsBeforeWritingWhenTheEffectCannotBeRead(t *testing.T) {
+	const ch = ChannelRgbMatrix
+	transport := &fakeTransport{queue: [][]byte{effectResponse(Unhandled, ch, 0)}}
+	protocol := Protocol{handle: transport}
+
+	if _, err := protocol.EffectTop(ch); err == nil {
+		t.Fatal("EffectTop() = nil error, want a failure for an effect that cannot be read")
+	}
+	if len(transport.reports) != 1 {
+		t.Errorf("requests = %d, want only the read that failed", len(transport.reports))
+	}
+}
+
 func TestChannelSubsystemNames(t *testing.T) {
 	tests := []struct {
 		channel Channel