read Vial's effect IDs, and tell a Vial keyboard from a VIA one

Vial keeps VIA's command set and adds its own behind a prefix byte, so
a keyboard on Vial firmware already answers everything this tool sends.
What it has that VIA does not is an answer to "which effects do you
have", asked without writing to the keyboard — where reading the range
by clamping writes 255 and puts it back.

Two things about the protocol were not what I assumed, and both came
out of reading vial-kb/vial-qmk rather than from memory:

VialRGB does not live behind the prefix. It answers the ordinary VIA
lighting command 0x08 and carries its own command in the channel byte, so
asking for the supported IDs is a custom get with a channel of 0x42.
And because the handlers take msg = &data[1], they echo neither the
channel nor the value ID, so readResponse's echoed-byte checks would
reject a correct answer. readPrefixed checks the command byte alone.

Telling the two firmwares apart is a discriminator rather than a guess,
because stock QMK marks a command its switch does not know. Its
raw_hid_receive ends with `default: { *command_id = id_unhandled; }`, so
0xFE comes back as 0xFF. Vial's default case does not set that marker —
it defers to the keyboard — and its keyboard-ID handler fills bytes 1
to 4 with a version, which is never zero. A keyboard echoing the request
also reads as not Vial, and one implementing raw_hid_receive_kb for
0xFE would defeat the probe, which is the one case the source cannot
rule out.

The list is a run of u16 from byte 2, paged from a cursor, and never
reports ID 0 because get_supported returns what is strictly greater than
a cursor starting at 0 — "off" is implied rather than listed. The reader
stops at the firmware's 0xFF padding and at 0, since a 0 in that list
cannot be an effect.

Written from that source and tested against a scripted keyboard. It has
not been run against Vial firmware, which is what makes it reviewable
and not what makes it right, so nothing is wired to it yet.

internal/via/vial.go

@@ -0,0 +1,200 @@
+package via
+
+import (
+	"encoding/binary"
+	"errors"
+	"fmt"
+)
+
+// Vial's protocol. This is a QMK fork that keeps VIA's command set unchanged and
+// hangs its own commands off a prefix byte, and it replaces the rgb_matrix
+// lighting handler with one of its own. The numbers below are read from
+// vial-kb/vial-qmk, branch `vial`:
+//
+//	quantum/via.h      enum via_command_id  -> id_vial_prefix
+//	quantum/vial.h     enum                -> vial_get_keyboard_id, VIAL_PROTOCOL_VERSION
+//	quantum/vialrgb.h  enum                -> vialrgb_get_info, vialrgb_get_supported
+//
+// None of this has been run against Vial firmware. It is written from that
+// source and tested against a scripted keyboard, which is what makes it
+// reviewable rather than what makes it right.
+const (
+	// vialPrefix is Vial's command prefix. VIA's own commands are 0x01 to 0x13 and
+	// untouched; this is where Vial's begin.
+	vialPrefix = 0xFE
+
+	// vialGetKeyboardID is the one command worth sending to find out whether a
+	// keyboard speaks Vial at all. It answers with the protocol version and
+	// nothing else, so it cannot be mistaken for state.
+	vialGetKeyboardID = 0x00
+)
+
+// VialRGB's commands, from quantum/vialrgb.h. They are not behind vialPrefix:
+// VialRGB answers the ordinary VIA lighting command 0x08 and carries its own
+// command in the channel byte, so a request for the supported effect IDs is a
+// custom get with a channel of 0x42. That is in vialrgb_get_value, which reads
+// `uint8_t cmd = data[1]` and switches on it.
+const (
+	lightingGet         = 0x08
+	vialrgbGetInfo      = 0x40
+	vialrgbGetSupported = 0x42
+)
+
+// vialrgbMaxPayload is how many bytes of a report the supported-ID list may
+// fill. The report is 32 bytes, the command byte and Vial's command byte take
+// two, and the list starts in the third.
+const vialrgbMaxPayload = 32 - 2
+
+// VialVersion asks whether the keyboard speaks Vial and, if it does, for its
+// protocol version.
+//
+// The answer is a discriminator rather than a guess because the two firmwares
+// differ in what they do with a command they do not know. Stock QMK's
+// raw_hid_receive ends its switch with
+//
+//	default: { *command_id = id_unhandled; }
+//
+// so an unknown command comes back as 0xFF. Vial's default case does not set that
+// marker — it defers to the keyboard instead — and its keyboard-ID handler fills
+// bytes 1 to 4 with the version. So 0xFF in byte 0 is a keyboard that is not
+// Vial, and a version that is not zero is one that is. A keyboard implementing
+// raw_hid_receive_kb for 0xFE would defeat this, which is the one case the
+// source cannot rule out.
+func (p *Protocol) VialVersion() (uint32, bool, error) {
+	report := make([]byte, 32)
+	report[0] = vialPrefix
+	report[1] = vialGetKeyboardID
+
+	if _, err := p.handle.SendReport(0x00, report); err != nil {
+		return 0, false, err
+	}
+	buf, err := p.readPrefixed(vialPrefix)
+	if err != nil {
+		if errors.Is(err, errUnhandled) {
+			// Stock QMK marking a command it does not know is the ordinary
+			// answer for a keyboard that is not Vial, not a failure.
+			return 0, false, nil
+		}
+		return 0, false, err
+	}
+	// Vial's handlers take msg = &data[1], so the version lands in bytes 1 to 4.
+	version := binary.LittleEndian.Uint32(buf[1:5])
+	if version == 0 {
+		// Byte 0 came back as the prefix and the rest as nothing, which is what a
+		// keyboard that echoes the request looks like. A Vial version is never 0.
+		return 0, false, nil
+	}
+	return version, true, nil
+}
+
+// VialEffectIDs returns the effect IDs the keyboard's rgb_matrix channel has
+// compiled in, as Vial numbers.
+//
+// It does not report effect 0. get_supported fills the report with the IDs
+// strictly greater than the cursor, and the first request asks from 0, so "off"
+// is implied rather than listed. A caller that wants to set 0 writes 0; nothing
+// here tells it that 0 exists.
+//
+// It is worth asking for where it answers, for two reasons. It does not write to
+// the keyboard, where reading the range by clamping does. And it is the firmware
+// reporting what it has, where the clamp is the firmware's answer to a question
+// it was not asked.
+//
+// Three things bound the answer. The list is a run of u16 with 0xFF padding, and
+// 0xFFFF is padding rather than an ID. The IDs ascend, and a list that does not
+// is not one this firmware produced. And the whole list is fetched in pages,
+// because get_supported fills one report at a time from a cursor.
+func (p *Protocol) VialEffectIDs() ([]uint16, error) {
+	var ids []uint16
+	cursor := uint16(0)
+	// Each page must advance the cursor or the loop would never end. One page per
+	// ten IDs is generous for a firmware whose largest list is under fifty, and a
+	// keyboard that needs more is not one this can make sense of.
+	for page := 0; page < 32; page++ {
+		batch, err := p.vialSupportedPage(cursor)
+		if err != nil {
+			return nil, err
+		}
+		if len(batch) == 0 {
+			break
+		}
+		ids = append(ids, batch...)
+		if batch[len(batch)-1] <= cursor {
+			return nil, fmt.Errorf("vialrgb effect list did not advance past %d", cursor)
+		}
+		cursor = batch[len(batch)-1]
+	}
+	return ascending(ids), nil
+}
+
+// vialSupportedPage asks for the effect IDs above a cursor and reads the run of
+// u16 the firmware wrote into the payload.
+func (p *Protocol) vialSupportedPage(cursor uint16) ([]uint16, error) {
+	report := make([]byte, 32)
+	report[0] = lightingGet
+	report[1] = vialrgbGetSupported
+	binary.LittleEndian.PutUint16(report[2:4], cursor)
+
+	if _, err := p.handle.SendReport(0x00, report); err != nil {
+		return nil, err
+	}
+	// The list starts at byte 2, where the request's cursor was: get_supported
+	// overwrites the cursor with the first ID it found, so those bytes are data
+	// and not an echo of what was sent.
+	buf, err := p.readPrefixed(lightingGet)
+	if err != nil {
+		return nil, err
+	}
+	payload := buf[2 : 2+vialrgbMaxPayload]
+
+	var out []uint16
+	for i := 0; i+1 < len(payload); i += 2 {
+		id := binary.LittleEndian.Uint16(payload[i : i+2])
+		if id == 0xFFFF || id == 0 {
+			// get_supported pads the rest of the report with 0xFF, so 0xFFFF is
+			// padding and not an effect. Zero stops it as well: the list holds the
+			// IDs strictly greater than a cursor that starts at 0, so a 0 in it is
+			// never an effect. That is what keeps a report that is zero-filled
+			// rather than 0xFF-filled from running on through fifteen zeros.
+			break
+		}
+		out = append(out, id)
+	}
+	return out, nil
+}
+
+// ascending reports whether the IDs came out in order. A list that did not is not
+// this firmware's, and returning it would put a name on a number the firmware
+// never said was there.
+func ascending(ids []uint16) []uint16 {
+	for i := 1; i < len(ids); i++ {
+		if ids[i] <= ids[i-1] {
+			return nil
+		}
+	}
+	return ids
+}
+
+// readPrefixed reads the report from a command that does not answer in VIA's
+// shape. readResponse checks the echoed channel and value ID, and Vial's handlers
+// fill their payload from byte 1 or byte 2 without echoing either — one of them
+// carries its own command in the channel byte — so those checks would reject a
+// correct answer. The command byte is the one thing both firmwares leave in
+// place, and 0xFF is how either says it does not handle the command.
+func (p *Protocol) readPrefixed(command byte) ([]byte, error) {
+	buf := make([]byte, 32)
+	n, err := p.handle.Read(buf)
+	if err != nil {
+		return nil, fmt.Errorf("read response: %w", err)
+	}
+	if n != len(buf) {
+		return nil, fmt.Errorf("short response: got %d bytes, want 32", n)
+	}
+	if buf[0] == byte(Unhandled) {
+		return nil, errUnhandled
+	}
+	if buf[0] != command {
+		return nil, fmt.Errorf("unexpected response command: 0x%02x, want 0x%02x", buf[0], command)
+	}
+	return buf, nil
+}

internal/via/vial_test.go

@@ -0,0 +1,222 @@
+package via
+
+import (
+	"encoding/binary"
+	"strings"
+	"testing"
+)
+
+// emptyVialPage is what get_supported returns once the list runs out: the whole
+// report is 0xFF padding and no ID is greater than the cursor. A list ends
+// because the firmware says so, not because the host ran out of answers.
+func emptyVialPage() []byte {
+	buf := make([]byte, 32)
+	buf[0] = lightingGet
+	buf[1] = vialrgbGetSupported
+	for i := 2; i < len(buf); i++ {
+		buf[i] = 0xFF
+	}
+	return buf
+}
+
+// vialVersionResponse is a Vial keyboard-ID answer: the prefix stays in byte 0
+// and the version fills bytes 1 to 4, because Vial's handlers take msg = &data[1].
+func vialVersionResponse(version uint32) []byte {
+	buf := make([]byte, 32)
+	buf[0] = vialPrefix
+	binary.LittleEndian.PutUint32(buf[1:5], version)
+	return buf
+}
+
+// unhandledResponse is stock QMK's answer to a command its switch does not know:
+// raw_hid_receive ends with `default: { *command_id = id_unhandled; }`.
+func unhandledResponse() []byte {
+	buf := make([]byte, 32)
+	buf[0] = byte(Unhandled)
+	return buf
+}
+
+// echoResponse is a keyboard that hands the request straight back. Nothing in
+// Vial's or QMK's source rules it out, and it is the answer that would make a
+// prefix probe believe anything, so it has to read as "not Vial".
+func echoResponse() []byte {
+	buf := make([]byte, 32)
+	buf[0] = vialPrefix
+	buf[1] = vialGetKeyboardID
+	return buf
+}
+
+func TestVialVersionReadsTheVersionOffAVialKeyboard(t *testing.T) {
+	transport := &fakeTransport{queue: [][]byte{vialVersionResponse(0x00000006)}}
+	protocol := Protocol{handle: transport}
+
+	version, isVial, err := protocol.VialVersion()
+	if err != nil {
+		t.Fatalf("VialVersion() error = %v", err)
+	}
+	if !isVial {
+		t.Error("VialVersion() = not Vial, want a keyboard that answered with a version")
+	}
+	if version != 6 {
+		t.Errorf("VialVersion() = %d, want 6", version)
+	}
+	// The probe has to be the prefixed command, and the rest of the report zeroed,
+	// so that an echo cannot look like an answer.
+	sent := transport.reports[0]
+	if sent[0] != vialPrefix || sent[1] != vialGetKeyboardID {
+		t.Errorf("request = %v, want the Vial prefix and the keyboard-ID command", sent[:2])
+	}
+	for i := 2; i < len(sent); i++ {
+		if sent[i] != 0 {
+			t.Errorf("request byte %d = %d, want 0 so that an echo cannot pass for a version", i, sent[i])
+		}
+	}
+}
+
+// Stock QMK marks a command it does not know, and that is a keyboard that is not
+// Vial rather than a failure to talk to it.
+func TestVialVersionTreatsAnUnhandledCommandAsNotVial(t *testing.T) {
+	transport := &fakeTransport{queue: [][]byte{unhandledResponse()}}
+	protocol := Protocol{handle: transport}
+
+	version, isVial, err := protocol.VialVersion()
+	if err != nil {
+		t.Fatalf("VialVersion() error = %v, want the unhandled marker read as an answer", err)
+	}
+	if isVial || version != 0 {
+		t.Errorf("VialVersion() = %d, %t, want 0, false", version, isVial)
+	}
+}
+
+// A keyboard that echoes the request answers with a prefix and a zero. A version
+// is never zero, so this must not read as Vial.
+func TestVialVersionDoesNotTrustAnEcho(t *testing.T) {
+	transport := &fakeTransport{queue: [][]byte{echoResponse()}}
+	protocol := Protocol{handle: transport}
+
+	_, isVial, err := protocol.VialVersion()
+	if err != nil {
+		t.Fatalf("VialVersion() error = %v", err)
+	}
+	if isVial {
+		t.Error("VialVersion() = Vial on an echoed request, which carries no version")
+	}
+}
+
+// The supported list is a run of u16 from byte 2, padded to the end of the
+// report with 0xFF. Reading the padding as an ID would put a name on a number
+// the firmware never said was there.
+func TestVialEffectIDsStopsAtTheFirmwarePadding(t *testing.T) {
+	page := make([]byte, 32)
+	page[0] = lightingGet
+	page[1] = vialrgbGetSupported
+	for i, id := range []uint16{1, 4, 5} {
+		binary.LittleEndian.PutUint16(page[2+i*2:], id)
+	}
+	// get_supported memsets the rest of the report to 0xFF, which is the padding
+	// the reader has to stop at.
+	for i := 8; i < len(page); i++ {
+		page[i] = 0xFF
+	}
+
+	transport := &fakeTransport{queue: [][]byte{page, emptyVialPage()}}
+	protocol := Protocol{handle: transport}
+
+	ids, err := protocol.VialEffectIDs()
+	if err != nil {
+		t.Fatalf("VialEffectIDs() error = %v", err)
+	}
+	want := []uint16{1, 4, 5}
+	if len(ids) != len(want) {
+		t.Fatalf("VialEffectIDs() = %v, want %v", ids, want)
+	}
+	for i := range want {
+		if ids[i] != want[i] {
+			t.Errorf("VialEffectIDs()[%d] = %d, want %d", i, ids[i], want[i])
+		}
+	}
+}
+
+// get_supported fills one report at a time from a cursor the request carries, so
+// a list longer than one page takes more than one round trip.
+func TestVialEffectIDsFollowsTheCursor(t *testing.T) {
+	first := make([]byte, 32)
+	first[0] = lightingGet
+	first[1] = vialrgbGetSupported
+	binary.LittleEndian.PutUint16(first[2:], 40)
+
+	second := make([]byte, 32)
+	second[0] = lightingGet
+	second[1] = vialrgbGetSupported
+	binary.LittleEndian.PutUint16(second[2:], 45)
+	for i := 4; i < len(second); i++ {
+		second[i] = 0xFF
+	}
+
+	transport := &fakeTransport{queue: [][]byte{first, second, emptyVialPage()}}
+	protocol := Protocol{handle: transport}
+
+	ids, err := protocol.VialEffectIDs()
+	if err != nil {
+		t.Fatalf("VialEffectIDs() error = %v", err)
+	}
+	if len(ids) != 2 || ids[0] != 40 || ids[1] != 45 {
+		t.Errorf("VialEffectIDs() = %v, want [40 45]", ids)
+	}
+	// The second request has to carry what the first page ended on, or the
+	// firmware would send the same page again.
+	cursor := binary.LittleEndian.Uint16(transport.reports[1][2:4])
+	if cursor != 40 {
+		t.Errorf("second request cursor = %d, want 40", cursor)
+	}
+}
+
+// Stock QMK's rgb_matrix get has no default case, so a value ID it does not know
+// leaves the report unmarked rather than answering 0xFF. That is why the effect
+// IDs must not be asked for before the firmware is known to be Vial: on a stock
+// board the answer is an echo, and an echo of this request is bytes that can be
+// read as a list.
+func TestVialEffectIDsRefusesAListThatDoesNotAscend(t *testing.T) {
+	page := make([]byte, 32)
+	page[0] = lightingGet
+	page[1] = vialrgbGetSupported
+	binary.LittleEndian.PutUint16(page[2:], 7)
+	binary.LittleEndian.PutUint16(page[4:], 3)
+
+	transport := &fakeTransport{queue: [][]byte{page, emptyVialPage()}}
+	protocol := Protocol{handle: transport}
+
+	ids, err := protocol.VialEffectIDs()
+	if err != nil {
+		t.Fatalf("VialEffectIDs() error = %v, want a list read and then rejected", err)
+	}
+	if ids != nil {
+		t.Errorf("VialEffectIDs() = %v, want nothing from a list that does not ascend", ids)
+	}
+}
+
+// A firmware that echoes the request answers the second page with the cursor it
+// was given, and a cursor that cannot advance would loop forever. That is the case
+// the guard is for: an echoing keyboard is not a Vial keyboard, and the loop has
+// to end rather than keep asking.
+func TestVialEffectIDsStopsWhenTheFirmwareEchoesTheCursor(t *testing.T) {
+	page := make([]byte, 32)
+	page[0] = lightingGet
+	page[1] = vialrgbGetSupported
+	binary.LittleEndian.PutUint16(page[2:], 1)
+	binary.LittleEndian.PutUint16(page[4:], 4)
+	for i := 6; i < len(page); i++ {
+		page[i] = 0xFF
+	}
+
+	transport := &fakeTransport{queue: [][]byte{page}}
+	protocol := Protocol{handle: transport}
+
+	_, err := protocol.VialEffectIDs()
+	if err == nil {
+		t.Fatal("VialEffectIDs() = nil error, want a failure when the cursor cannot advance")
+	}
+	if !strings.Contains(err.Error(), "advance") {
+		t.Errorf("error = %q, want it to say the list did not advance", err)
+	}
+}