From 67f9a7d0a032f4f273e2da66a342e5ed10e15661 Mon Sep 17 00:00:00 2001 From: DatCaptainHorse Date: Tue, 21 Oct 2025 18:41:45 +0300 Subject: [PATCH 1/6] Restructure protobufs and use them everywhere --- packages/input/package.json | 30 +- packages/input/src/controller.ts | 473 +++---- packages/input/src/keyboard.ts | 101 +- packages/input/src/messages.ts | 305 ----- packages/input/src/mouse.ts | 176 +-- packages/input/src/proto/messages_pb.ts | 142 ++- packages/input/src/proto/types_pb.ts | 434 ++++--- packages/input/src/streamwrapper.ts | 81 ++ packages/input/src/utils.ts | 95 ++ packages/input/src/webrtc-stream.ts | 116 +- .../play-standalone/src/pages/[room].astro | 2 +- packages/relay/go.mod | 2 +- packages/relay/go.sum | 4 +- packages/relay/internal/common/safebufio.go | 168 +-- .../relay/internal/connections/datachannel.go | 12 +- .../relay/internal/connections/messages.go | 94 -- packages/relay/internal/core/core.go | 13 + .../relay/internal/core/protocol_stream.go | 1100 +++++++++-------- packages/relay/internal/core/state.go | 48 +- packages/relay/internal/proto/messages.pb.go | 473 ++++++- packages/relay/internal/proto/types.pb.go | 840 ++++++------- packages/relay/internal/shared/participant.go | 48 +- packages/relay/internal/shared/room.go | 113 +- packages/server/Cargo.lock | 1 + packages/server/Cargo.toml | 1 + packages/server/src/input/controller.rs | 295 +++-- packages/server/src/main.rs | 16 +- packages/server/src/messages.rs | 50 - packages/server/src/nestrisink/imp.rs | 372 +++--- packages/server/src/nestrisink/mod.rs | 4 + .../server/src/p2p/p2p_protocol_stream.rs | 55 +- packages/server/src/p2p/p2p_safestream.rs | 50 +- packages/server/src/proto.rs | 34 + packages/server/src/proto/gen.rs | 202 --- packages/server/src/proto/proto.rs | 280 +++-- protobufs/messages.proto | 30 +- protobufs/types.proto | 141 ++- 37 files changed, 3391 insertions(+), 3010 deletions(-) delete mode 100644 packages/input/src/messages.ts create mode 100644 packages/input/src/streamwrapper.ts create mode 100644 packages/input/src/utils.ts delete mode 100644 packages/relay/internal/connections/messages.go delete mode 100644 packages/server/src/messages.rs delete mode 100644 packages/server/src/proto/gen.rs diff --git a/packages/input/package.json b/packages/input/package.json index 24dc4b74..2c4559a4 100644 --- a/packages/input/package.json +++ b/packages/input/package.json @@ -7,24 +7,22 @@ ".": "./src/index.ts" }, "devDependencies": { - "@bufbuild/buf": "^1.57.2", - "@bufbuild/protoc-gen-es": "^2.9.0" + "@bufbuild/buf": "^1.59.0", + "@bufbuild/protoc-gen-es": "^2.10.0" }, "dependencies": { - "@bufbuild/protobuf": "^2.9.0", - "@chainsafe/libp2p-noise": "^16.1.4", + "@bufbuild/protobuf": "^2.10.0", + "@chainsafe/libp2p-noise": "^17.0.0", "@chainsafe/libp2p-quic": "^1.1.3", - "@chainsafe/libp2p-yamux": "^7.0.4", - "@libp2p/identify": "^3.0.39", - "@libp2p/interface": "^2.11.0", - "@libp2p/ping": "^2.0.37", - "@libp2p/websockets": "^9.2.19", - "@libp2p/webtransport": "^5.0.51", - "@multiformats/multiaddr": "^12.5.1", - "it-length-prefixed": "^10.0.1", - "it-pipe": "^3.0.1", - "libp2p": "^2.10.0", - "uint8arraylist": "^2.4.8", - "uint8arrays": "^5.1.0" + "@chainsafe/libp2p-yamux": "^8.0.1", + "@libp2p/identify": "^4.0.5", + "@libp2p/interface": "^3.0.2", + "@libp2p/ping": "^3.0.5", + "@libp2p/websockets": "^10.0.6", + "@libp2p/webtransport": "^6.0.7", + "@libp2p/utils": "^7.0.5", + "@multiformats/multiaddr": "^13.0.1", + "libp2p": "^3.0.6", + "uint8arraylist": "^2.4.8" } } \ No newline at end of file diff --git a/packages/input/src/controller.ts b/packages/input/src/controller.ts index 8a618498..d1c5316d 100644 --- a/packages/input/src/controller.ts +++ b/packages/input/src/controller.ts @@ -1,12 +1,6 @@ import { controllerButtonToLinuxEventCode } from "./codes"; import { WebRTCStream } from "./webrtc-stream"; import { - ProtoMessageBase, - ProtoMessageInput, - ProtoMessageInputSchema, -} from "./proto/messages_pb"; -import { - ProtoInputSchema, ProtoControllerAttachSchema, ProtoControllerDetachSchema, ProtoControllerButtonSchema, @@ -16,6 +10,8 @@ import { ProtoControllerRumble, } from "./proto/types_pb"; import { create, toBinary, fromBinary } from "@bufbuild/protobuf"; +import { createMessage } from "./utils"; +import { ProtoMessageSchema } from "./proto/messages_pb"; interface Props { webrtc: WebRTCStream; @@ -36,7 +32,7 @@ interface GamepadState { export class Controller { protected wrtc: WebRTCStream; - protected slot: number; + protected slotMap: Map = new Map(); // local slot to server slot protected connected: boolean = false; protected gamepad: Gamepad | null = null; protected lastState: GamepadState = { @@ -54,17 +50,13 @@ export class Controller { protected stickDeadzone: number = 2048; // 2048 / 32768 = ~0.06 (6% of stick range) private updateInterval = 10.0; // 100 updates per second - private _dcRumbleHandler: ((data: ArrayBuffer) => void) | null = null; + private _dcHandler: ((data: ArrayBuffer) => void) | null = null; constructor({ webrtc, e }: Props) { this.wrtc = webrtc; - this.slot = e.gamepad.index; this.updateInterval = 1000 / webrtc.currentFrameRate; - // Gamepad connected - this.gamepad = e.gamepad; - // Get vendor of gamepad from id string (i.e. "... Vendor: 054c Product: 09cc") const vendorMatch = e.gamepad.id.match(/Vendor:\s?([0-9a-fA-F]{4})/); const vendorId = vendorMatch ? vendorMatch[1].toLowerCase() : "unknown"; @@ -72,30 +64,40 @@ export class Controller { const productMatch = e.gamepad.id.match(/Product:\s?([0-9a-fA-F]{4})/); const productId = productMatch ? productMatch[1].toLowerCase() : "unknown"; - const attachMsg = create(ProtoInputSchema, { - $typeName: "proto.ProtoInput", - inputType: { - case: "controllerAttach", - value: create(ProtoControllerAttachSchema, { - type: "ControllerAttach", - id: this.vendor_id_to_controller(vendorId, productId), - slot: this.slot, - }), - }, - }); - const message: ProtoMessageInput = { - $typeName: "proto.ProtoMessageInput", - messageBase: { - $typeName: "proto.ProtoMessageBase", - payloadType: "controllerInput", - } as ProtoMessageBase, - data: attachMsg, + // Listen to datachannel events from server + this._dcHandler = (data: ArrayBuffer) => { + if (!this.connected) return; + try { + // First decode the wrapper message + const uint8Data = new Uint8Array(data); + const messageWrapper = fromBinary(ProtoMessageSchema, uint8Data); + + if (messageWrapper.payload.case === "controllerRumble") { + this.rumbleCallback(messageWrapper.payload.value); + } else if (messageWrapper.payload.case === "controllerAttach") { + if (this.gamepad) return; // already attached + const attachMsg = messageWrapper.payload.value; + // Gamepad connected succesfully + this.gamepad = e.gamepad; + this.slotMap.set(e.gamepad.index, attachMsg.slot); + console.log( + `Gamepad connected: ${e.gamepad.id} assigned to slot ${attachMsg.slot} on server, local slot ${e.gamepad.index}`, + ); + } + } catch (err) { + console.error("Error decoding datachannel message:", err); + } }; - this.wrtc.sendBinary(toBinary(ProtoMessageInputSchema, message)); + this.wrtc.addDataChannelCallback(this._dcHandler); - // Listen to feedback rumble events from server - this._dcRumbleHandler = (data: any) => this.rumbleCallback(data as ArrayBuffer); - this.wrtc.addDataChannelCallback(this._dcRumbleHandler); + const attachMsg = createMessage( + create(ProtoControllerAttachSchema, { + id: this.vendor_id_to_controller(vendorId, productId), + sessionId: this.wrtc.getSessionID(), + }), + "controllerInput", + ); + this.wrtc.sendBinary(toBinary(ProtoMessageSchema, attachMsg)); this.run(); } @@ -150,12 +152,13 @@ export class Controller { } private pollGamepad() { + // Get updated gamepad state const gamepads = navigator.getGamepads(); - if (this.slot < gamepads.length) { - const gamepad = gamepads[this.slot]; - if (gamepad) { + if (this.gamepad) { + if (gamepads[this.gamepad.index]) { + this.gamepad = gamepads[this.gamepad!.index]; /* Button handling */ - gamepad.buttons.forEach((button, index) => { + this.gamepad.buttons.forEach((button, index) => { // Ignore d-pad buttons (12-15) as we handle those as axis if (index >= 12 && index <= 15) return; // ignore trigger buttons (6-7) as we handle those as axis @@ -169,29 +172,15 @@ export class Controller { return; } - const buttonProto = create(ProtoInputSchema, { - $typeName: "proto.ProtoInput", - inputType: { - case: "controllerButton", - value: create(ProtoControllerButtonSchema, { - type: "ControllerButton", - slot: this.slot, - button: linuxCode, - pressed: button.pressed, - }), - }, - }); - const buttonMessage: ProtoMessageInput = { - $typeName: "proto.ProtoMessageInput", - messageBase: { - $typeName: "proto.ProtoMessageBase", - payloadType: "controllerInput", - } as ProtoMessageBase, - data: buttonProto, - }; - this.wrtc.sendBinary( - toBinary(ProtoMessageInputSchema, buttonMessage), + const buttonMessage = createMessage( + create(ProtoControllerButtonSchema, { + slot: this.getServerSlot(), + button: linuxCode, + pressed: button.pressed, + }), + "controllerInput", ); + this.wrtc.sendBinary(toBinary(ProtoMessageSchema, buttonMessage)); // Store button state this.lastState.buttonState.set(index, button.pressed); } @@ -200,128 +189,100 @@ export class Controller { /* Trigger handling */ // map trigger value from 0.0 to 1.0 to -32768 to 32767 const leftTrigger = Math.round( - this.remapFromTo(gamepad.buttons[6]?.value ?? 0, 0, 1, -32768, 32767), + this.remapFromTo( + this.gamepad.buttons[6]?.value ?? 0, + 0, + 1, + -32768, + 32767, + ), ); // If state differs, send if (leftTrigger !== this.lastState.leftTrigger) { - const triggerProto = create(ProtoInputSchema, { - $typeName: "proto.ProtoInput", - inputType: { - case: "controllerTrigger", - value: create(ProtoControllerTriggerSchema, { - type: "ControllerTrigger", - slot: this.slot, - trigger: 0, // 0 = left, 1 = right - value: leftTrigger, - }), - }, - }); - const triggerMessage: ProtoMessageInput = { - $typeName: "proto.ProtoMessageInput", - messageBase: { - $typeName: "proto.ProtoMessageBase", - payloadType: "controllerInput", - } as ProtoMessageBase, - data: triggerProto, - }; - this.lastState.leftTrigger = leftTrigger; - this.wrtc.sendBinary( - toBinary(ProtoMessageInputSchema, triggerMessage), + const triggerMessage = createMessage( + create(ProtoControllerTriggerSchema, { + slot: this.getServerSlot(), + trigger: 0, // 0 = left, 1 = right + value: leftTrigger, + }), + "controllerInput", ); + this.wrtc.sendBinary(toBinary(ProtoMessageSchema, triggerMessage)); + this.lastState.leftTrigger = leftTrigger; } const rightTrigger = Math.round( - this.remapFromTo(gamepad.buttons[7]?.value ?? 0, 0, 1, -32768, 32767), + this.remapFromTo( + this.gamepad.buttons[7]?.value ?? 0, + 0, + 1, + -32768, + 32767, + ), ); // If state differs, send if (rightTrigger !== this.lastState.rightTrigger) { - const triggerProto = create(ProtoInputSchema, { - $typeName: "proto.ProtoInput", - inputType: { - case: "controllerTrigger", - value: create(ProtoControllerTriggerSchema, { - type: "ControllerTrigger", - slot: this.slot, - trigger: 1, // 0 = left, 1 = right - value: rightTrigger, - }), - }, - }); - const triggerMessage: ProtoMessageInput = { - $typeName: "proto.ProtoMessageInput", - messageBase: { - $typeName: "proto.ProtoMessageBase", - payloadType: "controllerInput", - } as ProtoMessageBase, - data: triggerProto, - }; - this.lastState.rightTrigger = rightTrigger; - this.wrtc.sendBinary( - toBinary(ProtoMessageInputSchema, triggerMessage), + const triggerMessage = createMessage( + create(ProtoControllerTriggerSchema, { + slot: this.getServerSlot(), + trigger: 1, // 0 = left, 1 = right + value: rightTrigger, + }), + "controllerInput", ); + this.wrtc.sendBinary(toBinary(ProtoMessageSchema, triggerMessage)); + this.lastState.rightTrigger = rightTrigger; } /* DPad handling */ // We send dpad buttons as axis values -1 to 1 for left/up, right/down - const dpadLeft = gamepad.buttons[14]?.pressed ? 1 : 0; - const dpadRight = gamepad.buttons[15]?.pressed ? 1 : 0; + const dpadLeft = this.gamepad.buttons[14]?.pressed ? 1 : 0; + const dpadRight = this.gamepad.buttons[15]?.pressed ? 1 : 0; const dpadX = dpadLeft ? -1 : dpadRight ? 1 : 0; if (dpadX !== this.lastState.dpadX) { - const dpadProto = create(ProtoInputSchema, { - $typeName: "proto.ProtoInput", - inputType: { - case: "controllerAxis", - value: create(ProtoControllerAxisSchema, { - type: "ControllerAxis", - slot: this.slot, - axis: 0, // 0 = dpadX, 1 = dpadY - value: dpadX, - }), - }, - }); - const dpadMessage: ProtoMessageInput = { - $typeName: "proto.ProtoMessageInput", - messageBase: { - $typeName: "proto.ProtoMessageBase", - payloadType: "controllerInput", - } as ProtoMessageBase, - data: dpadProto, - }; + const dpadMessage = createMessage( + create(ProtoControllerAxisSchema, { + slot: this.getServerSlot(), + axis: 0, // 0 = dpadX, 1 = dpadY + value: dpadX, + }), + "controllerInput", + ); this.lastState.dpadX = dpadX; - this.wrtc.sendBinary(toBinary(ProtoMessageInputSchema, dpadMessage)); + this.wrtc.sendBinary(toBinary(ProtoMessageSchema, dpadMessage)); } - const dpadUp = gamepad.buttons[12]?.pressed ? 1 : 0; - const dpadDown = gamepad.buttons[13]?.pressed ? 1 : 0; + const dpadUp = this.gamepad.buttons[12]?.pressed ? 1 : 0; + const dpadDown = this.gamepad.buttons[13]?.pressed ? 1 : 0; const dpadY = dpadUp ? -1 : dpadDown ? 1 : 0; if (dpadY !== this.lastState.dpadY) { - const dpadProto = create(ProtoInputSchema, { - $typeName: "proto.ProtoInput", - inputType: { - case: "controllerAxis", - value: create(ProtoControllerAxisSchema, { - type: "ControllerAxis", - slot: this.slot, - axis: 1, // 0 = dpadX, 1 = dpadY - value: dpadY, - }), - }, - }); - const dpadMessage: ProtoMessageInput = { - $typeName: "proto.ProtoMessageInput", - messageBase: { - $typeName: "proto.ProtoMessageBase", - payloadType: "controllerInput", - } as ProtoMessageBase, - data: dpadProto, - }; + const dpadMessage = createMessage( + create(ProtoControllerAxisSchema, { + slot: this.getServerSlot(), + axis: 1, // 0 = dpadX, 1 = dpadY + value: dpadY, + }), + "controllerInput", + ); + this.wrtc.sendBinary(toBinary(ProtoMessageSchema, dpadMessage)); this.lastState.dpadY = dpadY; - this.wrtc.sendBinary(toBinary(ProtoMessageInputSchema, dpadMessage)); } /* Stick handling */ // stick values need to be mapped from -1.0 to 1.0 to -32768 to 32767 - const leftX = this.remapFromTo(gamepad.axes[0] ?? 0, -1, 1, -32768, 32767); - const leftY = this.remapFromTo(gamepad.axes[1] ?? 0, -1, 1, -32768, 32767); + const leftX = this.remapFromTo( + this.gamepad.axes[0] ?? 0, + -1, + 1, + -32768, + 32767, + ); + const leftY = this.remapFromTo( + this.gamepad.axes[1] ?? 0, + -1, + 1, + -32768, + 32767, + ); // Apply deadzone const sendLeftX = Math.abs(leftX) > this.stickDeadzone ? Math.round(leftX) : 0; @@ -333,35 +294,33 @@ export class Controller { sendLeftX !== this.lastState.leftX || sendLeftY !== this.lastState.leftY ) { - // console.log("Sticks: ", sendLeftX, sendLeftY, sendRightX, sendRightY); - const stickProto = create(ProtoInputSchema, { - $typeName: "proto.ProtoInput", - inputType: { - case: "controllerStick", - value: create(ProtoControllerStickSchema, { - type: "ControllerStick", - slot: this.slot, - stick: 0, // 0 = left, 1 = right - x: sendLeftX, - y: sendLeftY, - }), - }, - }); - const stickMessage: ProtoMessageInput = { - $typeName: "proto.ProtoMessageInput", - messageBase: { - $typeName: "proto.ProtoMessageBase", - payloadType: "controllerInput", - } as ProtoMessageBase, - data: stickProto, - }; + const stickMessage = createMessage( + create(ProtoControllerStickSchema, { + stick: 0, // 0 = left, 1 = right + x: sendLeftX, + y: sendLeftY, + }), + "controllerInput", + ); + this.wrtc.sendBinary(toBinary(ProtoMessageSchema, stickMessage)); this.lastState.leftX = sendLeftX; this.lastState.leftY = sendLeftY; - this.wrtc.sendBinary(toBinary(ProtoMessageInputSchema, stickMessage)); } - const rightX = this.remapFromTo(gamepad.axes[2] ?? 0, -1, 1, -32768, 32767); - const rightY = this.remapFromTo(gamepad.axes[3] ?? 0, -1, 1, -32768, 32767); + const rightX = this.remapFromTo( + this.gamepad.axes[2] ?? 0, + -1, + 1, + -32768, + 32767, + ); + const rightY = this.remapFromTo( + this.gamepad.axes[3] ?? 0, + -1, + 1, + -32768, + 32767, + ); // Apply deadzone const sendRightX = Math.abs(rightX) > this.stickDeadzone ? Math.round(rightX) : 0; @@ -371,30 +330,17 @@ export class Controller { sendRightX !== this.lastState.rightX || sendRightY !== this.lastState.rightY ) { - const stickProto = create(ProtoInputSchema, { - $typeName: "proto.ProtoInput", - inputType: { - case: "controllerStick", - value: create(ProtoControllerStickSchema, { - type: "ControllerStick", - slot: this.slot, - stick: 1, // 0 = left, 1 = right - x: sendRightX, - y: sendRightY, - }), - }, - }); - const stickMessage: ProtoMessageInput = { - $typeName: "proto.ProtoMessageInput", - messageBase: { - $typeName: "proto.ProtoMessageBase", - payloadType: "controllerInput", - } as ProtoMessageBase, - data: stickProto, - }; + const stickMessage = createMessage( + create(ProtoControllerStickSchema, { + stick: 1, // 0 = left, 1 = right + x: sendRightX, + y: sendRightY, + }), + "controllerInput", + ); + this.wrtc.sendBinary(toBinary(ProtoMessageSchema, stickMessage)); this.lastState.rightX = sendRightX; this.lastState.rightY = sendRightY; - this.wrtc.sendBinary(toBinary(ProtoMessageInputSchema, stickMessage)); } } } @@ -403,8 +349,7 @@ export class Controller { private loopInterval: any = null; public run() { - if (this.connected) - this.stop(); + if (this.connected) this.stop(); this.connected = true; // Poll gamepads in setInterval loop @@ -421,89 +366,75 @@ export class Controller { this.connected = false; } - public getSlot() { - return this.slot; + public getLocalSlot(): number { + if (this.gamepad) { + return this.gamepad.index; + } + return -1; + } + + public getServerSlot(): number { + if (this.gamepad) { + const slot = this.slotMap.get(this.gamepad.index); + if (slot !== undefined) return slot; + } + return -1; } public dispose() { this.stop(); // Remove callback - if (this._dcRumbleHandler !== null) { - this.wrtc.removeDataChannelCallback(this._dcRumbleHandler); - this._dcRumbleHandler = null; + if (this._dcHandler !== null) { + this.wrtc.removeDataChannelCallback(this._dcHandler); + this._dcHandler = null; } // Gamepad disconnected - const detachMsg = create(ProtoInputSchema, { - $typeName: "proto.ProtoInput", - inputType: { - case: "controllerDetach", - value: create(ProtoControllerDetachSchema, { - type: "ControllerDetach", - slot: this.slot, - }), - }, - }); - const message: ProtoMessageInput = { - $typeName: "proto.ProtoMessageInput", - messageBase: { - $typeName: "proto.ProtoMessageBase", - payloadType: "controllerInput", - } as ProtoMessageBase, - data: detachMsg, - }; - this.wrtc.sendBinary(toBinary(ProtoMessageInputSchema, message)); + const detachMsg = createMessage( + create(ProtoControllerDetachSchema, { + slot: this.getServerSlot(), + }), + "controllerInput", + ); + this.wrtc.sendBinary(toBinary(ProtoMessageSchema, detachMsg)); } private controllerButtonToVirtualKeyCode(code: number) { return controllerButtonToLinuxEventCode[code] || undefined; } - private rumbleCallback(data: ArrayBuffer) { + private rumbleCallback(rumbleMsg: ProtoControllerRumble) { // If not connected, ignore if (!this.connected) return; - try { - // First decode the wrapper message - const uint8Data = new Uint8Array(data); - const messageWrapper = fromBinary(ProtoMessageInputSchema, uint8Data); - - // Check if it contains controller rumble data - if (messageWrapper.data?.inputType?.case === "controllerRumble") { - const rumbleMsg = messageWrapper.data.inputType.value as ProtoControllerRumble; - // Check if aimed at this controller slot - if (rumbleMsg.slot !== this.slot) return; + // Check if aimed at this controller slot + if (rumbleMsg.slot !== this.getServerSlot()) return; - // Trigger actual rumble - // Need to remap from 0-65535 to 0.0-1.0 ranges - const clampedLowFreq = Math.max(0, Math.min(65535, rumbleMsg.lowFrequency)); - const rumbleLowFreq = this.remapFromTo( - clampedLowFreq, - 0, - 65535, - 0.0, - 1.0, - ); - const clampedHighFreq = Math.max(0, Math.min(65535, rumbleMsg.highFrequency)); - const rumbleHighFreq = this.remapFromTo( - clampedHighFreq, - 0, - 65535, - 0.0, - 1.0, - ); - // Cap to valid range (max 5000) - const rumbleDuration = Math.max(0, Math.min(5000, rumbleMsg.duration)); - if (this.gamepad.vibrationActuator) { - this.gamepad.vibrationActuator.playEffect("dual-rumble", { - startDelay: 0, - duration: rumbleDuration, - weakMagnitude: rumbleLowFreq, - strongMagnitude: rumbleHighFreq, - }).catch(console.error); - } - } - } catch (error) { - console.error("Failed to decode rumble message:", error); + // Trigger actual rumble + // Need to remap from 0-65535 to 0.0-1.0 ranges + const clampedLowFreq = Math.max(0, Math.min(65535, rumbleMsg.lowFrequency)); + const rumbleLowFreq = this.remapFromTo(clampedLowFreq, 0, 65535, 0.0, 1.0); + const clampedHighFreq = Math.max( + 0, + Math.min(65535, rumbleMsg.highFrequency), + ); + const rumbleHighFreq = this.remapFromTo( + clampedHighFreq, + 0, + 65535, + 0.0, + 1.0, + ); + // Cap to valid range (max 5000) + const rumbleDuration = Math.max(0, Math.min(5000, rumbleMsg.duration)); + if (this.gamepad.vibrationActuator) { + this.gamepad.vibrationActuator + .playEffect("dual-rumble", { + startDelay: 0, + duration: rumbleDuration, + weakMagnitude: rumbleLowFreq, + strongMagnitude: rumbleHighFreq, + }) + .catch(console.error); } } } diff --git a/packages/input/src/keyboard.ts b/packages/input/src/keyboard.ts index d51a290f..15918ea2 100644 --- a/packages/input/src/keyboard.ts +++ b/packages/input/src/keyboard.ts @@ -1,16 +1,9 @@ -import {keyCodeToLinuxEventCode} from "./codes" -import {WebRTCStream} from "./webrtc-stream"; -import {LatencyTracker} from "./latency"; -import {ProtoLatencyTracker, ProtoTimestampEntry} from "./proto/latency_tracker_pb"; -import {timestampFromDate} from "@bufbuild/protobuf/wkt"; -import {ProtoMessageBase, ProtoMessageInput, ProtoMessageInputSchema} from "./proto/messages_pb"; -import { - ProtoInput, - ProtoInputSchema, - ProtoKeyDownSchema, - ProtoKeyUpSchema, -} from "./proto/types_pb"; -import {create, toBinary} from "@bufbuild/protobuf"; +import { keyCodeToLinuxEventCode } from "./codes"; +import { WebRTCStream } from "./webrtc-stream"; +import { ProtoKeyDownSchema, ProtoKeyUpSchema } from "./proto/types_pb"; +import { create, toBinary } from "@bufbuild/protobuf"; +import { createMessage } from "./utils"; +import { ProtoMessageSchema } from "./proto/messages_pb"; interface Props { webrtc: WebRTCStream; @@ -24,38 +17,29 @@ export class Keyboard { private readonly keydownListener: (e: KeyboardEvent) => void; private readonly keyupListener: (e: KeyboardEvent) => void; - constructor({webrtc}: Props) { + constructor({ webrtc }: Props) { this.wrtc = webrtc; - this.keydownListener = this.createKeyboardListener((e: any) => create(ProtoInputSchema, { - $typeName: "proto.ProtoInput", - inputType: { - case: "keyDown", - value: create(ProtoKeyDownSchema, { - type: "KeyDown", - key: this.keyToVirtualKeyCode(e.code) - }), - } - })); - this.keyupListener = this.createKeyboardListener((e: any) => create(ProtoInputSchema, { - $typeName: "proto.ProtoInput", - inputType: { - case: "keyUp", - value: create(ProtoKeyUpSchema, { - type: "KeyUp", - key: this.keyToVirtualKeyCode(e.code) - }), - } - })); - this.run() + this.keydownListener = this.createKeyboardListener((e: any) => + create(ProtoKeyDownSchema, { + key: this.keyToVirtualKeyCode(e.code), + }), + ); + this.keyupListener = this.createKeyboardListener((e: any) => + create(ProtoKeyUpSchema, { + key: this.keyToVirtualKeyCode(e.code), + }), + ); + this.run(); } private run() { - if (this.connected) - this.stop() + if (this.connected) this.stop(); - this.connected = true - document.addEventListener("keydown", this.keydownListener, {passive: false}); - document.addEventListener("keyup", this.keyupListener, {passive: false}); + this.connected = true; + document.addEventListener("keydown", this.keydownListener, { + passive: false, + }); + document.addEventListener("keyup", this.keyupListener, { passive: false }); } private stop() { @@ -65,42 +49,19 @@ export class Keyboard { } // Helper function to create and return mouse listeners - private createKeyboardListener(dataCreator: (e: Event) => ProtoInput): (e: Event) => void { + private createKeyboardListener( + dataCreator: (e: Event) => any, + ): (e: Event) => void { return (e: Event) => { e.preventDefault(); e.stopPropagation(); // Prevent repeated key events from being sent (important for games) - if ((e as any).repeat) - return; + if ((e as any).repeat) return; const data = dataCreator(e as any); - // Latency tracking - const tracker = new LatencyTracker("input-keyboard"); - tracker.addTimestamp("client_send"); - const protoTracker: ProtoLatencyTracker = { - $typeName: "proto.ProtoLatencyTracker", - sequenceId: tracker.sequence_id, - timestamps: [], - }; - for (const t of tracker.timestamps) { - protoTracker.timestamps.push({ - $typeName: "proto.ProtoTimestampEntry", - stage: t.stage, - time: timestampFromDate(t.time), - } as ProtoTimestampEntry); - } - - const message: ProtoMessageInput = { - $typeName: "proto.ProtoMessageInput", - messageBase: { - $typeName: "proto.ProtoMessageBase", - payloadType: "input", - latency: protoTracker, - } as ProtoMessageBase, - data: data, - }; - this.wrtc.sendBinary(toBinary(ProtoMessageInputSchema, message)); + const message = createMessage(data, "input"); + this.wrtc.sendBinary(toBinary(ProtoMessageSchema, message)); }; } @@ -114,4 +75,4 @@ export class Keyboard { if (code === "Home") return 1; return keyCodeToLinuxEventCode[code] || undefined; } -} \ No newline at end of file +} diff --git a/packages/input/src/messages.ts b/packages/input/src/messages.ts deleted file mode 100644 index d5031787..00000000 --- a/packages/input/src/messages.ts +++ /dev/null @@ -1,305 +0,0 @@ -import { LatencyTracker } from "./latency"; -import { Uint8ArrayList } from "uint8arraylist"; -import { allocUnsafe } from "uint8arrays/alloc"; -import { pipe } from "it-pipe"; -import { decode, encode } from "it-length-prefixed"; -import { Stream } from "@libp2p/interface"; - -export interface MessageBase { - payload_type: string; - latency?: LatencyTracker; -} - -export interface MessageRaw extends MessageBase { - data: any; -} - -export function NewMessageRaw(type: string, data: any): Uint8Array { - const msg = { - payload_type: type, - data: data, - }; - return new TextEncoder().encode(JSON.stringify(msg)); -} - -export interface MessageICE extends MessageBase { - candidate: RTCIceCandidateInit; -} - -export function NewMessageICE( - type: string, - candidate: RTCIceCandidateInit, -): Uint8Array { - const msg = { - payload_type: type, - candidate: candidate, - }; - return new TextEncoder().encode(JSON.stringify(msg)); -} - -export interface MessageSDP extends MessageBase { - sdp: RTCSessionDescriptionInit; -} - -export function NewMessageSDP( - type: string, - sdp: RTCSessionDescriptionInit, -): Uint8Array { - const msg = { - payload_type: type, - sdp: sdp, - }; - return new TextEncoder().encode(JSON.stringify(msg)); -} - -const MAX_SIZE = 1024 * 1024; // 1MB -const MAX_QUEUE_SIZE = 1000; // Maximum number of messages in the queue - -// Custom 4-byte length encoder -export const length4ByteEncoder = (length: number) => { - const buf = allocUnsafe(4); - - // Write the length as a 32-bit unsigned integer (4 bytes) - buf[0] = length >>> 24; - buf[1] = (length >>> 16) & 0xff; - buf[2] = (length >>> 8) & 0xff; - buf[3] = length & 0xff; - - // Set the bytes property to 4 - length4ByteEncoder.bytes = 4; - - return buf; -}; -length4ByteEncoder.bytes = 4; - -// Custom 4-byte length decoder -export const length4ByteDecoder = (data: Uint8ArrayList) => { - if (data.byteLength < 4) { - // Not enough bytes to read the length - return -1; - } - - // Read the length from the first 4 bytes - let length = 0; - length = - (data.subarray(0, 1)[0] >>> 0) * 0x1000000 + - (data.subarray(1, 2)[0] >>> 0) * 0x10000 + - (data.subarray(2, 3)[0] >>> 0) * 0x100 + - (data.subarray(3, 4)[0] >>> 0); - - // Set bytes read to 4 - length4ByteDecoder.bytes = 4; - - return length; -}; -length4ByteDecoder.bytes = 4; - -interface PromiseMessage { - data: Uint8Array; - resolve: () => void; - reject: (error: Error) => void; -} - -export class SafeStream { - private stream: Stream; - private callbacks: Map void)[]> = new Map(); - private isReading: boolean = false; - private isWriting: boolean = false; - private closed: boolean = false; - private messageQueue: PromiseMessage[] = []; - private writeLock = false; - private readRetries = 0; - private writeRetries = 0; - private readonly MAX_RETRIES = 5; - - constructor(stream: Stream) { - this.stream = stream; - this.startReading(); - this.startWriting(); - } - - private async startReading(): Promise { - if (this.isReading || this.closed) return; - - this.isReading = true; - - try { - const source = this.stream.source; - const decodedSource = decode(source, { - maxDataLength: MAX_SIZE, - lengthDecoder: length4ByteDecoder, - }); - - for await (const chunk of decodedSource) { - if (this.closed) break; - - this.readRetries = 0; - - try { - const data = chunk.slice(); - const message = JSON.parse( - new TextDecoder().decode(data), - ) as MessageBase; - const msgType = message.payload_type; - - if (this.callbacks.has(msgType)) { - const handlers = this.callbacks.get(msgType)!; - for (const handler of handlers) { - try { - handler(message); - } catch (err) { - console.error(`Error in message handler for ${msgType}:`, err); - } - } - } - } catch (err) { - console.error("Error processing message:", err); - } - } - } catch (err) { - console.error("Stream reading error:", err); - } finally { - this.isReading = false; - this.readRetries++; - - // If not closed, try to restart reading - if (!this.closed && this.readRetries < this.MAX_RETRIES) - setTimeout(() => this.startReading(), 100); - else if (this.readRetries >= this.MAX_RETRIES) - console.error( - "Max retries reached for reading stream, stopping attempts", - ); - } - } - - public registerCallback( - msgType: string, - callback: (data: any) => void, - ): void { - if (!this.callbacks.has(msgType)) { - this.callbacks.set(msgType, []); - } - - this.callbacks.get(msgType)!.push(callback); - } - - public removeCallback(msgType: string, callback: (data: any) => void): void { - if (this.callbacks.has(msgType)) { - const callbacks = this.callbacks.get(msgType)!; - const index = callbacks.indexOf(callback); - - if (index !== -1) { - callbacks.splice(index, 1); - } - - if (callbacks.length === 0) { - this.callbacks.delete(msgType); - } - } - } - - private async startWriting(): Promise { - if (this.isWriting || this.closed) return; - - this.isWriting = true; - - try { - // Create an async generator for real-time message processing - const messageSource = async function* (this: SafeStream) { - while (!this.closed) { - // Check if we have messages to send - if (this.messageQueue.length > 0) { - this.writeLock = true; - - try { - const message = this.messageQueue[0]; - - // Encode the message - const encoded = encode([message.data], { - maxDataLength: MAX_SIZE, - lengthEncoder: length4ByteEncoder, - }); - - for await (const chunk of encoded) { - yield chunk; - } - - // Remove message after successful sending - this.writeRetries = 0; - const sentMessage = this.messageQueue.shift(); - if (sentMessage) - sentMessage.resolve(); - } catch (err) { - console.error("Error encoding or sending message:", err); - const failedMessage = this.messageQueue.shift(); - if (failedMessage) - failedMessage.reject(new Error(`Failed to send message: ${err}`)); - } finally { - this.writeLock = false; - } - } else { - // No messages to send, wait for a short period - await new Promise((resolve) => setTimeout(resolve, 100)); - } - } - }.bind(this); - - await pipe(messageSource(), this.stream.sink).catch((err) => { - console.error("Sink error:", err); - this.isWriting = false; - this.writeRetries++; - - // Try to restart if not closed - if (!this.closed && this.writeRetries < this.MAX_RETRIES) { - setTimeout(() => this.startWriting(), 1000); - } else if (this.writeRetries >= this.MAX_RETRIES) { - console.error("Max retries reached for writing to stream sink, stopping attempts"); - } - }); - } catch (err) { - console.error("Stream writing error:", err); - this.isWriting = false; - this.writeRetries++; - - // Try to restart if not closed - if (!this.closed && this.writeRetries < this.MAX_RETRIES) { - setTimeout(() => this.startWriting(), 1000); - } else if (this.writeRetries >= this.MAX_RETRIES) { - console.error("Max retries reached for writing stream, stopping attempts"); - } - } - } - - public async writeMessage(message: Uint8Array): Promise { - if (this.closed) { - throw new Error("Cannot write to closed stream"); - } - - // Validate message size before queuing - if (message.length > MAX_SIZE) { - throw new Error("Message size exceeds maximum size limit"); - } - - // Check if the message queue is too large - if (this.messageQueue.length >= MAX_QUEUE_SIZE) { - throw new Error("Message queue is full, cannot write message"); - } - - // Create a promise to resolve when the message is sent - return new Promise((resolve, reject) => { - this.messageQueue.push({ data: message, resolve, reject } as PromiseMessage); - }); - } - - public close(): void { - this.closed = true; - this.callbacks.clear(); - // Reject pending messages - for (const msg of this.messageQueue) - msg.reject(new Error("Stream closed")); - - this.messageQueue = []; - this.readRetries = 0; - this.writeRetries = 0; - } -} diff --git a/packages/input/src/mouse.ts b/packages/input/src/mouse.ts index 34b1a1bf..a709c53c 100644 --- a/packages/input/src/mouse.ts +++ b/packages/input/src/mouse.ts @@ -1,18 +1,14 @@ -import {WebRTCStream} from "./webrtc-stream"; -import {LatencyTracker} from "./latency"; -import {ProtoMessageInput, ProtoMessageBase, ProtoMessageInputSchema} from "./proto/messages_pb"; +import { WebRTCStream } from "./webrtc-stream"; import { - ProtoInput, ProtoInputSchema, - ProtoMouseKeyDown, ProtoMouseKeyDownSchema, - ProtoMouseKeyUp, ProtoMouseKeyUpSchema, - ProtoMouseMove, + ProtoMouseKeyDownSchema, + ProtoMouseKeyUpSchema, ProtoMouseMoveSchema, - ProtoMouseWheel, ProtoMouseWheelSchema + ProtoMouseWheelSchema, } from "./proto/types_pb"; -import {mouseButtonToLinuxEventCode} from "./codes"; -import {ProtoLatencyTracker, ProtoTimestampEntry} from "./proto/latency_tracker_pb"; -import {create, toBinary} from "@bufbuild/protobuf"; -import {timestampFromDate} from "@bufbuild/protobuf/wkt"; +import { mouseButtonToLinuxEventCode } from "./codes"; +import { create, toBinary } from "@bufbuild/protobuf"; +import { createMessage } from "./utils"; +import { ProtoMessageSchema } from "./proto/messages_pb"; interface Props { webrtc: WebRTCStream; @@ -24,7 +20,7 @@ export class Mouse { protected canvas: HTMLCanvasElement; protected connected!: boolean; - private sendInterval = 10 // 100 updates per second + private sendInterval = 10; // 100 updates per second // Store references to event listeners private readonly mousemoveListener: (e: MouseEvent) => void; @@ -35,7 +31,7 @@ export class Mouse { private readonly mouseupListener: (e: MouseEvent) => void; private readonly mousewheelListener: (e: WheelEvent) => void; - constructor({webrtc, canvas}: Props) { + constructor({ webrtc, canvas }: Props) { this.wrtc = webrtc; this.canvas = canvas; @@ -48,65 +44,56 @@ export class Mouse { this.movementY += e.movementY; }; - this.mousedownListener = this.createMouseListener((e: any) => create(ProtoInputSchema, { - $typeName: "proto.ProtoInput", - inputType: { - case: "mouseKeyDown", - value: create(ProtoMouseKeyDownSchema, { - type: "MouseKeyDown", - key: this.keyToVirtualKeyCode(e.button) - }), - } - })); - this.mouseupListener = this.createMouseListener((e: any) => create(ProtoInputSchema, { - $typeName: "proto.ProtoInput", - inputType: { - case: "mouseKeyUp", - value: create(ProtoMouseKeyUpSchema, { - type: "MouseKeyUp", - key: this.keyToVirtualKeyCode(e.button) - }), - } - })); - this.mousewheelListener = this.createMouseListener((e: any) => create(ProtoInputSchema, { - $typeName: "proto.ProtoInput", - inputType: { - case: "mouseWheel", - value: create(ProtoMouseWheelSchema, { - type: "MouseWheel", - x: Math.round(e.deltaX), - y: Math.round(e.deltaY), - }), - } - })); - - this.run() + this.mousedownListener = this.createMouseListener((e: any) => + create(ProtoMouseKeyDownSchema, { + key: this.keyToVirtualKeyCode(e.button), + }), + ); + this.mouseupListener = this.createMouseListener((e: any) => + create(ProtoMouseKeyUpSchema, { + key: this.keyToVirtualKeyCode(e.button), + }), + ); + this.mousewheelListener = this.createMouseListener((e: any) => + create(ProtoMouseWheelSchema, { + x: Math.round(e.deltaX), + y: Math.round(e.deltaY), + }), + ); + + this.run(); this.startProcessing(); } private run() { //calls all the other functions if (!document.pointerLockElement) { - console.log("no pointerlock") + console.log("no pointerlock"); if (this.connected) { - this.stop() + this.stop(); } return; } if (document.pointerLockElement == this.canvas) { - this.connected = true - this.canvas.addEventListener("mousemove", this.mousemoveListener, {passive: false}); - this.canvas.addEventListener("mousedown", this.mousedownListener, {passive: false}); - this.canvas.addEventListener("mouseup", this.mouseupListener, {passive: false}); - this.canvas.addEventListener("wheel", this.mousewheelListener, {passive: false}); - + this.connected = true; + this.canvas.addEventListener("mousemove", this.mousemoveListener, { + passive: false, + }); + this.canvas.addEventListener("mousedown", this.mousedownListener, { + passive: false, + }); + this.canvas.addEventListener("mouseup", this.mouseupListener, { + passive: false, + }); + this.canvas.addEventListener("wheel", this.mousewheelListener, { + passive: false, + }); } else { if (this.connected) { - this.stop() + this.stop(); } } - } private stop() { @@ -128,79 +115,26 @@ export class Mouse { } private sendAggregatedMouseMove() { - const data = create(ProtoInputSchema, { - $typeName: "proto.ProtoInput", - inputType: { - case: "mouseMove", - value: create(ProtoMouseMoveSchema, { - type: "MouseMove", - x: Math.round(this.movementX), - y: Math.round(this.movementY), - }), - }, + const data = create(ProtoMouseMoveSchema, { + x: Math.round(this.movementX), + y: Math.round(this.movementY), }); - // Latency tracking - const tracker = new LatencyTracker("input-mouse"); - tracker.addTimestamp("client_send"); - const protoTracker: ProtoLatencyTracker = { - $typeName: "proto.ProtoLatencyTracker", - sequenceId: tracker.sequence_id, - timestamps: [], - }; - for (const t of tracker.timestamps) { - protoTracker.timestamps.push({ - $typeName: "proto.ProtoTimestampEntry", - stage: t.stage, - time: timestampFromDate(t.time), - } as ProtoTimestampEntry); - } - - const message: ProtoMessageInput = { - $typeName: "proto.ProtoMessageInput", - messageBase: { - $typeName: "proto.ProtoMessageBase", - payloadType: "input", - latency: protoTracker, - } as ProtoMessageBase, - data: data, - }; - this.wrtc.sendBinary(toBinary(ProtoMessageInputSchema, message)); + const message = createMessage(data, "input"); + this.wrtc.sendBinary(toBinary(ProtoMessageSchema, message)); } // Helper function to create and return mouse listeners - private createMouseListener(dataCreator: (e: Event) => ProtoInput): (e: Event) => void { + private createMouseListener( + dataCreator: (e: Event) => any, + ): (e: Event) => void { return (e: Event) => { e.preventDefault(); e.stopPropagation(); const data = dataCreator(e as any); - // Latency tracking - const tracker = new LatencyTracker("input-mouse"); - tracker.addTimestamp("client_send"); - const protoTracker: ProtoLatencyTracker = { - $typeName: "proto.ProtoLatencyTracker", - sequenceId: tracker.sequence_id, - timestamps: [], - }; - for (const t of tracker.timestamps) { - protoTracker.timestamps.push({ - $typeName: "proto.ProtoTimestampEntry", - stage: t.stage, - time: timestampFromDate(t.time), - } as ProtoTimestampEntry); - } - - const message: ProtoMessageInput = { - $typeName: "proto.ProtoMessageInput", - messageBase: { - $typeName: "proto.ProtoMessageBase", - payloadType: "input", - latency: protoTracker, - } as ProtoMessageBase, - data: data, - }; - this.wrtc.sendBinary(toBinary(ProtoMessageInputSchema, message)); + const message = createMessage(data, "input"); + this.wrtc.sendBinary(toBinary(ProtoMessageSchema, message)); }; } @@ -213,4 +147,4 @@ export class Mouse { private keyToVirtualKeyCode(code: number) { return mouseButtonToLinuxEventCode[code] || undefined; } -} \ No newline at end of file +} diff --git a/packages/input/src/proto/messages_pb.ts b/packages/input/src/proto/messages_pb.ts index 2e432370..8c33ee30 100644 --- a/packages/input/src/proto/messages_pb.ts +++ b/packages/input/src/proto/messages_pb.ts @@ -4,7 +4,7 @@ import type { GenFile, GenMessage } from "@bufbuild/protobuf/codegenv2"; import { fileDesc, messageDesc } from "@bufbuild/protobuf/codegenv2"; -import type { ProtoInput } from "./types_pb"; +import type { ProtoClientDisconnected, ProtoClientRequestRoomStream, ProtoControllerAttach, ProtoControllerAxis, ProtoControllerButton, ProtoControllerDetach, ProtoControllerRumble, ProtoControllerStick, ProtoControllerTrigger, ProtoICE, ProtoKeyDown, ProtoKeyUp, ProtoMouseKeyDown, ProtoMouseKeyUp, ProtoMouseMove, ProtoMouseMoveAbs, ProtoMouseWheel, ProtoRaw, ProtoSDP, ProtoServerPushStream } from "./types_pb"; import { file_types } from "./types_pb"; import type { ProtoLatencyTracker } from "./latency_tracker_pb"; import { file_latency_tracker } from "./latency_tracker_pb"; @@ -14,7 +14,7 @@ import type { Message } from "@bufbuild/protobuf"; * Describes the file messages.proto. */ export const file_messages: GenFile = /*@__PURE__*/ - fileDesc("Cg5tZXNzYWdlcy5wcm90bxIFcHJvdG8iVQoQUHJvdG9NZXNzYWdlQmFzZRIUCgxwYXlsb2FkX3R5cGUYASABKAkSKwoHbGF0ZW5jeRgCIAEoCzIaLnByb3RvLlByb3RvTGF0ZW5jeVRyYWNrZXIiYwoRUHJvdG9NZXNzYWdlSW5wdXQSLQoMbWVzc2FnZV9iYXNlGAEgASgLMhcucHJvdG8uUHJvdG9NZXNzYWdlQmFzZRIfCgRkYXRhGAIgASgLMhEucHJvdG8uUHJvdG9JbnB1dEIWWhRyZWxheS9pbnRlcm5hbC9wcm90b2IGcHJvdG8z", [file_types, file_latency_tracker]); + fileDesc("Cg5tZXNzYWdlcy5wcm90bxIFcHJvdG8iVQoQUHJvdG9NZXNzYWdlQmFzZRIUCgxwYXlsb2FkX3R5cGUYASABKAkSKwoHbGF0ZW5jeRgCIAEoCzIaLnByb3RvLlByb3RvTGF0ZW5jeVRyYWNrZXIiyQgKDFByb3RvTWVzc2FnZRItCgxtZXNzYWdlX2Jhc2UYASABKAsyFy5wcm90by5Qcm90b01lc3NhZ2VCYXNlEisKCm1vdXNlX21vdmUYAiABKAsyFS5wcm90by5Qcm90b01vdXNlTW92ZUgAEjIKDm1vdXNlX21vdmVfYWJzGAMgASgLMhgucHJvdG8uUHJvdG9Nb3VzZU1vdmVBYnNIABItCgttb3VzZV93aGVlbBgEIAEoCzIWLnByb3RvLlByb3RvTW91c2VXaGVlbEgAEjIKDm1vdXNlX2tleV9kb3duGAUgASgLMhgucHJvdG8uUHJvdG9Nb3VzZUtleURvd25IABIuCgxtb3VzZV9rZXlfdXAYBiABKAsyFi5wcm90by5Qcm90b01vdXNlS2V5VXBIABInCghrZXlfZG93bhgHIAEoCzITLnByb3RvLlByb3RvS2V5RG93bkgAEiMKBmtleV91cBgIIAEoCzIRLnByb3RvLlByb3RvS2V5VXBIABI5ChFjb250cm9sbGVyX2F0dGFjaBgJIAEoCzIcLnByb3RvLlByb3RvQ29udHJvbGxlckF0dGFjaEgAEjkKEWNvbnRyb2xsZXJfZGV0YWNoGAogASgLMhwucHJvdG8uUHJvdG9Db250cm9sbGVyRGV0YWNoSAASOQoRY29udHJvbGxlcl9idXR0b24YCyABKAsyHC5wcm90by5Qcm90b0NvbnRyb2xsZXJCdXR0b25IABI7ChJjb250cm9sbGVyX3RyaWdnZXIYDCABKAsyHS5wcm90by5Qcm90b0NvbnRyb2xsZXJUcmlnZ2VySAASNwoQY29udHJvbGxlcl9zdGljaxgNIAEoCzIbLnByb3RvLlByb3RvQ29udHJvbGxlclN0aWNrSAASNQoPY29udHJvbGxlcl9heGlzGA4gASgLMhoucHJvdG8uUHJvdG9Db250cm9sbGVyQXhpc0gAEjkKEWNvbnRyb2xsZXJfcnVtYmxlGA8gASgLMhwucHJvdG8uUHJvdG9Db250cm9sbGVyUnVtYmxlSAASHgoDaWNlGBQgASgLMg8ucHJvdG8uUHJvdG9JQ0VIABIeCgNzZHAYFSABKAsyDy5wcm90by5Qcm90b1NEUEgAEh4KA3JhdxgWIAEoCzIPLnByb3RvLlByb3RvUmF3SAASSQoaY2xpZW50X3JlcXVlc3Rfcm9vbV9zdHJlYW0YFyABKAsyIy5wcm90by5Qcm90b0NsaWVudFJlcXVlc3RSb29tU3RyZWFtSAASPQoTY2xpZW50X2Rpc2Nvbm5lY3RlZBgYIAEoCzIeLnByb3RvLlByb3RvQ2xpZW50RGlzY29ubmVjdGVkSAASOgoSc2VydmVyX3B1c2hfc3RyZWFtGBkgASgLMhwucHJvdG8uUHJvdG9TZXJ2ZXJQdXNoU3RyZWFtSABCCQoHcGF5bG9hZEIWWhRyZWxheS9pbnRlcm5hbC9wcm90b2IGcHJvdG8z", [file_types, file_latency_tracker]); /** * @generated from message proto.ProtoMessageBase @@ -39,24 +39,148 @@ export const ProtoMessageBaseSchema: GenMessage = /*@__PURE__* messageDesc(file_messages, 0); /** - * @generated from message proto.ProtoMessageInput + * @generated from message proto.ProtoMessage */ -export type ProtoMessageInput = Message<"proto.ProtoMessageInput"> & { +export type ProtoMessage = Message<"proto.ProtoMessage"> & { /** * @generated from field: proto.ProtoMessageBase message_base = 1; */ messageBase?: ProtoMessageBase; /** - * @generated from field: proto.ProtoInput data = 2; + * @generated from oneof proto.ProtoMessage.payload */ - data?: ProtoInput; + payload: { + /** + * Input types + * + * @generated from field: proto.ProtoMouseMove mouse_move = 2; + */ + value: ProtoMouseMove; + case: "mouseMove"; + } | { + /** + * @generated from field: proto.ProtoMouseMoveAbs mouse_move_abs = 3; + */ + value: ProtoMouseMoveAbs; + case: "mouseMoveAbs"; + } | { + /** + * @generated from field: proto.ProtoMouseWheel mouse_wheel = 4; + */ + value: ProtoMouseWheel; + case: "mouseWheel"; + } | { + /** + * @generated from field: proto.ProtoMouseKeyDown mouse_key_down = 5; + */ + value: ProtoMouseKeyDown; + case: "mouseKeyDown"; + } | { + /** + * @generated from field: proto.ProtoMouseKeyUp mouse_key_up = 6; + */ + value: ProtoMouseKeyUp; + case: "mouseKeyUp"; + } | { + /** + * @generated from field: proto.ProtoKeyDown key_down = 7; + */ + value: ProtoKeyDown; + case: "keyDown"; + } | { + /** + * @generated from field: proto.ProtoKeyUp key_up = 8; + */ + value: ProtoKeyUp; + case: "keyUp"; + } | { + /** + * @generated from field: proto.ProtoControllerAttach controller_attach = 9; + */ + value: ProtoControllerAttach; + case: "controllerAttach"; + } | { + /** + * @generated from field: proto.ProtoControllerDetach controller_detach = 10; + */ + value: ProtoControllerDetach; + case: "controllerDetach"; + } | { + /** + * @generated from field: proto.ProtoControllerButton controller_button = 11; + */ + value: ProtoControllerButton; + case: "controllerButton"; + } | { + /** + * @generated from field: proto.ProtoControllerTrigger controller_trigger = 12; + */ + value: ProtoControllerTrigger; + case: "controllerTrigger"; + } | { + /** + * @generated from field: proto.ProtoControllerStick controller_stick = 13; + */ + value: ProtoControllerStick; + case: "controllerStick"; + } | { + /** + * @generated from field: proto.ProtoControllerAxis controller_axis = 14; + */ + value: ProtoControllerAxis; + case: "controllerAxis"; + } | { + /** + * @generated from field: proto.ProtoControllerRumble controller_rumble = 15; + */ + value: ProtoControllerRumble; + case: "controllerRumble"; + } | { + /** + * Signaling types + * + * @generated from field: proto.ProtoICE ice = 20; + */ + value: ProtoICE; + case: "ice"; + } | { + /** + * @generated from field: proto.ProtoSDP sdp = 21; + */ + value: ProtoSDP; + case: "sdp"; + } | { + /** + * @generated from field: proto.ProtoRaw raw = 22; + */ + value: ProtoRaw; + case: "raw"; + } | { + /** + * @generated from field: proto.ProtoClientRequestRoomStream client_request_room_stream = 23; + */ + value: ProtoClientRequestRoomStream; + case: "clientRequestRoomStream"; + } | { + /** + * @generated from field: proto.ProtoClientDisconnected client_disconnected = 24; + */ + value: ProtoClientDisconnected; + case: "clientDisconnected"; + } | { + /** + * @generated from field: proto.ProtoServerPushStream server_push_stream = 25; + */ + value: ProtoServerPushStream; + case: "serverPushStream"; + } | { case: undefined; value?: undefined }; }; /** - * Describes the message proto.ProtoMessageInput. - * Use `create(ProtoMessageInputSchema)` to create a new message. + * Describes the message proto.ProtoMessage. + * Use `create(ProtoMessageSchema)` to create a new message. */ -export const ProtoMessageInputSchema: GenMessage = /*@__PURE__*/ +export const ProtoMessageSchema: GenMessage = /*@__PURE__*/ messageDesc(file_messages, 1); diff --git a/packages/input/src/proto/types_pb.ts b/packages/input/src/proto/types_pb.ts index a4647fe0..6538b8f6 100644 --- a/packages/input/src/proto/types_pb.ts +++ b/packages/input/src/proto/types_pb.ts @@ -10,7 +10,7 @@ import type { Message } from "@bufbuild/protobuf"; * Describes the file types.proto. */ export const file_types: GenFile = /*@__PURE__*/ - fileDesc("Cgt0eXBlcy5wcm90bxIFcHJvdG8iNAoOUHJvdG9Nb3VzZU1vdmUSDAoEdHlwZRgBIAEoCRIJCgF4GAIgASgFEgkKAXkYAyABKAUiNwoRUHJvdG9Nb3VzZU1vdmVBYnMSDAoEdHlwZRgBIAEoCRIJCgF4GAIgASgFEgkKAXkYAyABKAUiNQoPUHJvdG9Nb3VzZVdoZWVsEgwKBHR5cGUYASABKAkSCQoBeBgCIAEoBRIJCgF5GAMgASgFIi4KEVByb3RvTW91c2VLZXlEb3duEgwKBHR5cGUYASABKAkSCwoDa2V5GAIgASgFIiwKD1Byb3RvTW91c2VLZXlVcBIMCgR0eXBlGAEgASgJEgsKA2tleRgCIAEoBSIpCgxQcm90b0tleURvd24SDAoEdHlwZRgBIAEoCRILCgNrZXkYAiABKAUiJwoKUHJvdG9LZXlVcBIMCgR0eXBlGAEgASgJEgsKA2tleRgCIAEoBSI/ChVQcm90b0NvbnRyb2xsZXJBdHRhY2gSDAoEdHlwZRgBIAEoCRIKCgJpZBgCIAEoCRIMCgRzbG90GAMgASgFIjMKFVByb3RvQ29udHJvbGxlckRldGFjaBIMCgR0eXBlGAEgASgJEgwKBHNsb3QYAiABKAUiVAoVUHJvdG9Db250cm9sbGVyQnV0dG9uEgwKBHR5cGUYASABKAkSDAoEc2xvdBgCIAEoBRIOCgZidXR0b24YAyABKAUSDwoHcHJlc3NlZBgEIAEoCCJUChZQcm90b0NvbnRyb2xsZXJUcmlnZ2VyEgwKBHR5cGUYASABKAkSDAoEc2xvdBgCIAEoBRIPCgd0cmlnZ2VyGAMgASgFEg0KBXZhbHVlGAQgASgFIlcKFFByb3RvQ29udHJvbGxlclN0aWNrEgwKBHR5cGUYASABKAkSDAoEc2xvdBgCIAEoBRINCgVzdGljaxgDIAEoBRIJCgF4GAQgASgFEgkKAXkYBSABKAUiTgoTUHJvdG9Db250cm9sbGVyQXhpcxIMCgR0eXBlGAEgASgJEgwKBHNsb3QYAiABKAUSDAoEYXhpcxgDIAEoBRINCgV2YWx1ZRgEIAEoBSJ0ChVQcm90b0NvbnRyb2xsZXJSdW1ibGUSDAoEdHlwZRgBIAEoCRIMCgRzbG90GAIgASgFEhUKDWxvd19mcmVxdWVuY3kYAyABKAUSFgoOaGlnaF9mcmVxdWVuY3kYBCABKAUSEAoIZHVyYXRpb24YBSABKAUi9QUKClByb3RvSW5wdXQSKwoKbW91c2VfbW92ZRgBIAEoCzIVLnByb3RvLlByb3RvTW91c2VNb3ZlSAASMgoObW91c2VfbW92ZV9hYnMYAiABKAsyGC5wcm90by5Qcm90b01vdXNlTW92ZUFic0gAEi0KC21vdXNlX3doZWVsGAMgASgLMhYucHJvdG8uUHJvdG9Nb3VzZVdoZWVsSAASMgoObW91c2Vfa2V5X2Rvd24YBCABKAsyGC5wcm90by5Qcm90b01vdXNlS2V5RG93bkgAEi4KDG1vdXNlX2tleV91cBgFIAEoCzIWLnByb3RvLlByb3RvTW91c2VLZXlVcEgAEicKCGtleV9kb3duGAYgASgLMhMucHJvdG8uUHJvdG9LZXlEb3duSAASIwoGa2V5X3VwGAcgASgLMhEucHJvdG8uUHJvdG9LZXlVcEgAEjkKEWNvbnRyb2xsZXJfYXR0YWNoGAggASgLMhwucHJvdG8uUHJvdG9Db250cm9sbGVyQXR0YWNoSAASOQoRY29udHJvbGxlcl9kZXRhY2gYCSABKAsyHC5wcm90by5Qcm90b0NvbnRyb2xsZXJEZXRhY2hIABI5ChFjb250cm9sbGVyX2J1dHRvbhgKIAEoCzIcLnByb3RvLlByb3RvQ29udHJvbGxlckJ1dHRvbkgAEjsKEmNvbnRyb2xsZXJfdHJpZ2dlchgLIAEoCzIdLnByb3RvLlByb3RvQ29udHJvbGxlclRyaWdnZXJIABI3ChBjb250cm9sbGVyX3N0aWNrGAwgASgLMhsucHJvdG8uUHJvdG9Db250cm9sbGVyU3RpY2tIABI1Cg9jb250cm9sbGVyX2F4aXMYDSABKAsyGi5wcm90by5Qcm90b0NvbnRyb2xsZXJBeGlzSAASOQoRY29udHJvbGxlcl9ydW1ibGUYDiABKAsyHC5wcm90by5Qcm90b0NvbnRyb2xsZXJSdW1ibGVIAEIMCgppbnB1dF90eXBlQhZaFHJlbGF5L2ludGVybmFsL3Byb3RvYgZwcm90bzM"); + fileDesc("Cgt0eXBlcy5wcm90bxIFcHJvdG8iJgoOUHJvdG9Nb3VzZU1vdmUSCQoBeBgBIAEoBRIJCgF5GAIgASgFIikKEVByb3RvTW91c2VNb3ZlQWJzEgkKAXgYASABKAUSCQoBeRgCIAEoBSInCg9Qcm90b01vdXNlV2hlZWwSCQoBeBgBIAEoBRIJCgF5GAIgASgFIiAKEVByb3RvTW91c2VLZXlEb3duEgsKA2tleRgBIAEoBSIeCg9Qcm90b01vdXNlS2V5VXASCwoDa2V5GAEgASgFIhsKDFByb3RvS2V5RG93bhILCgNrZXkYASABKAUiGQoKUHJvdG9LZXlVcBILCgNrZXkYASABKAUiRQoVUHJvdG9Db250cm9sbGVyQXR0YWNoEgoKAmlkGAEgASgJEgwKBHNsb3QYAiABKAUSEgoKc2Vzc2lvbl9pZBgDIAEoCSIlChVQcm90b0NvbnRyb2xsZXJEZXRhY2gSDAoEc2xvdBgBIAEoBSJGChVQcm90b0NvbnRyb2xsZXJCdXR0b24SDAoEc2xvdBgBIAEoBRIOCgZidXR0b24YAiABKAUSDwoHcHJlc3NlZBgDIAEoCCJGChZQcm90b0NvbnRyb2xsZXJUcmlnZ2VyEgwKBHNsb3QYASABKAUSDwoHdHJpZ2dlchgCIAEoBRINCgV2YWx1ZRgDIAEoBSJJChRQcm90b0NvbnRyb2xsZXJTdGljaxIMCgRzbG90GAEgASgFEg0KBXN0aWNrGAIgASgFEgkKAXgYAyABKAUSCQoBeRgEIAEoBSJAChNQcm90b0NvbnRyb2xsZXJBeGlzEgwKBHNsb3QYASABKAUSDAoEYXhpcxgCIAEoBRINCgV2YWx1ZRgDIAEoBSJmChVQcm90b0NvbnRyb2xsZXJSdW1ibGUSDAoEc2xvdBgBIAEoBRIVCg1sb3dfZnJlcXVlbmN5GAIgASgFEhYKDmhpZ2hfZnJlcXVlbmN5GAMgASgFEhAKCGR1cmF0aW9uGAQgASgFIqoBChNSVENJY2VDYW5kaWRhdGVJbml0EhEKCWNhbmRpZGF0ZRgBIAEoCRIaCg1zZHBNTGluZUluZGV4GAIgASgNSACIAQESEwoGc2RwTWlkGAMgASgJSAGIAQESHQoQdXNlcm5hbWVGcmFnbWVudBgEIAEoCUgCiAEBQhAKDl9zZHBNTGluZUluZGV4QgkKB19zZHBNaWRCEwoRX3VzZXJuYW1lRnJhZ21lbnQiNgoZUlRDU2Vzc2lvbkRlc2NyaXB0aW9uSW5pdBILCgNzZHAYASABKAkSDAoEdHlwZRgCIAEoCSI5CghQcm90b0lDRRItCgljYW5kaWRhdGUYASABKAsyGi5wcm90by5SVENJY2VDYW5kaWRhdGVJbml0IjkKCFByb3RvU0RQEi0KA3NkcBgBIAEoCzIgLnByb3RvLlJUQ1Nlc3Npb25EZXNjcmlwdGlvbkluaXQiGAoIUHJvdG9SYXcSDAoEZGF0YRgBIAEoCSJFChxQcm90b0NsaWVudFJlcXVlc3RSb29tU3RyZWFtEhEKCXJvb21fbmFtZRgBIAEoCRISCgpzZXNzaW9uX2lkGAIgASgJIkcKF1Byb3RvQ2xpZW50RGlzY29ubmVjdGVkEhIKCnNlc3Npb25faWQYASABKAkSGAoQY29udHJvbGxlcl9zbG90cxgCIAMoBSIqChVQcm90b1NlcnZlclB1c2hTdHJlYW0SEQoJcm9vbV9uYW1lGAEgASgJQhZaFHJlbGF5L2ludGVybmFsL3Byb3RvYgZwcm90bzM"); /** * MouseMove message @@ -19,19 +19,12 @@ export const file_types: GenFile = /*@__PURE__*/ */ export type ProtoMouseMove = Message<"proto.ProtoMouseMove"> & { /** - * Fixed value "MouseMove" - * - * @generated from field: string type = 1; - */ - type: string; - - /** - * @generated from field: int32 x = 2; + * @generated from field: int32 x = 1; */ x: number; /** - * @generated from field: int32 y = 3; + * @generated from field: int32 y = 2; */ y: number; }; @@ -50,19 +43,12 @@ export const ProtoMouseMoveSchema: GenMessage = /*@__PURE__*/ */ export type ProtoMouseMoveAbs = Message<"proto.ProtoMouseMoveAbs"> & { /** - * Fixed value "MouseMoveAbs" - * - * @generated from field: string type = 1; - */ - type: string; - - /** - * @generated from field: int32 x = 2; + * @generated from field: int32 x = 1; */ x: number; /** - * @generated from field: int32 y = 3; + * @generated from field: int32 y = 2; */ y: number; }; @@ -81,19 +67,12 @@ export const ProtoMouseMoveAbsSchema: GenMessage = /*@__PURE_ */ export type ProtoMouseWheel = Message<"proto.ProtoMouseWheel"> & { /** - * Fixed value "MouseWheel" - * - * @generated from field: string type = 1; - */ - type: string; - - /** - * @generated from field: int32 x = 2; + * @generated from field: int32 x = 1; */ x: number; /** - * @generated from field: int32 y = 3; + * @generated from field: int32 y = 2; */ y: number; }; @@ -112,14 +91,7 @@ export const ProtoMouseWheelSchema: GenMessage = /*@__PURE__*/ */ export type ProtoMouseKeyDown = Message<"proto.ProtoMouseKeyDown"> & { /** - * Fixed value "MouseKeyDown" - * - * @generated from field: string type = 1; - */ - type: string; - - /** - * @generated from field: int32 key = 2; + * @generated from field: int32 key = 1; */ key: number; }; @@ -138,14 +110,7 @@ export const ProtoMouseKeyDownSchema: GenMessage = /*@__PURE_ */ export type ProtoMouseKeyUp = Message<"proto.ProtoMouseKeyUp"> & { /** - * Fixed value "MouseKeyUp" - * - * @generated from field: string type = 1; - */ - type: string; - - /** - * @generated from field: int32 key = 2; + * @generated from field: int32 key = 1; */ key: number; }; @@ -164,14 +129,7 @@ export const ProtoMouseKeyUpSchema: GenMessage = /*@__PURE__*/ */ export type ProtoKeyDown = Message<"proto.ProtoKeyDown"> & { /** - * Fixed value "KeyDown" - * - * @generated from field: string type = 1; - */ - type: string; - - /** - * @generated from field: int32 key = 2; + * @generated from field: int32 key = 1; */ key: number; }; @@ -190,14 +148,7 @@ export const ProtoKeyDownSchema: GenMessage = /*@__PURE__*/ */ export type ProtoKeyUp = Message<"proto.ProtoKeyUp"> & { /** - * Fixed value "KeyUp" - * - * @generated from field: string type = 1; - */ - type: string; - - /** - * @generated from field: int32 key = 2; + * @generated from field: int32 key = 1; */ key: number; }; @@ -215,26 +166,26 @@ export const ProtoKeyUpSchema: GenMessage = /*@__PURE__*/ * @generated from message proto.ProtoControllerAttach */ export type ProtoControllerAttach = Message<"proto.ProtoControllerAttach"> & { - /** - * Fixed value "ControllerAttach" - * - * @generated from field: string type = 1; - */ - type: string; - /** * One of the following enums: "ps", "xbox" or "switch" * - * @generated from field: string id = 2; + * @generated from field: string id = 1; */ id: string; /** * Slot number (0-3) * - * @generated from field: int32 slot = 3; + * @generated from field: int32 slot = 2; */ slot: number; + + /** + * Session ID of the client attaching the controller + * + * @generated from field: string session_id = 3; + */ + sessionId: string; }; /** @@ -250,17 +201,10 @@ export const ProtoControllerAttachSchema: GenMessage = /* * @generated from message proto.ProtoControllerDetach */ export type ProtoControllerDetach = Message<"proto.ProtoControllerDetach"> & { - /** - * Fixed value "ControllerDetach" - * - * @generated from field: string type = 1; - */ - type: string; - /** * Slot number (0-3) * - * @generated from field: int32 slot = 2; + * @generated from field: int32 slot = 1; */ slot: number; }; @@ -278,31 +222,24 @@ export const ProtoControllerDetachSchema: GenMessage = /* * @generated from message proto.ProtoControllerButton */ export type ProtoControllerButton = Message<"proto.ProtoControllerButton"> & { - /** - * Fixed value "ControllerButtons" - * - * @generated from field: string type = 1; - */ - type: string; - /** * Slot number (0-3) * - * @generated from field: int32 slot = 2; + * @generated from field: int32 slot = 1; */ slot: number; /** * Button code (linux input event code) * - * @generated from field: int32 button = 3; + * @generated from field: int32 button = 2; */ button: number; /** * true if pressed, false if released * - * @generated from field: bool pressed = 4; + * @generated from field: bool pressed = 3; */ pressed: boolean; }; @@ -320,31 +257,24 @@ export const ProtoControllerButtonSchema: GenMessage = /* * @generated from message proto.ProtoControllerTrigger */ export type ProtoControllerTrigger = Message<"proto.ProtoControllerTrigger"> & { - /** - * Fixed value "ControllerTriggers" - * - * @generated from field: string type = 1; - */ - type: string; - /** * Slot number (0-3) * - * @generated from field: int32 slot = 2; + * @generated from field: int32 slot = 1; */ slot: number; /** * Trigger number (0 for left, 1 for right) * - * @generated from field: int32 trigger = 3; + * @generated from field: int32 trigger = 2; */ trigger: number; /** * trigger value (-32768 to 32767) * - * @generated from field: int32 value = 4; + * @generated from field: int32 value = 3; */ value: number; }; @@ -362,38 +292,31 @@ export const ProtoControllerTriggerSchema: GenMessage = * @generated from message proto.ProtoControllerStick */ export type ProtoControllerStick = Message<"proto.ProtoControllerStick"> & { - /** - * Fixed value "ControllerStick" - * - * @generated from field: string type = 1; - */ - type: string; - /** * Slot number (0-3) * - * @generated from field: int32 slot = 2; + * @generated from field: int32 slot = 1; */ slot: number; /** * Stick number (0 for left, 1 for right) * - * @generated from field: int32 stick = 3; + * @generated from field: int32 stick = 2; */ stick: number; /** * X axis value (-32768 to 32767) * - * @generated from field: int32 x = 4; + * @generated from field: int32 x = 3; */ x: number; /** * Y axis value (-32768 to 32767) * - * @generated from field: int32 y = 5; + * @generated from field: int32 y = 4; */ y: number; }; @@ -411,31 +334,24 @@ export const ProtoControllerStickSchema: GenMessage = /*@_ * @generated from message proto.ProtoControllerAxis */ export type ProtoControllerAxis = Message<"proto.ProtoControllerAxis"> & { - /** - * Fixed value "ControllerAxis" - * - * @generated from field: string type = 1; - */ - type: string; - /** * Slot number (0-3) * - * @generated from field: int32 slot = 2; + * @generated from field: int32 slot = 1; */ slot: number; /** * Axis number (0 for d-pad horizontal, 1 for d-pad vertical) * - * @generated from field: int32 axis = 3; + * @generated from field: int32 axis = 2; */ axis: number; /** * axis value (-1 to 1) * - * @generated from field: int32 value = 4; + * @generated from field: int32 value = 3; */ value: number; }; @@ -453,38 +369,31 @@ export const ProtoControllerAxisSchema: GenMessage = /*@__P * @generated from message proto.ProtoControllerRumble */ export type ProtoControllerRumble = Message<"proto.ProtoControllerRumble"> & { - /** - * Fixed value "ControllerRumble" - * - * @generated from field: string type = 1; - */ - type: string; - /** * Slot number (0-3) * - * @generated from field: int32 slot = 2; + * @generated from field: int32 slot = 1; */ slot: number; /** * Low frequency rumble (0-65535) * - * @generated from field: int32 low_frequency = 3; + * @generated from field: int32 low_frequency = 2; */ lowFrequency: number; /** * High frequency rumble (0-65535) * - * @generated from field: int32 high_frequency = 4; + * @generated from field: int32 high_frequency = 3; */ highFrequency: number; /** * Duration in milliseconds * - * @generated from field: int32 duration = 5; + * @generated from field: int32 duration = 4; */ duration: number; }; @@ -497,105 +406,180 @@ export const ProtoControllerRumbleSchema: GenMessage = /* messageDesc(file_types, 13); /** - * Union of all Input types - * - * @generated from message proto.ProtoInput - */ -export type ProtoInput = Message<"proto.ProtoInput"> & { - /** - * @generated from oneof proto.ProtoInput.input_type - */ - inputType: { - /** - * @generated from field: proto.ProtoMouseMove mouse_move = 1; - */ - value: ProtoMouseMove; - case: "mouseMove"; - } | { - /** - * @generated from field: proto.ProtoMouseMoveAbs mouse_move_abs = 2; - */ - value: ProtoMouseMoveAbs; - case: "mouseMoveAbs"; - } | { - /** - * @generated from field: proto.ProtoMouseWheel mouse_wheel = 3; - */ - value: ProtoMouseWheel; - case: "mouseWheel"; - } | { - /** - * @generated from field: proto.ProtoMouseKeyDown mouse_key_down = 4; - */ - value: ProtoMouseKeyDown; - case: "mouseKeyDown"; - } | { - /** - * @generated from field: proto.ProtoMouseKeyUp mouse_key_up = 5; - */ - value: ProtoMouseKeyUp; - case: "mouseKeyUp"; - } | { - /** - * @generated from field: proto.ProtoKeyDown key_down = 6; - */ - value: ProtoKeyDown; - case: "keyDown"; - } | { - /** - * @generated from field: proto.ProtoKeyUp key_up = 7; - */ - value: ProtoKeyUp; - case: "keyUp"; - } | { - /** - * @generated from field: proto.ProtoControllerAttach controller_attach = 8; - */ - value: ProtoControllerAttach; - case: "controllerAttach"; - } | { - /** - * @generated from field: proto.ProtoControllerDetach controller_detach = 9; - */ - value: ProtoControllerDetach; - case: "controllerDetach"; - } | { - /** - * @generated from field: proto.ProtoControllerButton controller_button = 10; - */ - value: ProtoControllerButton; - case: "controllerButton"; - } | { - /** - * @generated from field: proto.ProtoControllerTrigger controller_trigger = 11; - */ - value: ProtoControllerTrigger; - case: "controllerTrigger"; - } | { - /** - * @generated from field: proto.ProtoControllerStick controller_stick = 12; - */ - value: ProtoControllerStick; - case: "controllerStick"; - } | { - /** - * @generated from field: proto.ProtoControllerAxis controller_axis = 13; - */ - value: ProtoControllerAxis; - case: "controllerAxis"; - } | { - /** - * @generated from field: proto.ProtoControllerRumble controller_rumble = 14; - */ - value: ProtoControllerRumble; - case: "controllerRumble"; - } | { case: undefined; value?: undefined }; + * @generated from message proto.RTCIceCandidateInit + */ +export type RTCIceCandidateInit = Message<"proto.RTCIceCandidateInit"> & { + /** + * @generated from field: string candidate = 1; + */ + candidate: string; + + /** + * @generated from field: optional uint32 sdpMLineIndex = 2; + */ + sdpMLineIndex?: number; + + /** + * @generated from field: optional string sdpMid = 3; + */ + sdpMid?: string; + + /** + * @generated from field: optional string usernameFragment = 4; + */ + usernameFragment?: string; }; /** - * Describes the message proto.ProtoInput. - * Use `create(ProtoInputSchema)` to create a new message. + * Describes the message proto.RTCIceCandidateInit. + * Use `create(RTCIceCandidateInitSchema)` to create a new message. */ -export const ProtoInputSchema: GenMessage = /*@__PURE__*/ +export const RTCIceCandidateInitSchema: GenMessage = /*@__PURE__*/ messageDesc(file_types, 14); +/** + * @generated from message proto.RTCSessionDescriptionInit + */ +export type RTCSessionDescriptionInit = Message<"proto.RTCSessionDescriptionInit"> & { + /** + * @generated from field: string sdp = 1; + */ + sdp: string; + + /** + * @generated from field: string type = 2; + */ + type: string; +}; + +/** + * Describes the message proto.RTCSessionDescriptionInit. + * Use `create(RTCSessionDescriptionInitSchema)` to create a new message. + */ +export const RTCSessionDescriptionInitSchema: GenMessage = /*@__PURE__*/ + messageDesc(file_types, 15); + +/** + * ProtoICE message + * + * @generated from message proto.ProtoICE + */ +export type ProtoICE = Message<"proto.ProtoICE"> & { + /** + * @generated from field: proto.RTCIceCandidateInit candidate = 1; + */ + candidate?: RTCIceCandidateInit; +}; + +/** + * Describes the message proto.ProtoICE. + * Use `create(ProtoICESchema)` to create a new message. + */ +export const ProtoICESchema: GenMessage = /*@__PURE__*/ + messageDesc(file_types, 16); + +/** + * ProtoSDP message + * + * @generated from message proto.ProtoSDP + */ +export type ProtoSDP = Message<"proto.ProtoSDP"> & { + /** + * @generated from field: proto.RTCSessionDescriptionInit sdp = 1; + */ + sdp?: RTCSessionDescriptionInit; +}; + +/** + * Describes the message proto.ProtoSDP. + * Use `create(ProtoSDPSchema)` to create a new message. + */ +export const ProtoSDPSchema: GenMessage = /*@__PURE__*/ + messageDesc(file_types, 17); + +/** + * ProtoRaw message + * + * @generated from message proto.ProtoRaw + */ +export type ProtoRaw = Message<"proto.ProtoRaw"> & { + /** + * @generated from field: string data = 1; + */ + data: string; +}; + +/** + * Describes the message proto.ProtoRaw. + * Use `create(ProtoRawSchema)` to create a new message. + */ +export const ProtoRawSchema: GenMessage = /*@__PURE__*/ + messageDesc(file_types, 18); + +/** + * ProtoClientRequestRoomStream message + * + * @generated from message proto.ProtoClientRequestRoomStream + */ +export type ProtoClientRequestRoomStream = Message<"proto.ProtoClientRequestRoomStream"> & { + /** + * @generated from field: string room_name = 1; + */ + roomName: string; + + /** + * @generated from field: string session_id = 2; + */ + sessionId: string; +}; + +/** + * Describes the message proto.ProtoClientRequestRoomStream. + * Use `create(ProtoClientRequestRoomStreamSchema)` to create a new message. + */ +export const ProtoClientRequestRoomStreamSchema: GenMessage = /*@__PURE__*/ + messageDesc(file_types, 19); + +/** + * ProtoClientDisconnected message + * + * @generated from message proto.ProtoClientDisconnected + */ +export type ProtoClientDisconnected = Message<"proto.ProtoClientDisconnected"> & { + /** + * @generated from field: string session_id = 1; + */ + sessionId: string; + + /** + * @generated from field: repeated int32 controller_slots = 2; + */ + controllerSlots: number[]; +}; + +/** + * Describes the message proto.ProtoClientDisconnected. + * Use `create(ProtoClientDisconnectedSchema)` to create a new message. + */ +export const ProtoClientDisconnectedSchema: GenMessage = /*@__PURE__*/ + messageDesc(file_types, 20); + +/** + * ProtoServerPushStream message + * + * @generated from message proto.ProtoServerPushStream + */ +export type ProtoServerPushStream = Message<"proto.ProtoServerPushStream"> & { + /** + * @generated from field: string room_name = 1; + */ + roomName: string; +}; + +/** + * Describes the message proto.ProtoServerPushStream. + * Use `create(ProtoServerPushStreamSchema)` to create a new message. + */ +export const ProtoServerPushStreamSchema: GenMessage = /*@__PURE__*/ + messageDesc(file_types, 21); + diff --git a/packages/input/src/streamwrapper.ts b/packages/input/src/streamwrapper.ts new file mode 100644 index 00000000..26fc18fa --- /dev/null +++ b/packages/input/src/streamwrapper.ts @@ -0,0 +1,81 @@ +import { pbStream, type ProtobufStream } from "@libp2p/utils"; +import type { Stream } from "@libp2p/interface"; +import { bufbuildAdapter } from "./utils"; +import { + ProtoMessage, + ProtoMessageSchema, + ProtoMessageBase, +} from "./proto/messages_pb"; + +type MessageHandler = ( + data: any, + base: ProtoMessageBase, +) => void | Promise; + +export class P2PMessageStream { + private pb: ProtobufStream; + private handlers = new Map(); + private closed = false; + private readLoopRunning = false; + + constructor(stream: Stream) { + this.pb = pbStream(stream); + } + + public on(payloadType: string, handler: MessageHandler): void { + if (!this.handlers.has(payloadType)) { + this.handlers.set(payloadType, []); + } + this.handlers.get(payloadType)!.push(handler); + + if (!this.readLoopRunning) this.startReading().catch(console.error); + } + + private async startReading(): Promise { + if (this.readLoopRunning || this.closed) return; + this.readLoopRunning = true; + + while (!this.closed) { + try { + const msg: ProtoMessage = await this.pb.read( + bufbuildAdapter(ProtoMessageSchema), + ); + + const payloadType = msg.messageBase?.payloadType; + if (payloadType && this.handlers.has(payloadType)) { + const handlers = this.handlers.get(payloadType)!; + if (msg.payload.value) { + for (const handler of handlers) { + try { + await handler(msg.payload.value, msg.messageBase); + } catch (err) { + console.error(`Error in handler for ${payloadType}:`, err); + } + } + } + } + } catch (err) { + if (this.closed) break; + console.error("Stream read error:", err); + this.close(); + } + } + + this.readLoopRunning = false; + } + + public async write( + message: ProtoMessage, + options?: { signal?: AbortSignal }, + ): Promise { + if (this.closed) + throw new Error("Cannot write to closed stream"); + + await this.pb.write(message, bufbuildAdapter(ProtoMessageSchema), options); + } + + public close(): void { + this.closed = true; + this.handlers.clear(); + } +} diff --git a/packages/input/src/utils.ts b/packages/input/src/utils.ts new file mode 100644 index 00000000..4db1786f --- /dev/null +++ b/packages/input/src/utils.ts @@ -0,0 +1,95 @@ +import { create, toBinary, fromBinary } from "@bufbuild/protobuf"; +import type { Message } from "@bufbuild/protobuf"; +import { Uint8ArrayList } from "uint8arraylist"; +import type { GenMessage } from "@bufbuild/protobuf/codegenv2"; +import { timestampFromDate } from "@bufbuild/protobuf/wkt"; +import { + ProtoLatencyTracker, + ProtoLatencyTrackerSchema, + ProtoTimestampEntrySchema, +} from "./proto/latency_tracker_pb"; +import { + ProtoMessage, + ProtoMessageSchema, + ProtoMessageBaseSchema, +} from "./proto/messages_pb"; + +export function bufbuildAdapter(schema: GenMessage) { + return { + encode: (data: T): Uint8Array => { + return toBinary(schema, data); + }, + decode: (data: Uint8Array | Uint8ArrayList): T => { + // Convert Uint8ArrayList to Uint8Array if needed + const bytes = data instanceof Uint8ArrayList ? data.subarray() : data; + return fromBinary(schema, bytes); + }, + }; +} + +// Latency tracker helpers +export function createLatencyTracker(sequenceId?: string): ProtoLatencyTracker { + return create(ProtoLatencyTrackerSchema, { + sequenceId: sequenceId || crypto.randomUUID(), + timestamps: [], + }); +} + +export function addLatencyTimestamp( + tracker: ProtoLatencyTracker, + stage: string, +): ProtoLatencyTracker { + const entry = create(ProtoTimestampEntrySchema, { + stage, + time: timestampFromDate(new Date()), + }); + + return { + ...tracker, + timestamps: [...tracker.timestamps, entry], + }; +} + +interface CreateMessageOptions { + sequenceId?: string; +} + +function derivePayloadCase(data: Message): string { + // Extract case from $typeName: "proto.ProtoICE" -> "ice" + // "proto.ProtoControllerAttach" -> "controllerAttach" + const typeName = data.$typeName; + if (!typeName) + throw new Error("Message has no $typeName"); + + // Remove "proto.Proto" prefix and convert first char to lowercase + const caseName = typeName.replace(/^proto\.Proto/, ""); + + // Convert PascalCase to camelCase + // If it's all caps (like SDP, ICE), lowercase everything + // Otherwise, just lowercase the first character + if (caseName === caseName.toUpperCase()) { + return caseName.toLowerCase(); + } + return caseName.charAt(0).toLowerCase() + caseName.slice(1); +} + +export function createMessage( + data: Message, + payloadType: string, + options?: CreateMessageOptions, +): ProtoMessage { + const payloadCase = derivePayloadCase(data); + + return create(ProtoMessageSchema, { + messageBase: create(ProtoMessageBaseSchema, { + payloadType, + latency: options?.sequenceId + ? createLatencyTracker(options.sequenceId) + : undefined, + }), + payload: { + case: payloadCase, + value: data, + } as any, // Type assertion needed for dynamic case + }); +} diff --git a/packages/input/src/webrtc-stream.ts b/packages/input/src/webrtc-stream.ts index 879cba35..b91e5b3c 100644 --- a/packages/input/src/webrtc-stream.ts +++ b/packages/input/src/webrtc-stream.ts @@ -1,9 +1,3 @@ -import { - NewMessageRaw, - NewMessageSDP, - NewMessageICE, - SafeStream, -} from "./messages"; import { webSockets } from "@libp2p/websockets"; import { webTransport } from "@libp2p/webtransport"; import { createLibp2p, Libp2p } from "libp2p"; @@ -13,19 +7,32 @@ import { identify } from "@libp2p/identify"; import { multiaddr } from "@multiformats/multiaddr"; import { Connection } from "@libp2p/interface"; import { ping } from "@libp2p/ping"; +import { createMessage } from "./utils"; +import { create } from "@bufbuild/protobuf"; +import { + ProtoClientRequestRoomStream, + ProtoClientRequestRoomStreamSchema, + ProtoICE, + ProtoICESchema, ProtoRaw, + ProtoSDP, + ProtoSDPSchema +} from "./proto/types_pb"; +import { P2PMessageStream } from "./streamwrapper"; const NESTRI_PROTOCOL_STREAM_REQUEST = "/nestri-relay/stream-request/1.0.0"; export class WebRTCStream { private _p2p: Libp2p | undefined = undefined; private _p2pConn: Connection | undefined = undefined; - private _p2pSafeStream: SafeStream | undefined = undefined; + private _msgStream: P2PMessageStream | undefined = undefined; private _pc: RTCPeerConnection | undefined = undefined; private _audioTrack: MediaStreamTrack | undefined = undefined; private _videoTrack: MediaStreamTrack | undefined = undefined; private _dataChannel: RTCDataChannel | undefined = undefined; - private _onConnected: ((stream: MediaStream | null) => void) | undefined = undefined; - private _connectionTimer: NodeJS.Timeout | NodeJS.Timer | undefined = undefined; + private _onConnected: ((stream: MediaStream | null) => void) | undefined = + undefined; + private _connectionTimer: NodeJS.Timeout | NodeJS.Timer | undefined = + undefined; private _serverURL: string | undefined = undefined; private _roomName: string | undefined = undefined; private _isConnected: boolean = false; @@ -89,14 +96,20 @@ export class WebRTCStream { .newStream(NESTRI_PROTOCOL_STREAM_REQUEST) .catch(console.error); if (stream) { - this._p2pSafeStream = new SafeStream(stream); + this._msgStream = new P2PMessageStream(stream); console.log("Stream opened with peer"); let iceHolder: RTCIceCandidateInit[] = []; - this._p2pSafeStream.registerCallback("ice-candidate", (data) => { + this._msgStream.on("ice-candidate", (data: ProtoICE) => { + const cand: RTCIceCandidateInit = { + candidate: data.candidate.candidate, + sdpMLineIndex: data.candidate.sdpMLineIndex, + sdpMid: data.candidate.sdpMid, + usernameFragment: data.candidate.usernameFragment, + }; if (this._pc) { if (this._pc.remoteDescription) { - this._pc.addIceCandidate(data.candidate).catch((err) => { + this._pc.addIceCandidate(cand).catch((err) => { console.error("Error adding ICE candidate:", err); }); // Add held candidates @@ -107,45 +120,70 @@ export class WebRTCStream { }); iceHolder = []; } else { - iceHolder.push(data.candidate); + iceHolder.push(cand); } } else { - iceHolder.push(data.candidate); + iceHolder.push(cand); } }); - this._p2pSafeStream.registerCallback("offer", async (data) => { + this._msgStream.on("session-assigned", (data: ProtoClientRequestRoomStream) => { + const sessionId = data.sessionId; + localStorage.setItem("nestri-session-id", sessionId); + console.log("Session ID assigned:", sessionId, "for room:", data.roomName); + }); + + this._msgStream.on("offer", async (data: ProtoSDP) => { if (!this._pc) { // Setup peer connection now this._setupPeerConnection(); } - await this._pc!.setRemoteDescription(data.sdp); + await this._pc!.setRemoteDescription({ + sdp: data.sdp.sdp, + type: data.sdp.type as RTCSdpType, + }); // Create our answer const answer = await this._pc!.createAnswer(); // Force stereo in Chromium browsers answer.sdp = this.forceOpusStereo(answer.sdp!); await this._pc!.setLocalDescription(answer); // Send answer back - const answerMsg = NewMessageSDP("answer", answer); - await this._p2pSafeStream?.writeMessage(answerMsg); + const answerMsg = createMessage( + create(ProtoSDPSchema, { + sdp: answer, + }), + "answer", + ); + await this._msgStream?.write(answerMsg); }); - this._p2pSafeStream.registerCallback("request-stream-offline", (data) => { - console.warn("Stream is offline for room:", data.roomName); + this._msgStream.on("request-stream-offline", (msg: ProtoRaw) => { + console.warn("Stream is offline for room:", msg.data); this._onConnected?.(null); }); + const clientId = localStorage.getItem("nestri-session-id"); + if (clientId) { + console.debug("Using existing session ID:", clientId); + } + // Send stream request - // marshal room name into json - const request = NewMessageRaw( + const requestMsg = createMessage( + create(ProtoClientRequestRoomStreamSchema, { + roomName: roomName, + sessionId: clientId, + }), "request-stream-room", - roomName, ); - await this._p2pSafeStream.writeMessage(request); + await this._msgStream.write(requestMsg); } } } + public getSessionID(): string { + return localStorage.getItem("nestri-session-id") || ""; + } + // Forces opus to stereo in Chromium browsers, because of course private forceOpusStereo(SDP: string): string { // Look for "minptime=10;useinbandfec=1" and replace with "minptime=10;useinbandfec=1;stereo=1;sprop-stereo=1;" @@ -200,11 +238,16 @@ export class WebRTCStream { this._pc.onicecandidate = (e) => { if (e.candidate) { - const iceMsg = NewMessageICE("ice-candidate", e.candidate); - if (this._p2pSafeStream) { - this._p2pSafeStream.writeMessage(iceMsg).catch((err) => - console.error("Error sending ICE candidate:", err), - ); + const iceMsg = createMessage( + create(ProtoICESchema, { + candidate: e.candidate, + }), + "ice-candidate", + ); + if (this._msgStream) { + this._msgStream + .write(iceMsg) + .catch((err) => console.error("Error sending ICE candidate:", err)); } else { console.warn("P2P stream not established, cannot send ICE candidate"); } @@ -218,8 +261,7 @@ export class WebRTCStream { } private _checkConnectionState() { - if (!this._pc || !this._p2p || !this._p2pConn) - return; + if (!this._pc || !this._p2p || !this._p2pConn) return; console.debug("Checking connection state:", { connectionState: this._pc.connectionState, @@ -286,7 +328,9 @@ export class WebRTCStream { // Attempt to reconnect only if not already connected if (!this._isConnected && this._serverURL && this._roomName) { - this._setup(this._serverURL, this._roomName).catch((err) => console.error("Reconnection failed:", err)); + this._setup(this._serverURL, this._roomName).catch((err) => + console.error("Reconnection failed:", err), + ); } } @@ -335,7 +379,9 @@ export class WebRTCStream { } public removeDataChannelCallback(callback: (data: any) => void) { - this._dataChannelCallbacks = this._dataChannelCallbacks.filter(cb => cb !== callback); + this._dataChannelCallbacks = this._dataChannelCallbacks.filter( + (cb) => cb !== callback, + ); } private _setupDataChannelEvents() { @@ -343,7 +389,7 @@ export class WebRTCStream { this._dataChannel.onclose = () => console.log("sendChannel has closed"); this._dataChannel.onopen = () => console.log("sendChannel has opened"); - this._dataChannel.onmessage = (event => { + this._dataChannel.onmessage = (event) => { // Parse as ProtoBuf message const data = event.data; // Call registered callback if exists @@ -354,7 +400,7 @@ export class WebRTCStream { console.error("Error in data channel callback:", err); } }); - }); + }; } private _gatherFrameRate() { diff --git a/packages/play-standalone/src/pages/[room].astro b/packages/play-standalone/src/pages/[room].astro index fbf0c7e1..22e5305b 100644 --- a/packages/play-standalone/src/pages/[room].astro +++ b/packages/play-standalone/src/pages/[room].astro @@ -106,7 +106,7 @@ if (envs_map.size > 0) { if (e.gamepad.id.toLowerCase().includes("nestri")) return; - let disconnected = nestriControllers.find((c) => c.getSlot() === e.gamepad.index); + let disconnected = nestriControllers.find((c) => c.getLocalSlot() === e.gamepad.index); if (disconnected) { disconnected.dispose(); nestriControllers = nestriControllers.filter((c) => c !== disconnected); diff --git a/packages/relay/go.mod b/packages/relay/go.mod index 7e5a254c..f0f53dd3 100644 --- a/packages/relay/go.mod +++ b/packages/relay/go.mod @@ -33,7 +33,7 @@ require ( github.com/ipfs/go-cid v0.5.0 // indirect github.com/jackpal/go-nat-pmp v1.0.2 // indirect github.com/jbenet/go-temp-err-catcher v0.1.0 // indirect - github.com/klauspost/compress v1.18.0 // indirect + github.com/klauspost/compress v1.18.1 // indirect github.com/klauspost/cpuid/v2 v2.3.0 // indirect github.com/koron/go-ssdp v0.1.0 // indirect github.com/libp2p/go-buffer-pool v0.1.0 // indirect diff --git a/packages/relay/go.sum b/packages/relay/go.sum index d3890e37..7f5ff0b3 100644 --- a/packages/relay/go.sum +++ b/packages/relay/go.sum @@ -82,8 +82,8 @@ github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCV github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= -github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= +github.com/klauspost/compress v1.18.1 h1:bcSGx7UbpBqMChDtsF28Lw6v/G94LPrrbMbdC3JH2co= +github.com/klauspost/compress v1.18.1/go.mod h1:ZQFFVG+MdnR0P+l6wpXgIL4NTtwiKIdBnrBd8Nrxr+0= github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y= github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= github.com/koron/go-ssdp v0.1.0 h1:ckl5x5H6qSNFmi+wCuROvvGUu2FQnMbQrU95IHCcv3Y= diff --git a/packages/relay/internal/common/safebufio.go b/packages/relay/internal/common/safebufio.go index e0f8b337..33558609 100644 --- a/packages/relay/internal/common/safebufio.go +++ b/packages/relay/internal/common/safebufio.go @@ -3,16 +3,28 @@ package common import ( "bufio" "encoding/binary" - "encoding/json" "errors" "io" + gen "relay/internal/proto" "sync" "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/reflect/protoreflect" + "google.golang.org/protobuf/types/known/timestamppb" ) -// MaxSize is the maximum allowed data size (1MB) -const MaxSize = 1024 * 1024 +// readUvarint reads an unsigned varint from the reader +func readUvarint(r io.ByteReader) (uint64, error) { + return binary.ReadUvarint(r) +} + +// writeUvarint writes an unsigned varint to the writer +func writeUvarint(w io.Writer, x uint64) error { + buf := make([]byte, binary.MaxVarintLen64) + n := binary.PutUvarint(buf, x) + _, err := w.Write(buf[:n]) + return err +} // SafeBufioRW wraps a bufio.ReadWriter for sending and receiving JSON and protobufs safely type SafeBufioRW struct { @@ -24,83 +36,6 @@ func NewSafeBufioRW(brw *bufio.ReadWriter) *SafeBufioRW { return &SafeBufioRW{brw: brw} } -// SendJSON serializes the given data as JSON and sends it with a 4-byte length prefix -func (bu *SafeBufioRW) SendJSON(data interface{}) error { - bu.mutex.Lock() - defer bu.mutex.Unlock() - - jsonData, err := json.Marshal(data) - if err != nil { - return err - } - - if len(jsonData) > MaxSize { - return errors.New("JSON data exceeds maximum size") - } - - // Write the 4-byte length prefix - if err = binary.Write(bu.brw, binary.BigEndian, uint32(len(jsonData))); err != nil { - return err - } - - // Write the JSON data - if _, err = bu.brw.Write(jsonData); err != nil { - return err - } - - // Flush the writer to ensure data is sent - return bu.brw.Flush() -} - -// ReceiveJSON reads a 4-byte length prefix, then reads and unmarshals the JSON -func (bu *SafeBufioRW) ReceiveJSON(dest interface{}) error { - bu.mutex.RLock() - defer bu.mutex.RUnlock() - - // Read the 4-byte length prefix - var length uint32 - if err := binary.Read(bu.brw, binary.BigEndian, &length); err != nil { - return err - } - - if length > MaxSize { - return errors.New("received JSON data exceeds maximum size") - } - - // Read the JSON data - data := make([]byte, length) - if _, err := io.ReadFull(bu.brw, data); err != nil { - return err - } - - return json.Unmarshal(data, dest) -} - -// Receive reads a 4-byte length prefix, then reads the raw data -func (bu *SafeBufioRW) Receive() ([]byte, error) { - bu.mutex.RLock() - defer bu.mutex.RUnlock() - - // Read the 4-byte length prefix - var length uint32 - if err := binary.Read(bu.brw, binary.BigEndian, &length); err != nil { - return nil, err - } - - if length > MaxSize { - return nil, errors.New("received data exceeds maximum size") - } - - // Read the raw data - data := make([]byte, length) - if _, err := io.ReadFull(bu.brw, data); err != nil { - return nil, err - } - - return data, nil -} - -// SendProto serializes the given protobuf message and sends it with a 4-byte length prefix func (bu *SafeBufioRW) SendProto(msg proto.Message) error { bu.mutex.Lock() defer bu.mutex.Unlock() @@ -110,12 +45,8 @@ func (bu *SafeBufioRW) SendProto(msg proto.Message) error { return err } - if len(protoData) > MaxSize { - return errors.New("protobuf data exceeds maximum size") - } - - // Write the 4-byte length prefix - if err = binary.Write(bu.brw, binary.BigEndian, uint32(len(protoData))); err != nil { + // Write varint length prefix + if err := writeUvarint(bu.brw, uint64(len(protoData))); err != nil { return err } @@ -124,25 +55,19 @@ func (bu *SafeBufioRW) SendProto(msg proto.Message) error { return err } - // Flush the writer to ensure data is sent return bu.brw.Flush() } -// ReceiveProto reads a 4-byte length prefix, then reads and unmarshals the protobuf func (bu *SafeBufioRW) ReceiveProto(msg proto.Message) error { bu.mutex.RLock() defer bu.mutex.RUnlock() - // Read the 4-byte length prefix - var length uint32 - if err := binary.Read(bu.brw, binary.BigEndian, &length); err != nil { + // Read varint length prefix + length, err := readUvarint(bu.brw) + if err != nil { return err } - if length > MaxSize { - return errors.New("received Protobuf data exceeds maximum size") - } - // Read the Protobuf data data := make([]byte, length) if _, err := io.ReadFull(bu.brw, data); err != nil { @@ -152,24 +77,51 @@ func (bu *SafeBufioRW) ReceiveProto(msg proto.Message) error { return proto.Unmarshal(data, msg) } -// Write writes raw data to the underlying buffer -func (bu *SafeBufioRW) Write(data []byte) (int, error) { - bu.mutex.Lock() - defer bu.mutex.Unlock() +type CreateMessageOptions struct { + SequenceID string + Latency *gen.ProtoLatencyTracker +} - if len(data) > MaxSize { - return 0, errors.New("data exceeds maximum size") +func CreateMessage(payload proto.Message, payloadType string, opts *CreateMessageOptions) (*gen.ProtoMessage, error) { + msg := &gen.ProtoMessage{ + MessageBase: &gen.ProtoMessageBase{ + PayloadType: payloadType, + }, } - n, err := bu.brw.Write(data) - if err != nil { - return n, err + if opts != nil { + if opts.Latency != nil { + msg.MessageBase.Latency = opts.Latency + } else if opts.SequenceID != "" { + msg.MessageBase.Latency = &gen.ProtoLatencyTracker{ + SequenceId: opts.SequenceID, + Timestamps: []*gen.ProtoTimestampEntry{ + { + Stage: "created", + Time: timestamppb.Now(), + }, + }, + } + } + } + + // Use reflection to set the oneof field automatically + msgReflect := msg.ProtoReflect() + payloadReflect := payload.ProtoReflect() + + oneofDesc := msgReflect.Descriptor().Oneofs().ByName("payload") + if oneofDesc == nil { + return nil, errors.New("payload oneof not found") } - // Flush the writer to ensure data is sent - if err = bu.brw.Flush(); err != nil { - return n, err + fields := oneofDesc.Fields() + for i := 0; i < fields.Len(); i++ { + field := fields.Get(i) + if field.Message() != nil && field.Message().FullName() == payloadReflect.Descriptor().FullName() { + msgReflect.Set(field, protoreflect.ValueOfMessage(payloadReflect)) + return msg, nil + } } - return n, nil + return nil, errors.New("payload type not found in oneof") } diff --git a/packages/relay/internal/connections/datachannel.go b/packages/relay/internal/connections/datachannel.go index ec35cf5e..07ec4992 100644 --- a/packages/relay/internal/connections/datachannel.go +++ b/packages/relay/internal/connections/datachannel.go @@ -31,16 +31,18 @@ func NewNestriDataChannel(dc *webrtc.DataChannel) *NestriDataChannel { } // Decode message - var base gen.ProtoMessageInput + var base gen.ProtoMessage if err := proto.Unmarshal(msg.Data, &base); err != nil { slog.Error("failed to decode binary DataChannel message", "err", err) return } - // Handle message type callback - if callback, ok := ndc.callbacks["input"]; ok { - go callback(msg.Data) - } // We don't care about unhandled messages + // Route based on PayloadType + if base.MessageBase != nil && len(base.MessageBase.PayloadType) > 0 { + if callback, ok := ndc.callbacks[base.MessageBase.PayloadType]; ok { + go callback(msg.Data) + } + } }) return ndc diff --git a/packages/relay/internal/connections/messages.go b/packages/relay/internal/connections/messages.go deleted file mode 100644 index 1998b122..00000000 --- a/packages/relay/internal/connections/messages.go +++ /dev/null @@ -1,94 +0,0 @@ -package connections - -import ( - "encoding/json" - "relay/internal/common" - - "github.com/pion/webrtc/v4" -) - -// MessageBase is the base type for any JSON message -type MessageBase struct { - Type string `json:"payload_type"` - Latency *common.LatencyTracker `json:"latency,omitempty"` -} - -type MessageRaw struct { - MessageBase - Data json.RawMessage `json:"data"` -} - -func NewMessageRaw(t string, data json.RawMessage) *MessageRaw { - return &MessageRaw{ - MessageBase: MessageBase{ - Type: t, - }, - Data: data, - } -} - -type MessageLog struct { - MessageBase - Level string `json:"level"` - Message string `json:"message"` - Time string `json:"time"` -} - -func NewMessageLog(t string, level, message, time string) *MessageLog { - return &MessageLog{ - MessageBase: MessageBase{ - Type: t, - }, - Level: level, - Message: message, - Time: time, - } -} - -type MessageMetrics struct { - MessageBase - UsageCPU float64 `json:"usage_cpu"` - UsageMemory float64 `json:"usage_memory"` - Uptime uint64 `json:"uptime"` - PipelineLatency float64 `json:"pipeline_latency"` -} - -func NewMessageMetrics(t string, usageCPU, usageMemory float64, uptime uint64, pipelineLatency float64) *MessageMetrics { - return &MessageMetrics{ - MessageBase: MessageBase{ - Type: t, - }, - UsageCPU: usageCPU, - UsageMemory: usageMemory, - Uptime: uptime, - PipelineLatency: pipelineLatency, - } -} - -type MessageICE struct { - MessageBase - Candidate webrtc.ICECandidateInit `json:"candidate"` -} - -func NewMessageICE(t string, candidate webrtc.ICECandidateInit) *MessageICE { - return &MessageICE{ - MessageBase: MessageBase{ - Type: t, - }, - Candidate: candidate, - } -} - -type MessageSDP struct { - MessageBase - SDP webrtc.SessionDescription `json:"sdp"` -} - -func NewMessageSDP(t string, sdp webrtc.SessionDescription) *MessageSDP { - return &MessageSDP{ - MessageBase: MessageBase{ - Type: t, - }, - SDP: sdp, - } -} diff --git a/packages/relay/internal/core/core.go b/packages/relay/internal/core/core.go index 09b26bb2..6288d5f6 100644 --- a/packages/relay/internal/core/core.go +++ b/packages/relay/internal/core/core.go @@ -10,6 +10,7 @@ import ( "os" "relay/internal/common" "relay/internal/shared" + "time" "github.com/libp2p/go-libp2p" pubsub "github.com/libp2p/go-libp2p-pubsub" @@ -37,6 +38,16 @@ var globalRelay *Relay // -- Structs -- +// ClientSession tracks browser client connections +type ClientSession struct { + PeerID peer.ID + SessionID string + RoomName string + ConnectedAt time.Time + LastActivity time.Time + ControllerSlots []int32 // Track which controller slots this client owns +} + // Relay structure enhanced with metrics and state type Relay struct { *PeerInfo @@ -48,6 +59,7 @@ type Relay struct { // Local LocalRooms *common.SafeMap[ulid.ULID, *shared.Room] // room ID -> local Room struct (hosted by this relay) LocalMeshConnections *common.SafeMap[peer.ID, *webrtc.PeerConnection] // peer ID -> PeerConnection (connected to this relay) + ClientSessions *common.SafeMap[peer.ID, *ClientSession] // peer ID -> ClientSession // Protocols ProtocolRegistry @@ -144,6 +156,7 @@ func NewRelay(ctx context.Context, port int, identityKey crypto.PrivKey) (*Relay PingService: pingSvc, LocalRooms: common.NewSafeMap[ulid.ULID, *shared.Room](), LocalMeshConnections: common.NewSafeMap[peer.ID, *webrtc.PeerConnection](), + ClientSessions: common.NewSafeMap[peer.ID, *ClientSession](), } // Add network notifier after relay is initialized diff --git a/packages/relay/internal/core/protocol_stream.go b/packages/relay/internal/core/protocol_stream.go index 4272dd06..bff77ccc 100644 --- a/packages/relay/internal/core/protocol_stream.go +++ b/packages/relay/internal/core/protocol_stream.go @@ -3,7 +3,6 @@ package core import ( "bufio" "context" - "encoding/json" "errors" "fmt" "io" @@ -11,6 +10,11 @@ import ( "relay/internal/common" "relay/internal/connections" "relay/internal/shared" + "time" + + gen "relay/internal/proto" + + "google.golang.org/protobuf/proto" "github.com/libp2p/go-libp2p/core/network" "github.com/libp2p/go-libp2p/core/peer" @@ -69,7 +73,8 @@ func (sp *StreamProtocol) handleStreamRequest(stream network.Stream) { var currentRoomName string // Track the current room for this stream iceHolder := make([]webrtc.ICECandidateInit, 0) for { - data, err := safeBRW.Receive() + var msgWrapper gen.ProtoMessage + err := safeBRW.ReceiveProto(&msgWrapper) if err != nil { if errors.Is(err, io.EOF) || errors.Is(err, network.ErrReset) { slog.Debug("Stream request connection closed by peer", "peer", stream.Conn().RemotePeer()) @@ -82,390 +87,441 @@ func (sp *StreamProtocol) handleStreamRequest(stream network.Stream) { return } - var baseMsg connections.MessageBase - if err = json.Unmarshal(data, &baseMsg); err != nil { - slog.Error("Failed to unmarshal base message", "err", err) - continue + if msgWrapper.MessageBase == nil { + slog.Error("No MessageBase in stream request") + _ = stream.Reset() + return } - switch baseMsg.Type { + switch msgWrapper.MessageBase.PayloadType { case "request-stream-room": - var rawMsg connections.MessageRaw - if err = json.Unmarshal(data, &rawMsg); err != nil { - slog.Error("Failed to unmarshal raw message for room stream request", "err", err) - continue - } + reqMsg := msgWrapper.GetClientRequestRoomStream() + if reqMsg != nil { + currentRoomName = reqMsg.RoomName + + // Generate session ID if not provided (first connection) + sessionID := reqMsg.SessionId + if sessionID == "" { + ulid, err := common.NewULID() + if err != nil { + slog.Error("Failed to generate session ID", "err", err) + continue + } + sessionID = ulid.String() + } - var roomName string - if err = json.Unmarshal(rawMsg.Data, &roomName); err != nil { - slog.Error("Failed to unmarshal room name from raw message", "err", err) - continue - } + session := &ClientSession{ + PeerID: stream.Conn().RemotePeer(), + SessionID: sessionID, + RoomName: reqMsg.RoomName, + ConnectedAt: time.Now(), + LastActivity: time.Now(), + } + sp.relay.ClientSessions.Set(stream.Conn().RemotePeer(), session) - currentRoomName = roomName // Store the room name - slog.Info("Received stream request for room", "room", roomName) + slog.Info("Client session established", "peer", session.PeerID, "session", sessionID, "room", reqMsg.RoomName) - room := sp.relay.GetRoomByName(roomName) - if room == nil || !room.IsOnline() || room.OwnerID != sp.relay.ID { - // TODO: Allow forward requests to other relays from here? - slog.Debug("Cannot provide stream for nil, offline or non-owned room", "room", roomName, "is_online", room != nil && room.IsOnline(), "is_owner", room != nil && room.OwnerID == sp.relay.ID) - // Respond with "request-stream-offline" message with room name - // TODO: Store the peer and send "online" message when the room comes online - roomNameData, err := json.Marshal(roomName) + // Send session ID back to client + sesMsg, err := common.CreateMessage( + &gen.ProtoClientRequestRoomStream{SessionId: sessionID, RoomName: reqMsg.RoomName}, + "session-assigned", nil, + ) if err != nil { - slog.Error("Failed to marshal room name for request stream offline", "room", roomName, "err", err) + slog.Error("Failed to create proto message", "err", err) continue - } else { - if err = safeBRW.SendJSON(connections.NewMessageRaw( - "request-stream-offline", - roomNameData, - )); err != nil { - slog.Error("Failed to send request stream offline message", "room", roomName, "err", err) - } } - continue - } - - pc, err := common.CreatePeerConnection(func() { - slog.Info("PeerConnection closed for requested stream", "room", roomName) - // Cleanup the stream connection - if roomMap, ok := sp.servedConns.Get(roomName); ok { - roomMap.Delete(stream.Conn().RemotePeer()) - // If the room map is empty, delete it - if roomMap.Len() == 0 { - sp.servedConns.Delete(roomName) - } + if err = safeBRW.SendProto(sesMsg); err != nil { + slog.Error("Failed to send session assignment", "err", err) } - }) - if err != nil { - slog.Error("Failed to create PeerConnection for requested stream", "room", roomName, "err", err) - continue - } - // Add tracks - if room.AudioTrack != nil { - if _, err = pc.AddTrack(room.AudioTrack); err != nil { - slog.Error("Failed to add audio track for requested stream", "room", roomName, "err", err) - continue - } - } - if room.VideoTrack != nil { - if _, err = pc.AddTrack(room.VideoTrack); err != nil { - slog.Error("Failed to add video track for requested stream", "room", roomName, "err", err) + slog.Info("Received stream request for room", "room", reqMsg.RoomName) + + room := sp.relay.GetRoomByName(reqMsg.RoomName) + if room == nil || !room.IsOnline() || room.OwnerID != sp.relay.ID { + // TODO: Allow forward requests to other relays from here? + slog.Debug("Cannot provide stream for nil, offline or non-owned room", "room", reqMsg.RoomName, "is_online", room != nil && room.IsOnline(), "is_owner", room != nil && room.OwnerID == sp.relay.ID) + // Respond with "request-stream-offline" message with room name + // TODO: Store the peer and send "online" message when the room comes online + rawMsg, err := common.CreateMessage( + &gen.ProtoRaw{ + Data: reqMsg.RoomName, + }, + "request-stream-offline", nil, + ) + if err != nil { + slog.Error("Failed to create proto message", "err", err) + continue + } + if err = safeBRW.SendProto(rawMsg); err != nil { + slog.Error("Failed to send request stream offline message", "room", reqMsg.RoomName, "err", err) + } continue } - } - // DataChannel setup - settingOrdered := true - settingMaxRetransmits := uint16(2) - dc, err := pc.CreateDataChannel("relay-data", &webrtc.DataChannelInit{ - Ordered: &settingOrdered, - MaxRetransmits: &settingMaxRetransmits, - }) - if err != nil { - slog.Error("Failed to create DataChannel for requested stream", "room", roomName, "err", err) - continue - } - ndc := connections.NewNestriDataChannel(dc) - - ndc.RegisterOnOpen(func() { - slog.Debug("Relay DataChannel opened for requested stream", "room", roomName) - }) - ndc.RegisterOnClose(func() { - slog.Debug("Relay DataChannel closed for requested stream", "room", roomName) - }) - ndc.RegisterMessageCallback("input", func(data []byte) { - if room.DataChannel != nil { - if err = room.DataChannel.SendBinary(data); err != nil { - slog.Error("Failed to forward input message from mesh to upstream room", "room", roomName, "err", err) + pc, err := common.CreatePeerConnection(func() { + slog.Info("PeerConnection closed for requested stream", "room", reqMsg.RoomName) + // Cleanup the stream connection + if roomMap, ok := sp.servedConns.Get(reqMsg.RoomName); ok { + roomMap.Delete(stream.Conn().RemotePeer()) + // If the room map is empty, delete it + if roomMap.Len() == 0 { + sp.servedConns.Delete(reqMsg.RoomName) + } } + }) + if err != nil { + slog.Error("Failed to create PeerConnection for requested stream", "room", reqMsg.RoomName, "err", err) + continue } - }) - // ICE Candidate handling - pc.OnICECandidate(func(candidate *webrtc.ICECandidate) { - if candidate == nil { - return + // Create participant for this viewer + participant, err := shared.NewParticipant( + "", // session ID will be set if this is a client session + stream.Conn().RemotePeer(), + ) + if err != nil { + slog.Error("Failed to create participant", "room", reqMsg.RoomName, "err", err) + continue } - if err = safeBRW.SendJSON(connections.NewMessageICE("ice-candidate", candidate.ToJSON())); err != nil { - slog.Error("Failed to send ICE candidate message for requested stream", "room", roomName, "err", err) - return + // If this is a client session, link it + if session, ok := sp.relay.ClientSessions.Get(stream.Conn().RemotePeer()); ok { + participant.SessionID = session.SessionID } - }) - // Create offer - offer, err := pc.CreateOffer(nil) - if err != nil { - slog.Error("Failed to create offer for requested stream", "room", roomName, "err", err) - continue - } - if err = pc.SetLocalDescription(offer); err != nil { - slog.Error("Failed to set local description for requested stream", "room", roomName, "err", err) - continue - } - if err = safeBRW.SendJSON(connections.NewMessageSDP("offer", offer)); err != nil { - slog.Error("Failed to send offer for requested stream", "room", roomName, "err", err) - continue - } + participant.PeerConnection = pc - // Store the connection - roomMap, ok := sp.servedConns.Get(roomName) - if !ok { - roomMap = common.NewSafeMap[peer.ID, *StreamConnection]() - sp.servedConns.Set(roomName, roomMap) - } - roomMap.Set(stream.Conn().RemotePeer(), &StreamConnection{ - pc: pc, - ndc: ndc, - }) + // Create per-participant tracks + if room.VideoTrack != nil { + participant.VideoTrack, err = webrtc.NewTrackLocalStaticRTP( + room.VideoTrack.Codec(), + "video-"+participant.ID.String(), + "nestri-"+reqMsg.RoomName+"-video", + ) + if err != nil { + slog.Error("Failed to create participant video track", "room", reqMsg.RoomName, "err", err) + continue + } - slog.Debug("Sent offer for requested stream") - case "ice-candidate": - var iceMsg connections.MessageICE - if err := json.Unmarshal(data, &iceMsg); err != nil { - slog.Error("Failed to unmarshal ICE message", "err", err) - continue - } - // Use currentRoomName to get the connection from nested map - if len(currentRoomName) > 0 { - if roomMap, ok := sp.servedConns.Get(currentRoomName); ok { - if conn, ok := roomMap.Get(stream.Conn().RemotePeer()); ok && conn.pc.RemoteDescription() != nil { - if err := conn.pc.AddICECandidate(iceMsg.Candidate); err != nil { - slog.Error("Failed to add ICE candidate", "err", err) - } - for _, heldIce := range iceHolder { - if err := conn.pc.AddICECandidate(heldIce); err != nil { - slog.Error("Failed to add held ICE candidate", "err", err) + rtpSender, err := pc.AddTrack(participant.VideoTrack) + if err != nil { + slog.Error("Failed to add participant video track", "room", reqMsg.RoomName, "err", err) + continue + } + + slog.Info("Added video track for participant", + "room", reqMsg.RoomName, + "participant", participant.ID, + "sender_id", fmt.Sprintf("%p", rtpSender)) + + // Relay packets from channel to track (VIDEO) + go func() { + for pkt := range participant.VideoChan { + // Use a context with timeout + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + + done := make(chan error, 1) + go func() { + done <- participant.VideoTrack.WriteRTP(pkt) + }() + + select { + case err := <-done: + cancel() + if err != nil { + if !errors.Is(err, io.ErrClosedPipe) { + slog.Debug("Failed to write video", "room", reqMsg.RoomName, "err", err) + } + return + } + case <-ctx.Done(): + cancel() + slog.Error("WriteRTP BLOCKED for >100ms!", + "participant", participant.ID, + "room", reqMsg.RoomName) + // Don't return, continue processing } } - // Clear the held candidates - iceHolder = make([]webrtc.ICECandidateInit, 0) - } else { - // Hold the candidate until remote description is set - iceHolder = append(iceHolder, iceMsg.Candidate) - } + }() } - } else { - // Hold the candidate until remote description is set - iceHolder = append(iceHolder, iceMsg.Candidate) - } - case "answer": - var answerMsg connections.MessageSDP - if err := json.Unmarshal(data, &answerMsg); err != nil { - slog.Error("Failed to unmarshal answer from signaling message", "err", err) - continue - } - // Use currentRoomName to get the connection from nested map - if len(currentRoomName) > 0 { - if roomMap, ok := sp.servedConns.Get(currentRoomName); ok { - if conn, ok := roomMap.Get(stream.Conn().RemotePeer()); ok { - if err := conn.pc.SetRemoteDescription(answerMsg.SDP); err != nil { - slog.Error("Failed to set remote description for answer", "err", err) - continue - } - slog.Debug("Set remote description for answer") - } else { - slog.Warn("Received answer without active PeerConnection") + if room.AudioTrack != nil { + participant.AudioTrack, err = webrtc.NewTrackLocalStaticRTP( + room.AudioTrack.Codec(), + "audio-"+participant.ID.String(), + "nestri-"+reqMsg.RoomName+"-audio", + ) + if err != nil { + slog.Error("Failed to create participant audio track", "room", reqMsg.RoomName, "err", err) + continue } - } - } else { - slog.Warn("Received answer without active PeerConnection") - } - } - } -} - -// requestStream manages the internals of the stream request -func (sp *StreamProtocol) requestStream(stream network.Stream, room *shared.Room) error { - brw := bufio.NewReadWriter(bufio.NewReader(stream), bufio.NewWriter(stream)) - safeBRW := common.NewSafeBufioRW(brw) - - slog.Debug("Requesting room stream from peer", "room", room.Name, "peer", stream.Conn().RemotePeer()) - - // Send room name to the remote peer - roomData, err := json.Marshal(room.Name) - if err != nil { - _ = stream.Close() - return fmt.Errorf("failed to marshal room name: %w", err) - } - if err = safeBRW.SendJSON(connections.NewMessageRaw( - "request-stream-room", - roomData, - )); err != nil { - _ = stream.Close() - return fmt.Errorf("failed to send room request: %w", err) - } - pc, err := common.CreatePeerConnection(func() { - slog.Info("Relay PeerConnection closed for requested stream", "room", room.Name) - _ = stream.Close() // ignore error as may be closed already - // Cleanup the stream connection - if ok := sp.requestedConns.Has(room.Name); ok { - sp.requestedConns.Delete(room.Name) - } - }) - if err != nil { - _ = stream.Close() - return fmt.Errorf("failed to create PeerConnection: %w", err) - } - - pc.OnTrack(func(track *webrtc.TrackRemote, receiver *webrtc.RTPReceiver) { - localTrack, _ := webrtc.NewTrackLocalStaticRTP(track.Codec().RTPCodecCapability, track.ID(), "relay-"+room.Name+"-"+track.Kind().String()) - slog.Debug("Received track for requested stream", "room", room.Name, "track_kind", track.Kind().String()) - - room.SetTrack(track.Kind(), localTrack) - - go func() { - for { - rtpPacket, _, err := track.ReadRTP() - if err != nil { - if !errors.Is(err, io.EOF) { - slog.Error("Failed to read RTP packet for requested stream room", "room", room.Name, "err", err) + _, err := pc.AddTrack(participant.AudioTrack) + if err != nil { + slog.Error("Failed to add participant audio track", "room", reqMsg.RoomName, "err", err) + continue } - break - } - err = localTrack.WriteRTP(rtpPacket) - if err != nil && !errors.Is(err, io.ErrClosedPipe) { - slog.Error("Failed to write RTP to local track for requested stream room", "room", room.Name, "err", err) - break + // Relay packets from channel to track (AUDIO) + go func() { + for pkt := range participant.AudioChan { + start := time.Now() + if err := participant.AudioTrack.WriteRTP(pkt); err != nil { + if !errors.Is(err, io.ErrClosedPipe) { + slog.Debug("Failed to write audio to participant", "room", reqMsg.RoomName, "err", err) + } + return + } + duration := time.Since(start) + if duration > 50*time.Millisecond { + slog.Warn("Slow audio WriteRTP detected", + "duration", duration, + "participant", participant.ID, + "room", reqMsg.RoomName) + } + } + }() } - } - }() - }) - - pc.OnDataChannel(func(dc *webrtc.DataChannel) { - ndc := connections.NewNestriDataChannel(dc) - ndc.RegisterOnOpen(func() { - slog.Debug("Relay DataChannel opened for requested stream", "room", room.Name) - }) - ndc.RegisterOnClose(func() { - slog.Debug("Relay DataChannel closed for requested stream", "room", room.Name) - }) - - // Set the DataChannel in the requestedConns map - if conn, ok := sp.requestedConns.Get(room.Name); ok { - conn.ndc = ndc - } else { - sp.requestedConns.Set(room.Name, &StreamConnection{ - pc: pc, - ndc: ndc, - }) - } - // We do not handle any messages from upstream here - }) - - pc.OnICECandidate(func(candidate *webrtc.ICECandidate) { - if candidate == nil { - return - } + // Add participant to room + room.AddParticipant(participant) + + // Cleanup on disconnect + cleanupParticipantID := participant.ID + pc.OnConnectionStateChange(func(state webrtc.PeerConnectionState) { + if state == webrtc.PeerConnectionStateClosed || + state == webrtc.PeerConnectionStateFailed || + state == webrtc.PeerConnectionStateDisconnected { + slog.Info("Participant disconnected from room", "room", reqMsg.RoomName, "participant", cleanupParticipantID) + room.RemoveParticipantByID(cleanupParticipantID) + participant.Close() + } + }) - if err = safeBRW.SendJSON(connections.NewMessageICE( - "ice-candidate", - candidate.ToJSON(), - )); err != nil { - slog.Error("Failed to send ICE candidate message for requested stream", "room", room.Name, "err", err) - return - } - }) + // DataChannel setup + settingOrdered := true + settingMaxRetransmits := uint16(2) + dc, err := pc.CreateDataChannel("relay-data", &webrtc.DataChannelInit{ + Ordered: &settingOrdered, + MaxRetransmits: &settingMaxRetransmits, + }) + if err != nil { + slog.Error("Failed to create DataChannel for requested stream", "room", reqMsg.RoomName, "err", err) + continue + } + ndc := connections.NewNestriDataChannel(dc) - // Handle incoming messages (offer and candidates) - go func() { - iceHolder := make([]webrtc.ICECandidateInit, 0) + ndc.RegisterOnOpen(func() { + slog.Debug("Relay DataChannel opened for requested stream", "room", reqMsg.RoomName) + }) + ndc.RegisterOnClose(func() { + slog.Debug("Relay DataChannel closed for requested stream", "room", reqMsg.RoomName) + }) + ndc.RegisterMessageCallback("input", func(data []byte) { + if room.DataChannel != nil { + if err = room.DataChannel.SendBinary(data); err != nil { + slog.Error("Failed to forward input message from mesh to upstream room", "room", reqMsg.RoomName, "err", err) + } + } + }) + // Track controller input separately + ndc.RegisterMessageCallback("controllerInput", func(data []byte) { + // Parse the message to track controller slots for client sessions + var msgWrapper gen.ProtoMessage + if err = proto.Unmarshal(data, &msgWrapper); err != nil { + slog.Error("Failed to unmarshal controller input", "err", err) + } else if msgWrapper.Payload != nil { + // Get the peer ID for this connection + peerID := stream.Conn().RemotePeer() + + // Check if it's a controller attach with assigned slot + if attach := msgWrapper.GetControllerAttach(); attach != nil && attach.Slot >= 0 { + if session, ok := sp.relay.ClientSessions.Get(peerID); ok { + // Check if slot already tracked + hasSlot := false + for _, slot := range session.ControllerSlots { + if slot == attach.Slot { + hasSlot = true + break + } + } + if !hasSlot { + session.ControllerSlots = append(session.ControllerSlots, attach.Slot) + session.LastActivity = time.Now() + slog.Info("Controller slot assigned to client session", + "session", session.SessionID, + "slot", attach.Slot, + "total_slots", len(session.ControllerSlots)) + } + } + } - for { - data, err := safeBRW.Receive() - if err != nil { - if errors.Is(err, io.EOF) || errors.Is(err, network.ErrReset) { - slog.Debug("Connection for requested stream closed by peer", "room", room.Name) - return - } + // Check if it's a controller detach + if detach := msgWrapper.GetControllerDetach(); detach != nil && detach.Slot >= 0 { + if session, ok := sp.relay.ClientSessions.Get(peerID); ok { + newSlots := make([]int32, 0, len(session.ControllerSlots)) + for _, slot := range session.ControllerSlots { + if slot != detach.Slot { + newSlots = append(newSlots, slot) + } + } + session.ControllerSlots = newSlots + session.LastActivity = time.Now() + slog.Info("Controller slot removed from client session", + "session", session.SessionID, + "slot", detach.Slot, + "remaining_slots", len(session.ControllerSlots)) + } + } - slog.Error("Failed to receive data for requested stream", "room", room.Name, "err", err) - _ = stream.Reset() + // Update last activity on any controller input + if session, ok := sp.relay.ClientSessions.Get(peerID); ok { + session.LastActivity = time.Now() + } + } - return - } + // Forward to upstream room + if room.DataChannel != nil { + if err = room.DataChannel.SendBinary(data); err != nil { + slog.Error("Failed to forward controller input from mesh to upstream room", "room", reqMsg.RoomName, "err", err) + } + } + }) - var baseMsg connections.MessageBase - if err = json.Unmarshal(data, &baseMsg); err != nil { - slog.Error("Failed to unmarshal base message for requested stream", "room", room.Name, "err", err) - return - } + // ICE Candidate handling + pc.OnICECandidate(func(candidate *webrtc.ICECandidate) { + if candidate == nil { + return + } - switch baseMsg.Type { - case "ice-candidate": - var iceMsg connections.MessageICE - if err = json.Unmarshal(data, &iceMsg); err != nil { - slog.Error("Failed to unmarshal ICE candidate for requested stream", "room", room.Name, "err", err) - continue - } - if conn, ok := sp.requestedConns.Get(room.Name); ok && conn.pc.RemoteDescription() != nil { - if err = conn.pc.AddICECandidate(iceMsg.Candidate); err != nil { - slog.Error("Failed to add ICE candidate for requested stream", "room", room.Name, "err", err) + candInit := candidate.ToJSON() + biggified := uint32(*candInit.SDPMLineIndex) + iceMsg, err := common.CreateMessage( + &gen.ProtoICE{ + Candidate: &gen.RTCIceCandidateInit{ + Candidate: candInit.Candidate, + SdpMLineIndex: &biggified, + SdpMid: candInit.SDPMid, + }, + }, + "ice-candidate", nil, + ) + if err != nil { + slog.Error("Failed to create proto message", "err", err) + return } - // Add held candidates - for _, heldCandidate := range iceHolder { - if err = conn.pc.AddICECandidate(heldCandidate); err != nil { - slog.Error("Failed to add held ICE candidate for requested stream", "room", room.Name, "err", err) - } + if err = safeBRW.SendProto(iceMsg); err != nil { + slog.Error("Failed to send ICE candidate message for requested stream", "room", reqMsg.RoomName, "err", err) + return } - // Clear the held candidates - iceHolder = make([]webrtc.ICECandidateInit, 0) - } else { - // Hold the candidate until remote description is set - iceHolder = append(iceHolder, iceMsg.Candidate) - } - case "offer": - var offerMsg connections.MessageSDP - if err = json.Unmarshal(data, &offerMsg); err != nil { - slog.Error("Failed to unmarshal offer for requested stream", "room", room.Name, "err", err) + }) + + // Create offer + offer, err := pc.CreateOffer(nil) + if err != nil { + slog.Error("Failed to create offer for requested stream", "room", reqMsg.RoomName, "err", err) continue } - if err = pc.SetRemoteDescription(offerMsg.SDP); err != nil { - slog.Error("Failed to set remote description for requested stream", "room", room.Name, "err", err) + if err = pc.SetLocalDescription(offer); err != nil { + slog.Error("Failed to set local description for requested stream", "room", reqMsg.RoomName, "err", err) continue } - answer, err := pc.CreateAnswer(nil) + offerMsg, err := common.CreateMessage( + &gen.ProtoSDP{ + Sdp: &gen.RTCSessionDescriptionInit{ + Sdp: offer.SDP, + Type: offer.Type.String(), + }, + }, + "offer", nil, + ) if err != nil { - slog.Error("Failed to create answer for requested stream", "room", room.Name, "err", err) - if err = stream.Reset(); err != nil { - slog.Error("Failed to reset stream for requested stream", "err", err) - } - return - } - if err = pc.SetLocalDescription(answer); err != nil { - slog.Error("Failed to set local description for requested stream", "room", room.Name, "err", err) - if err = stream.Reset(); err != nil { - slog.Error("Failed to reset stream for requested stream", "err", err) - } - return + slog.Error("Failed to create proto message", "err", err) + continue } - if err = safeBRW.SendJSON(connections.NewMessageSDP( - "answer", - answer, - )); err != nil { - slog.Error("Failed to send answer for requested stream", "room", room.Name, "err", err) + if err = safeBRW.SendProto(offerMsg); err != nil { + slog.Error("Failed to send offer for requested stream", "room", reqMsg.RoomName, "err", err) continue } // Store the connection - sp.requestedConns.Set(room.Name, &StreamConnection{ + roomMap, ok := sp.servedConns.Get(reqMsg.RoomName) + if !ok { + roomMap = common.NewSafeMap[peer.ID, *StreamConnection]() + sp.servedConns.Set(reqMsg.RoomName, roomMap) + } + roomMap.Set(stream.Conn().RemotePeer(), &StreamConnection{ pc: pc, - ndc: nil, + ndc: ndc, }) - slog.Debug("Sent answer for requested stream", "room", room.Name) - default: - slog.Warn("Unknown signaling message type", "room", room.Name, "type", baseMsg.Type) + slog.Debug("Sent offer for requested stream") + } else { + slog.Error("Could not get ClientRequestRoomStream for stream request") + } + case "ice-candidate": + iceMsg := msgWrapper.GetIce() + if iceMsg != nil { + smollified := uint16(*iceMsg.Candidate.SdpMLineIndex) + cand := webrtc.ICECandidateInit{ + Candidate: iceMsg.Candidate.Candidate, + SDPMid: iceMsg.Candidate.SdpMid, + SDPMLineIndex: &smollified, + UsernameFragment: iceMsg.Candidate.UsernameFragment, + } + // Use currentRoomName to get the connection from nested map + if len(currentRoomName) > 0 { + if roomMap, ok := sp.servedConns.Get(currentRoomName); ok { + if conn, ok := roomMap.Get(stream.Conn().RemotePeer()); ok && conn.pc.RemoteDescription() != nil { + if err = conn.pc.AddICECandidate(cand); err != nil { + slog.Error("Failed to add ICE candidate", "err", err) + } + for _, heldIce := range iceHolder { + if err := conn.pc.AddICECandidate(heldIce); err != nil { + slog.Error("Failed to add held ICE candidate", "err", err) + } + } + // Clear the held candidates + iceHolder = make([]webrtc.ICECandidateInit, 0) + } else { + // Hold the candidate until remote description is set + iceHolder = append(iceHolder, cand) + } + } + } else { + // Hold the candidate until remote description is set + iceHolder = append(iceHolder, cand) + } + } else { + slog.Error("Could not GetIce from ice-candidate") + } + case "answer": + answerMsg := msgWrapper.GetSdp() + if answerMsg != nil { + ansSdp := webrtc.SessionDescription{ + SDP: answerMsg.Sdp.Sdp, + Type: webrtc.NewSDPType(answerMsg.Sdp.Type), + } + // Use currentRoomName to get the connection from nested map + if len(currentRoomName) > 0 { + if roomMap, ok := sp.servedConns.Get(currentRoomName); ok { + if conn, ok := roomMap.Get(stream.Conn().RemotePeer()); ok { + if err = conn.pc.SetRemoteDescription(ansSdp); err != nil { + slog.Error("Failed to set remote description for answer", "err", err) + continue + } + slog.Debug("Set remote description for answer") + } else { + slog.Warn("Received answer without active PeerConnection") + } + } + } else { + slog.Warn("Received answer without active PeerConnection") + } + } else { + slog.Warn("Could not GetSdp from answer") } } - }() - - return nil + } } // handleStreamPush manages a stream push from a node (nestri-server) @@ -476,7 +532,8 @@ func (sp *StreamProtocol) handleStreamPush(stream network.Stream) { var room *shared.Room iceHolder := make([]webrtc.ICECandidateInit, 0) for { - data, err := safeBRW.Receive() + var msgWrapper gen.ProtoMessage + err := safeBRW.ReceiveProto(&msgWrapper) if err != nil { if errors.Is(err, io.EOF) || errors.Is(err, network.ErrReset) { slog.Debug("Stream push connection closed by peer", "peer", stream.Conn().RemotePeer(), "error", err) @@ -489,76 +546,78 @@ func (sp *StreamProtocol) handleStreamPush(stream network.Stream) { return } - var baseMsg connections.MessageBase - if err = json.Unmarshal(data, &baseMsg); err != nil { - slog.Error("Failed to unmarshal base message from base message", "err", err) - continue + if msgWrapper.MessageBase == nil { + slog.Error("No MessageBase in stream push") + _ = stream.Reset() + return } - switch baseMsg.Type { + switch msgWrapper.MessageBase.PayloadType { case "push-stream-room": - var rawMsg connections.MessageRaw - if err = json.Unmarshal(data, &rawMsg); err != nil { - slog.Error("Failed to unmarshal room name from data", "err", err) - continue - } - - var roomName string - if err = json.Unmarshal(rawMsg.Data, &roomName); err != nil { - slog.Error("Failed to unmarshal room name from raw message", "err", err) - continue - } - - slog.Info("Received stream push request for room", "room", roomName) + pushMsg := msgWrapper.GetServerPushStream() + if pushMsg != nil { + slog.Info("Received stream push request for room", "room", pushMsg.RoomName) + + room = sp.relay.GetRoomByName(pushMsg.RoomName) + if room != nil { + if room.OwnerID != sp.relay.ID { + slog.Error("Cannot push a stream to non-owned room", "room", room.Name, "owner_id", room.OwnerID) + continue + } + if room.IsOnline() { + slog.Error("Cannot push a stream to already online room", "room", room.Name) + continue + } + } else { + // Create a new room if it doesn't exist + room = sp.relay.CreateRoom(pushMsg.RoomName) + } - room = sp.relay.GetRoomByName(roomName) - if room != nil { - if room.OwnerID != sp.relay.ID { - slog.Error("Cannot push a stream to non-owned room", "room", room.Name, "owner_id", room.OwnerID) + // Respond with an OK with the room name + resMsg, err := common.CreateMessage( + &gen.ProtoServerPushStream{ + RoomName: pushMsg.RoomName, + }, + "push-stream-ok", nil, + ) + if err != nil { + slog.Error("Failed to create proto message", "err", err) continue } - if room.IsOnline() { - slog.Error("Cannot push a stream to already online room", "room", room.Name) + if err = safeBRW.SendProto(resMsg); err != nil { + slog.Error("Failed to send push stream OK response", "room", room.Name, "err", err) continue } } else { - // Create a new room if it doesn't exist - room = sp.relay.CreateRoom(roomName) - } - - // Respond with an OK with the room name - roomData, err := json.Marshal(room.Name) - if err != nil { - slog.Error("Failed to marshal room name for push stream response", "err", err) - continue - } - if err = safeBRW.SendJSON(connections.NewMessageRaw( - "push-stream-ok", - roomData, - )); err != nil { - slog.Error("Failed to send push stream OK response", "room", room.Name, "err", err) - continue + slog.Error("Failed to GetServerPushStream in push-stream-room") } case "ice-candidate": - var iceMsg connections.MessageICE - if err = json.Unmarshal(data, &iceMsg); err != nil { - slog.Error("Failed to unmarshal ICE candidate from data", "err", err) - continue - } - if conn, ok := sp.incomingConns.Get(room.Name); ok && conn.pc.RemoteDescription() != nil { - if err = conn.pc.AddICECandidate(iceMsg.Candidate); err != nil { - slog.Error("Failed to add ICE candidate for pushed stream", "err", err) + iceMsg := msgWrapper.GetIce() + if iceMsg != nil { + smollified := uint16(*iceMsg.Candidate.SdpMLineIndex) + cand := webrtc.ICECandidateInit{ + Candidate: iceMsg.Candidate.Candidate, + SDPMid: iceMsg.Candidate.SdpMid, + SDPMLineIndex: &smollified, + UsernameFragment: iceMsg.Candidate.UsernameFragment, } - for _, heldIce := range iceHolder { - if err := conn.pc.AddICECandidate(heldIce); err != nil { - slog.Error("Failed to add held ICE candidate for pushed stream", "err", err) + if conn, ok := sp.incomingConns.Get(room.Name); ok && conn.pc.RemoteDescription() != nil { + if err = conn.pc.AddICECandidate(cand); err != nil { + slog.Error("Failed to add ICE candidate for pushed stream", "err", err) + } + for _, heldIce := range iceHolder { + if err := conn.pc.AddICECandidate(heldIce); err != nil { + slog.Error("Failed to add held ICE candidate for pushed stream", "err", err) + } } + // Clear the held candidates + iceHolder = make([]webrtc.ICECandidateInit, 0) + } else { + // Hold the candidate until remote description is set + iceHolder = append(iceHolder, cand) } - // Clear the held candidates - iceHolder = make([]webrtc.ICECandidateInit, 0) } else { - // Hold the candidate until remote description is set - iceHolder = append(iceHolder, iceMsg.Candidate) + slog.Error("Failed to GetIce in pushed stream ice-candidate") } case "offer": // Make sure we have room set to push to (set by "push-stream-room") @@ -567,158 +626,177 @@ func (sp *StreamProtocol) handleStreamPush(stream network.Stream) { continue } - var offerMsg connections.MessageSDP - if err = json.Unmarshal(data, &offerMsg); err != nil { - slog.Error("Failed to unmarshal offer from data", "err", err) - continue - } - - // Create PeerConnection for the incoming stream - pc, err := common.CreatePeerConnection(func() { - slog.Info("PeerConnection closed for pushed stream", "room", room.Name) - // Cleanup the stream connection - if ok := sp.incomingConns.Has(room.Name); ok { - sp.incomingConns.Delete(room.Name) + offerMsg := msgWrapper.GetSdp() + if offerMsg != nil { + offSdp := webrtc.SessionDescription{ + SDP: offerMsg.Sdp.Sdp, + Type: webrtc.NewSDPType(offerMsg.Sdp.Type), } - }) - if err != nil { - slog.Error("Failed to create PeerConnection for pushed stream", "room", room.Name, "err", err) - continue - } - - pc.OnDataChannel(func(dc *webrtc.DataChannel) { - // TODO: Is this the best way to handle DataChannel? Should we just use the map directly? - room.DataChannel = connections.NewNestriDataChannel(dc) - room.DataChannel.RegisterOnOpen(func() { - slog.Debug("DataChannel opened for pushed stream", "room", room.Name) - }) - room.DataChannel.RegisterOnClose(func() { - slog.Debug("DataChannel closed for pushed stream", "room", room.Name) + // Create PeerConnection for the incoming stream + pc, err := common.CreatePeerConnection(func() { + slog.Info("PeerConnection closed for pushed stream", "room", room.Name) + // Cleanup the stream connection + if ok := sp.incomingConns.Has(room.Name); ok { + sp.incomingConns.Delete(room.Name) + } }) - room.DataChannel.RegisterMessageCallback("input", func(data []byte) { - if room.DataChannel != nil { - // Pass to servedConns DataChannels for this specific room + if err != nil { + slog.Error("Failed to create PeerConnection for pushed stream", "room", room.Name, "err", err) + continue + } + + pc.OnDataChannel(func(dc *webrtc.DataChannel) { + // TODO: Is this the best way to handle DataChannel? Should we just use the map directly? + room.DataChannel = connections.NewNestriDataChannel(dc) + room.DataChannel.RegisterOnOpen(func() { + slog.Debug("DataChannel opened for pushed stream", "room", room.Name) + }) + room.DataChannel.RegisterOnClose(func() { + slog.Debug("DataChannel closed for pushed stream", "room", room.Name) + }) + // Handle controller feedback reverse-flow (like rumble events coming from game to client) + room.DataChannel.RegisterMessageCallback("controllerInput", func(data []byte) { + // Forward controller input to all viewers if roomMap, ok := sp.servedConns.Get(room.Name); ok { roomMap.Range(func(peerID peer.ID, conn *StreamConnection) bool { if conn.ndc != nil { if err = conn.ndc.SendBinary(data); err != nil { - slog.Error("Failed to forward input message from pushed stream to viewer", "room", room.Name, "peer", peerID, "err", err) + slog.Error("Failed to forward controller input from pushed stream to viewer", "room", room.Name, "peer", peerID, "err", err) } } - return true // Continue iteration + return true }) } - } - }) - - // Set the DataChannel in the incomingConns map - if conn, ok := sp.incomingConns.Get(room.Name); ok { - conn.ndc = room.DataChannel - } else { - sp.incomingConns.Set(room.Name, &StreamConnection{ - pc: pc, - ndc: room.DataChannel, }) - } - }) - pc.OnICECandidate(func(candidate *webrtc.ICECandidate) { - if candidate == nil { - return - } + // Set the DataChannel in the incomingConns map + if conn, ok := sp.incomingConns.Get(room.Name); ok { + conn.ndc = room.DataChannel + } else { + sp.incomingConns.Set(room.Name, &StreamConnection{ + pc: pc, + ndc: room.DataChannel, + }) + } + }) - if err = safeBRW.SendJSON(connections.NewMessageICE( - "ice-candidate", - candidate.ToJSON(), - )); err != nil { - slog.Error("Failed to send ICE candidate message for pushed stream", "room", room.Name, "err", err) - return - } - }) + pc.OnICECandidate(func(candidate *webrtc.ICECandidate) { + if candidate == nil { + return + } - pc.OnTrack(func(remoteTrack *webrtc.TrackRemote, receiver *webrtc.RTPReceiver) { - localTrack, err := webrtc.NewTrackLocalStaticRTP(remoteTrack.Codec().RTPCodecCapability, remoteTrack.Kind().String(), fmt.Sprintf("nestri-%s-%s", room.Name, remoteTrack.Kind().String())) - if err != nil { - slog.Error("Failed to create local track for pushed stream", "room", room.Name, "track_kind", remoteTrack.Kind().String(), "err", err) - return - } + candInit := candidate.ToJSON() + biggified := uint32(*candInit.SDPMLineIndex) + iceMsg, err := common.CreateMessage( + &gen.ProtoICE{ + Candidate: &gen.RTCIceCandidateInit{ + Candidate: candInit.Candidate, + SdpMLineIndex: &biggified, + SdpMid: candInit.SDPMid, + }, + }, + "ice-candidate", nil, + ) + if err != nil { + slog.Error("Failed to create proto message", "err", err) + return + } + if err = safeBRW.SendProto(iceMsg); err != nil { + slog.Error("Failed to send ICE candidate message for pushed stream", "room", room.Name, "err", err) + return + } + }) - slog.Debug("Received track for pushed stream", "room", room.Name, "track_kind", remoteTrack.Kind().String()) + pc.OnTrack(func(remoteTrack *webrtc.TrackRemote, receiver *webrtc.RTPReceiver) { + localTrack, err := webrtc.NewTrackLocalStaticRTP(remoteTrack.Codec().RTPCodecCapability, remoteTrack.Kind().String(), fmt.Sprintf("nestri-%s-%s", room.Name, remoteTrack.Kind().String())) + if err != nil { + slog.Error("Failed to create local track for pushed stream", "room", room.Name, "track_kind", remoteTrack.Kind().String(), "err", err) + return + } - // Set track for Room - room.SetTrack(remoteTrack.Kind(), localTrack) + slog.Debug("Received track for pushed stream", "room", room.Name, "track_kind", remoteTrack.Kind().String()) - // Prepare PlayoutDelayExtension so we don't need to recreate it for each packet - playoutExt := &rtp.PlayoutDelayExtension{ - MinDelay: 0, - MaxDelay: 0, - } - playoutPayload, err := playoutExt.Marshal() - if err != nil { - slog.Error("Failed to marshal PlayoutDelayExtension for room", "room", room.Name, "err", err) - return - } + // Set track for Room + room.SetTrack(remoteTrack.Kind(), localTrack) - for { - rtpPacket, _, err := remoteTrack.ReadRTP() + // Prepare PlayoutDelayExtension so we don't need to recreate it for each packet + playoutExt := &rtp.PlayoutDelayExtension{ + MinDelay: 0, + MaxDelay: 0, + } + playoutPayload, err := playoutExt.Marshal() if err != nil { - if !errors.Is(err, io.EOF) { - slog.Error("Failed to read RTP from remote track for room", "room", room.Name, "err", err) - } - break + slog.Error("Failed to marshal PlayoutDelayExtension for room", "room", room.Name, "err", err) + return } - // Use PlayoutDelayExtension for low latency, if set for this track kind - if extID, ok := common.GetExtension(remoteTrack.Kind(), common.ExtensionPlayoutDelay); ok { - if err := rtpPacket.SetExtension(extID, playoutPayload); err != nil { - slog.Error("Failed to set PlayoutDelayExtension for room", "room", room.Name, "err", err) - continue + for { + rtpPacket, _, err := remoteTrack.ReadRTP() + if err != nil { + if !errors.Is(err, io.EOF) { + slog.Error("Failed to read RTP from remote track for room", "room", room.Name, "err", err) + } + break } - } - err = localTrack.WriteRTP(rtpPacket) - if err != nil && !errors.Is(err, io.ErrClosedPipe) { - slog.Error("Failed to write RTP to local track for room", "room", room.Name, "err", err) - break + // Use PlayoutDelayExtension for low latency, if set for this track kind + if extID, ok := common.GetExtension(remoteTrack.Kind(), common.ExtensionPlayoutDelay); ok { + if err := rtpPacket.SetExtension(extID, playoutPayload); err != nil { + slog.Error("Failed to set PlayoutDelayExtension for room", "room", room.Name, "err", err) + continue + } + } + + room.BroadcastPacket(remoteTrack.Kind(), rtpPacket) } - } - slog.Debug("Track closed for room", "room", room.Name, "track_kind", remoteTrack.Kind().String()) + slog.Debug("Track closed for room", "room", room.Name, "track_kind", remoteTrack.Kind().String()) - // Cleanup the track from the room - room.SetTrack(remoteTrack.Kind(), nil) - }) + // Cleanup the track from the room + room.SetTrack(remoteTrack.Kind(), nil) + }) - // Set the remote description - if err = pc.SetRemoteDescription(offerMsg.SDP); err != nil { - slog.Error("Failed to set remote description for pushed stream", "room", room.Name, "err", err) - continue - } - slog.Debug("Set remote description for pushed stream", "room", room.Name) + // Set the remote description + if err = pc.SetRemoteDescription(offSdp); err != nil { + slog.Error("Failed to set remote description for pushed stream", "room", room.Name, "err", err) + continue + } + slog.Debug("Set remote description for pushed stream", "room", room.Name) - // Create an answer - answer, err := pc.CreateAnswer(nil) - if err != nil { - slog.Error("Failed to create answer for pushed stream", "room", room.Name, "err", err) - continue - } - if err = pc.SetLocalDescription(answer); err != nil { - slog.Error("Failed to set local description for pushed stream", "room", room.Name, "err", err) - continue - } - if err = safeBRW.SendJSON(connections.NewMessageSDP( - "answer", - answer, - )); err != nil { - slog.Error("Failed to send answer for pushed stream", "room", room.Name, "err", err) - } + // Create an answer + answer, err := pc.CreateAnswer(nil) + if err != nil { + slog.Error("Failed to create answer for pushed stream", "room", room.Name, "err", err) + continue + } + if err = pc.SetLocalDescription(answer); err != nil { + slog.Error("Failed to set local description for pushed stream", "room", room.Name, "err", err) + continue + } + answerMsg, err := common.CreateMessage( + &gen.ProtoSDP{ + Sdp: &gen.RTCSessionDescriptionInit{ + Sdp: answer.SDP, + Type: answer.Type.String(), + }, + }, + "answer", nil, + ) + if err != nil { + slog.Error("Failed to create proto message", "err", err) + continue + } + if err = safeBRW.SendProto(answerMsg); err != nil { + slog.Error("Failed to send answer for pushed stream", "room", room.Name, "err", err) + } - // Store the connection - sp.incomingConns.Set(room.Name, &StreamConnection{ - pc: pc, - ndc: room.DataChannel, // if it exists, if not it will be set later - }) - slog.Debug("Sent answer for pushed stream", "room", room.Name) + // Store the connection + sp.incomingConns.Set(room.Name, &StreamConnection{ + pc: pc, + ndc: room.DataChannel, // if it exists, if not it will be set later + }) + slog.Debug("Sent answer for pushed stream", "room", room.Name) + } } } } @@ -727,10 +805,10 @@ func (sp *StreamProtocol) handleStreamPush(stream network.Stream) { // RequestStream sends a request to get room stream from another relay func (sp *StreamProtocol) RequestStream(ctx context.Context, room *shared.Room, peerID peer.ID) error { - stream, err := sp.relay.Host.NewStream(ctx, peerID, protocolStreamRequest) + _, err := sp.relay.Host.NewStream(ctx, peerID, protocolStreamRequest) if err != nil { return fmt.Errorf("failed to create stream: %w", err) } - return sp.requestStream(stream, room) + return nil /* TODO: This? */ } diff --git a/packages/relay/internal/core/state.go b/packages/relay/internal/core/state.go index 47dff042..d4d088c2 100644 --- a/packages/relay/internal/core/state.go +++ b/packages/relay/internal/core/state.go @@ -5,9 +5,14 @@ import ( "encoding/json" "errors" "log/slog" + "relay/internal/common" "relay/internal/shared" "time" + gen "relay/internal/proto" + + "google.golang.org/protobuf/proto" + pubsub "github.com/libp2p/go-libp2p-pubsub" "github.com/libp2p/go-libp2p/core/network" "github.com/libp2p/go-libp2p/core/peer" @@ -129,12 +134,51 @@ func (r *Relay) onPeerConnected(peerID peer.ID) { // onPeerDisconnected marks a peer as disconnected in our status view and removes latency info func (r *Relay) onPeerDisconnected(peerID peer.ID) { + // Check if this was a client session disconnect + if session, ok := r.ClientSessions.Get(peerID); ok { + slog.Info("Client session disconnected", + "peer", peerID, + "session", session.SessionID, + "room", session.RoomName, + "controller_slots", session.ControllerSlots) + + // Send cleanup message to nestri-server if client had active controllers + if len(session.ControllerSlots) > 0 { + room := r.GetRoomByName(session.RoomName) + if room != nil && room.DataChannel != nil { + // Create disconnect notification + disconnectMsg, err := common.CreateMessage(&gen.ProtoClientDisconnected{ + SessionId: session.SessionID, + ControllerSlots: session.ControllerSlots, + }, "client-disconnected", nil) + if err != nil { + slog.Error("Failed to create client disconnect message", "err", err) + } + + disMarshal, err := proto.Marshal(disconnectMsg) + if err != nil { + slog.Error("Failed to marshal client disconnect message", "err", err) + } else { + if err = room.DataChannel.SendBinary(disMarshal); err != nil { + slog.Error("Failed to send client disconnect notification", "err", err) + } else { + slog.Info("Sent controller cleanup notification to nestri-server", + "session", session.SessionID, + "slots", session.ControllerSlots) + } + } + } + } + + r.ClientSessions.Delete(peerID) + return + } + + // Relay peer disconnect handling slog.Info("Mesh peer disconnected, deleting from local peer map", "peer", peerID) - // Remove peer from local mesh peers if r.Peers.Has(peerID) { r.Peers.Delete(peerID) } - // Remove any rooms associated with this peer if r.Rooms.Has(peerID.String()) { r.Rooms.Delete(peerID.String()) } diff --git a/packages/relay/internal/proto/messages.pb.go b/packages/relay/internal/proto/messages.pb.go index de708ff7..fb9a2f6e 100644 --- a/packages/relay/internal/proto/messages.pb.go +++ b/packages/relay/internal/proto/messages.pb.go @@ -73,28 +73,50 @@ func (x *ProtoMessageBase) GetLatency() *ProtoLatencyTracker { return nil } -type ProtoMessageInput struct { - state protoimpl.MessageState `protogen:"open.v1"` - MessageBase *ProtoMessageBase `protobuf:"bytes,1,opt,name=message_base,json=messageBase,proto3" json:"message_base,omitempty"` - Data *ProtoInput `protobuf:"bytes,2,opt,name=data,proto3" json:"data,omitempty"` +type ProtoMessage struct { + state protoimpl.MessageState `protogen:"open.v1"` + MessageBase *ProtoMessageBase `protobuf:"bytes,1,opt,name=message_base,json=messageBase,proto3" json:"message_base,omitempty"` + // Types that are valid to be assigned to Payload: + // + // *ProtoMessage_MouseMove + // *ProtoMessage_MouseMoveAbs + // *ProtoMessage_MouseWheel + // *ProtoMessage_MouseKeyDown + // *ProtoMessage_MouseKeyUp + // *ProtoMessage_KeyDown + // *ProtoMessage_KeyUp + // *ProtoMessage_ControllerAttach + // *ProtoMessage_ControllerDetach + // *ProtoMessage_ControllerButton + // *ProtoMessage_ControllerTrigger + // *ProtoMessage_ControllerStick + // *ProtoMessage_ControllerAxis + // *ProtoMessage_ControllerRumble + // *ProtoMessage_Ice + // *ProtoMessage_Sdp + // *ProtoMessage_Raw + // *ProtoMessage_ClientRequestRoomStream + // *ProtoMessage_ClientDisconnected + // *ProtoMessage_ServerPushStream + Payload isProtoMessage_Payload `protobuf_oneof:"payload"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } -func (x *ProtoMessageInput) Reset() { - *x = ProtoMessageInput{} +func (x *ProtoMessage) Reset() { + *x = ProtoMessage{} mi := &file_messages_proto_msgTypes[1] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } -func (x *ProtoMessageInput) String() string { +func (x *ProtoMessage) String() string { return protoimpl.X.MessageStringOf(x) } -func (*ProtoMessageInput) ProtoMessage() {} +func (*ProtoMessage) ProtoMessage() {} -func (x *ProtoMessageInput) ProtoReflect() protoreflect.Message { +func (x *ProtoMessage) ProtoReflect() protoreflect.Message { mi := &file_messages_proto_msgTypes[1] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) @@ -106,25 +128,331 @@ func (x *ProtoMessageInput) ProtoReflect() protoreflect.Message { return mi.MessageOf(x) } -// Deprecated: Use ProtoMessageInput.ProtoReflect.Descriptor instead. -func (*ProtoMessageInput) Descriptor() ([]byte, []int) { +// Deprecated: Use ProtoMessage.ProtoReflect.Descriptor instead. +func (*ProtoMessage) Descriptor() ([]byte, []int) { return file_messages_proto_rawDescGZIP(), []int{1} } -func (x *ProtoMessageInput) GetMessageBase() *ProtoMessageBase { +func (x *ProtoMessage) GetMessageBase() *ProtoMessageBase { if x != nil { return x.MessageBase } return nil } -func (x *ProtoMessageInput) GetData() *ProtoInput { +func (x *ProtoMessage) GetPayload() isProtoMessage_Payload { + if x != nil { + return x.Payload + } + return nil +} + +func (x *ProtoMessage) GetMouseMove() *ProtoMouseMove { + if x != nil { + if x, ok := x.Payload.(*ProtoMessage_MouseMove); ok { + return x.MouseMove + } + } + return nil +} + +func (x *ProtoMessage) GetMouseMoveAbs() *ProtoMouseMoveAbs { + if x != nil { + if x, ok := x.Payload.(*ProtoMessage_MouseMoveAbs); ok { + return x.MouseMoveAbs + } + } + return nil +} + +func (x *ProtoMessage) GetMouseWheel() *ProtoMouseWheel { + if x != nil { + if x, ok := x.Payload.(*ProtoMessage_MouseWheel); ok { + return x.MouseWheel + } + } + return nil +} + +func (x *ProtoMessage) GetMouseKeyDown() *ProtoMouseKeyDown { + if x != nil { + if x, ok := x.Payload.(*ProtoMessage_MouseKeyDown); ok { + return x.MouseKeyDown + } + } + return nil +} + +func (x *ProtoMessage) GetMouseKeyUp() *ProtoMouseKeyUp { + if x != nil { + if x, ok := x.Payload.(*ProtoMessage_MouseKeyUp); ok { + return x.MouseKeyUp + } + } + return nil +} + +func (x *ProtoMessage) GetKeyDown() *ProtoKeyDown { + if x != nil { + if x, ok := x.Payload.(*ProtoMessage_KeyDown); ok { + return x.KeyDown + } + } + return nil +} + +func (x *ProtoMessage) GetKeyUp() *ProtoKeyUp { + if x != nil { + if x, ok := x.Payload.(*ProtoMessage_KeyUp); ok { + return x.KeyUp + } + } + return nil +} + +func (x *ProtoMessage) GetControllerAttach() *ProtoControllerAttach { + if x != nil { + if x, ok := x.Payload.(*ProtoMessage_ControllerAttach); ok { + return x.ControllerAttach + } + } + return nil +} + +func (x *ProtoMessage) GetControllerDetach() *ProtoControllerDetach { + if x != nil { + if x, ok := x.Payload.(*ProtoMessage_ControllerDetach); ok { + return x.ControllerDetach + } + } + return nil +} + +func (x *ProtoMessage) GetControllerButton() *ProtoControllerButton { + if x != nil { + if x, ok := x.Payload.(*ProtoMessage_ControllerButton); ok { + return x.ControllerButton + } + } + return nil +} + +func (x *ProtoMessage) GetControllerTrigger() *ProtoControllerTrigger { + if x != nil { + if x, ok := x.Payload.(*ProtoMessage_ControllerTrigger); ok { + return x.ControllerTrigger + } + } + return nil +} + +func (x *ProtoMessage) GetControllerStick() *ProtoControllerStick { + if x != nil { + if x, ok := x.Payload.(*ProtoMessage_ControllerStick); ok { + return x.ControllerStick + } + } + return nil +} + +func (x *ProtoMessage) GetControllerAxis() *ProtoControllerAxis { + if x != nil { + if x, ok := x.Payload.(*ProtoMessage_ControllerAxis); ok { + return x.ControllerAxis + } + } + return nil +} + +func (x *ProtoMessage) GetControllerRumble() *ProtoControllerRumble { + if x != nil { + if x, ok := x.Payload.(*ProtoMessage_ControllerRumble); ok { + return x.ControllerRumble + } + } + return nil +} + +func (x *ProtoMessage) GetIce() *ProtoICE { + if x != nil { + if x, ok := x.Payload.(*ProtoMessage_Ice); ok { + return x.Ice + } + } + return nil +} + +func (x *ProtoMessage) GetSdp() *ProtoSDP { + if x != nil { + if x, ok := x.Payload.(*ProtoMessage_Sdp); ok { + return x.Sdp + } + } + return nil +} + +func (x *ProtoMessage) GetRaw() *ProtoRaw { + if x != nil { + if x, ok := x.Payload.(*ProtoMessage_Raw); ok { + return x.Raw + } + } + return nil +} + +func (x *ProtoMessage) GetClientRequestRoomStream() *ProtoClientRequestRoomStream { + if x != nil { + if x, ok := x.Payload.(*ProtoMessage_ClientRequestRoomStream); ok { + return x.ClientRequestRoomStream + } + } + return nil +} + +func (x *ProtoMessage) GetClientDisconnected() *ProtoClientDisconnected { if x != nil { - return x.Data + if x, ok := x.Payload.(*ProtoMessage_ClientDisconnected); ok { + return x.ClientDisconnected + } } return nil } +func (x *ProtoMessage) GetServerPushStream() *ProtoServerPushStream { + if x != nil { + if x, ok := x.Payload.(*ProtoMessage_ServerPushStream); ok { + return x.ServerPushStream + } + } + return nil +} + +type isProtoMessage_Payload interface { + isProtoMessage_Payload() +} + +type ProtoMessage_MouseMove struct { + // Input types + MouseMove *ProtoMouseMove `protobuf:"bytes,2,opt,name=mouse_move,json=mouseMove,proto3,oneof"` +} + +type ProtoMessage_MouseMoveAbs struct { + MouseMoveAbs *ProtoMouseMoveAbs `protobuf:"bytes,3,opt,name=mouse_move_abs,json=mouseMoveAbs,proto3,oneof"` +} + +type ProtoMessage_MouseWheel struct { + MouseWheel *ProtoMouseWheel `protobuf:"bytes,4,opt,name=mouse_wheel,json=mouseWheel,proto3,oneof"` +} + +type ProtoMessage_MouseKeyDown struct { + MouseKeyDown *ProtoMouseKeyDown `protobuf:"bytes,5,opt,name=mouse_key_down,json=mouseKeyDown,proto3,oneof"` +} + +type ProtoMessage_MouseKeyUp struct { + MouseKeyUp *ProtoMouseKeyUp `protobuf:"bytes,6,opt,name=mouse_key_up,json=mouseKeyUp,proto3,oneof"` +} + +type ProtoMessage_KeyDown struct { + KeyDown *ProtoKeyDown `protobuf:"bytes,7,opt,name=key_down,json=keyDown,proto3,oneof"` +} + +type ProtoMessage_KeyUp struct { + KeyUp *ProtoKeyUp `protobuf:"bytes,8,opt,name=key_up,json=keyUp,proto3,oneof"` +} + +type ProtoMessage_ControllerAttach struct { + ControllerAttach *ProtoControllerAttach `protobuf:"bytes,9,opt,name=controller_attach,json=controllerAttach,proto3,oneof"` +} + +type ProtoMessage_ControllerDetach struct { + ControllerDetach *ProtoControllerDetach `protobuf:"bytes,10,opt,name=controller_detach,json=controllerDetach,proto3,oneof"` +} + +type ProtoMessage_ControllerButton struct { + ControllerButton *ProtoControllerButton `protobuf:"bytes,11,opt,name=controller_button,json=controllerButton,proto3,oneof"` +} + +type ProtoMessage_ControllerTrigger struct { + ControllerTrigger *ProtoControllerTrigger `protobuf:"bytes,12,opt,name=controller_trigger,json=controllerTrigger,proto3,oneof"` +} + +type ProtoMessage_ControllerStick struct { + ControllerStick *ProtoControllerStick `protobuf:"bytes,13,opt,name=controller_stick,json=controllerStick,proto3,oneof"` +} + +type ProtoMessage_ControllerAxis struct { + ControllerAxis *ProtoControllerAxis `protobuf:"bytes,14,opt,name=controller_axis,json=controllerAxis,proto3,oneof"` +} + +type ProtoMessage_ControllerRumble struct { + ControllerRumble *ProtoControllerRumble `protobuf:"bytes,15,opt,name=controller_rumble,json=controllerRumble,proto3,oneof"` +} + +type ProtoMessage_Ice struct { + // Signaling types + Ice *ProtoICE `protobuf:"bytes,20,opt,name=ice,proto3,oneof"` +} + +type ProtoMessage_Sdp struct { + Sdp *ProtoSDP `protobuf:"bytes,21,opt,name=sdp,proto3,oneof"` +} + +type ProtoMessage_Raw struct { + Raw *ProtoRaw `protobuf:"bytes,22,opt,name=raw,proto3,oneof"` +} + +type ProtoMessage_ClientRequestRoomStream struct { + ClientRequestRoomStream *ProtoClientRequestRoomStream `protobuf:"bytes,23,opt,name=client_request_room_stream,json=clientRequestRoomStream,proto3,oneof"` +} + +type ProtoMessage_ClientDisconnected struct { + ClientDisconnected *ProtoClientDisconnected `protobuf:"bytes,24,opt,name=client_disconnected,json=clientDisconnected,proto3,oneof"` +} + +type ProtoMessage_ServerPushStream struct { + ServerPushStream *ProtoServerPushStream `protobuf:"bytes,25,opt,name=server_push_stream,json=serverPushStream,proto3,oneof"` +} + +func (*ProtoMessage_MouseMove) isProtoMessage_Payload() {} + +func (*ProtoMessage_MouseMoveAbs) isProtoMessage_Payload() {} + +func (*ProtoMessage_MouseWheel) isProtoMessage_Payload() {} + +func (*ProtoMessage_MouseKeyDown) isProtoMessage_Payload() {} + +func (*ProtoMessage_MouseKeyUp) isProtoMessage_Payload() {} + +func (*ProtoMessage_KeyDown) isProtoMessage_Payload() {} + +func (*ProtoMessage_KeyUp) isProtoMessage_Payload() {} + +func (*ProtoMessage_ControllerAttach) isProtoMessage_Payload() {} + +func (*ProtoMessage_ControllerDetach) isProtoMessage_Payload() {} + +func (*ProtoMessage_ControllerButton) isProtoMessage_Payload() {} + +func (*ProtoMessage_ControllerTrigger) isProtoMessage_Payload() {} + +func (*ProtoMessage_ControllerStick) isProtoMessage_Payload() {} + +func (*ProtoMessage_ControllerAxis) isProtoMessage_Payload() {} + +func (*ProtoMessage_ControllerRumble) isProtoMessage_Payload() {} + +func (*ProtoMessage_Ice) isProtoMessage_Payload() {} + +func (*ProtoMessage_Sdp) isProtoMessage_Payload() {} + +func (*ProtoMessage_Raw) isProtoMessage_Payload() {} + +func (*ProtoMessage_ClientRequestRoomStream) isProtoMessage_Payload() {} + +func (*ProtoMessage_ClientDisconnected) isProtoMessage_Payload() {} + +func (*ProtoMessage_ServerPushStream) isProtoMessage_Payload() {} + var File_messages_proto protoreflect.FileDescriptor const file_messages_proto_rawDesc = "" + @@ -132,10 +460,35 @@ const file_messages_proto_rawDesc = "" + "\x0emessages.proto\x12\x05proto\x1a\vtypes.proto\x1a\x15latency_tracker.proto\"k\n" + "\x10ProtoMessageBase\x12!\n" + "\fpayload_type\x18\x01 \x01(\tR\vpayloadType\x124\n" + - "\alatency\x18\x02 \x01(\v2\x1a.proto.ProtoLatencyTrackerR\alatency\"v\n" + - "\x11ProtoMessageInput\x12:\n" + - "\fmessage_base\x18\x01 \x01(\v2\x17.proto.ProtoMessageBaseR\vmessageBase\x12%\n" + - "\x04data\x18\x02 \x01(\v2\x11.proto.ProtoInputR\x04dataB\x16Z\x14relay/internal/protob\x06proto3" + "\alatency\x18\x02 \x01(\v2\x1a.proto.ProtoLatencyTrackerR\alatency\"\xef\n" + + "\n" + + "\fProtoMessage\x12:\n" + + "\fmessage_base\x18\x01 \x01(\v2\x17.proto.ProtoMessageBaseR\vmessageBase\x126\n" + + "\n" + + "mouse_move\x18\x02 \x01(\v2\x15.proto.ProtoMouseMoveH\x00R\tmouseMove\x12@\n" + + "\x0emouse_move_abs\x18\x03 \x01(\v2\x18.proto.ProtoMouseMoveAbsH\x00R\fmouseMoveAbs\x129\n" + + "\vmouse_wheel\x18\x04 \x01(\v2\x16.proto.ProtoMouseWheelH\x00R\n" + + "mouseWheel\x12@\n" + + "\x0emouse_key_down\x18\x05 \x01(\v2\x18.proto.ProtoMouseKeyDownH\x00R\fmouseKeyDown\x12:\n" + + "\fmouse_key_up\x18\x06 \x01(\v2\x16.proto.ProtoMouseKeyUpH\x00R\n" + + "mouseKeyUp\x120\n" + + "\bkey_down\x18\a \x01(\v2\x13.proto.ProtoKeyDownH\x00R\akeyDown\x12*\n" + + "\x06key_up\x18\b \x01(\v2\x11.proto.ProtoKeyUpH\x00R\x05keyUp\x12K\n" + + "\x11controller_attach\x18\t \x01(\v2\x1c.proto.ProtoControllerAttachH\x00R\x10controllerAttach\x12K\n" + + "\x11controller_detach\x18\n" + + " \x01(\v2\x1c.proto.ProtoControllerDetachH\x00R\x10controllerDetach\x12K\n" + + "\x11controller_button\x18\v \x01(\v2\x1c.proto.ProtoControllerButtonH\x00R\x10controllerButton\x12N\n" + + "\x12controller_trigger\x18\f \x01(\v2\x1d.proto.ProtoControllerTriggerH\x00R\x11controllerTrigger\x12H\n" + + "\x10controller_stick\x18\r \x01(\v2\x1b.proto.ProtoControllerStickH\x00R\x0fcontrollerStick\x12E\n" + + "\x0fcontroller_axis\x18\x0e \x01(\v2\x1a.proto.ProtoControllerAxisH\x00R\x0econtrollerAxis\x12K\n" + + "\x11controller_rumble\x18\x0f \x01(\v2\x1c.proto.ProtoControllerRumbleH\x00R\x10controllerRumble\x12#\n" + + "\x03ice\x18\x14 \x01(\v2\x0f.proto.ProtoICEH\x00R\x03ice\x12#\n" + + "\x03sdp\x18\x15 \x01(\v2\x0f.proto.ProtoSDPH\x00R\x03sdp\x12#\n" + + "\x03raw\x18\x16 \x01(\v2\x0f.proto.ProtoRawH\x00R\x03raw\x12b\n" + + "\x1aclient_request_room_stream\x18\x17 \x01(\v2#.proto.ProtoClientRequestRoomStreamH\x00R\x17clientRequestRoomStream\x12Q\n" + + "\x13client_disconnected\x18\x18 \x01(\v2\x1e.proto.ProtoClientDisconnectedH\x00R\x12clientDisconnected\x12L\n" + + "\x12server_push_stream\x18\x19 \x01(\v2\x1c.proto.ProtoServerPushStreamH\x00R\x10serverPushStreamB\t\n" + + "\apayloadB\x16Z\x14relay/internal/protob\x06proto3" var ( file_messages_proto_rawDescOnce sync.Once @@ -151,20 +504,58 @@ func file_messages_proto_rawDescGZIP() []byte { var file_messages_proto_msgTypes = make([]protoimpl.MessageInfo, 2) var file_messages_proto_goTypes = []any{ - (*ProtoMessageBase)(nil), // 0: proto.ProtoMessageBase - (*ProtoMessageInput)(nil), // 1: proto.ProtoMessageInput - (*ProtoLatencyTracker)(nil), // 2: proto.ProtoLatencyTracker - (*ProtoInput)(nil), // 3: proto.ProtoInput + (*ProtoMessageBase)(nil), // 0: proto.ProtoMessageBase + (*ProtoMessage)(nil), // 1: proto.ProtoMessage + (*ProtoLatencyTracker)(nil), // 2: proto.ProtoLatencyTracker + (*ProtoMouseMove)(nil), // 3: proto.ProtoMouseMove + (*ProtoMouseMoveAbs)(nil), // 4: proto.ProtoMouseMoveAbs + (*ProtoMouseWheel)(nil), // 5: proto.ProtoMouseWheel + (*ProtoMouseKeyDown)(nil), // 6: proto.ProtoMouseKeyDown + (*ProtoMouseKeyUp)(nil), // 7: proto.ProtoMouseKeyUp + (*ProtoKeyDown)(nil), // 8: proto.ProtoKeyDown + (*ProtoKeyUp)(nil), // 9: proto.ProtoKeyUp + (*ProtoControllerAttach)(nil), // 10: proto.ProtoControllerAttach + (*ProtoControllerDetach)(nil), // 11: proto.ProtoControllerDetach + (*ProtoControllerButton)(nil), // 12: proto.ProtoControllerButton + (*ProtoControllerTrigger)(nil), // 13: proto.ProtoControllerTrigger + (*ProtoControllerStick)(nil), // 14: proto.ProtoControllerStick + (*ProtoControllerAxis)(nil), // 15: proto.ProtoControllerAxis + (*ProtoControllerRumble)(nil), // 16: proto.ProtoControllerRumble + (*ProtoICE)(nil), // 17: proto.ProtoICE + (*ProtoSDP)(nil), // 18: proto.ProtoSDP + (*ProtoRaw)(nil), // 19: proto.ProtoRaw + (*ProtoClientRequestRoomStream)(nil), // 20: proto.ProtoClientRequestRoomStream + (*ProtoClientDisconnected)(nil), // 21: proto.ProtoClientDisconnected + (*ProtoServerPushStream)(nil), // 22: proto.ProtoServerPushStream } var file_messages_proto_depIdxs = []int32{ - 2, // 0: proto.ProtoMessageBase.latency:type_name -> proto.ProtoLatencyTracker - 0, // 1: proto.ProtoMessageInput.message_base:type_name -> proto.ProtoMessageBase - 3, // 2: proto.ProtoMessageInput.data:type_name -> proto.ProtoInput - 3, // [3:3] is the sub-list for method output_type - 3, // [3:3] is the sub-list for method input_type - 3, // [3:3] is the sub-list for extension type_name - 3, // [3:3] is the sub-list for extension extendee - 0, // [0:3] is the sub-list for field type_name + 2, // 0: proto.ProtoMessageBase.latency:type_name -> proto.ProtoLatencyTracker + 0, // 1: proto.ProtoMessage.message_base:type_name -> proto.ProtoMessageBase + 3, // 2: proto.ProtoMessage.mouse_move:type_name -> proto.ProtoMouseMove + 4, // 3: proto.ProtoMessage.mouse_move_abs:type_name -> proto.ProtoMouseMoveAbs + 5, // 4: proto.ProtoMessage.mouse_wheel:type_name -> proto.ProtoMouseWheel + 6, // 5: proto.ProtoMessage.mouse_key_down:type_name -> proto.ProtoMouseKeyDown + 7, // 6: proto.ProtoMessage.mouse_key_up:type_name -> proto.ProtoMouseKeyUp + 8, // 7: proto.ProtoMessage.key_down:type_name -> proto.ProtoKeyDown + 9, // 8: proto.ProtoMessage.key_up:type_name -> proto.ProtoKeyUp + 10, // 9: proto.ProtoMessage.controller_attach:type_name -> proto.ProtoControllerAttach + 11, // 10: proto.ProtoMessage.controller_detach:type_name -> proto.ProtoControllerDetach + 12, // 11: proto.ProtoMessage.controller_button:type_name -> proto.ProtoControllerButton + 13, // 12: proto.ProtoMessage.controller_trigger:type_name -> proto.ProtoControllerTrigger + 14, // 13: proto.ProtoMessage.controller_stick:type_name -> proto.ProtoControllerStick + 15, // 14: proto.ProtoMessage.controller_axis:type_name -> proto.ProtoControllerAxis + 16, // 15: proto.ProtoMessage.controller_rumble:type_name -> proto.ProtoControllerRumble + 17, // 16: proto.ProtoMessage.ice:type_name -> proto.ProtoICE + 18, // 17: proto.ProtoMessage.sdp:type_name -> proto.ProtoSDP + 19, // 18: proto.ProtoMessage.raw:type_name -> proto.ProtoRaw + 20, // 19: proto.ProtoMessage.client_request_room_stream:type_name -> proto.ProtoClientRequestRoomStream + 21, // 20: proto.ProtoMessage.client_disconnected:type_name -> proto.ProtoClientDisconnected + 22, // 21: proto.ProtoMessage.server_push_stream:type_name -> proto.ProtoServerPushStream + 22, // [22:22] is the sub-list for method output_type + 22, // [22:22] is the sub-list for method input_type + 22, // [22:22] is the sub-list for extension type_name + 22, // [22:22] is the sub-list for extension extendee + 0, // [0:22] is the sub-list for field type_name } func init() { file_messages_proto_init() } @@ -174,6 +565,28 @@ func file_messages_proto_init() { } file_types_proto_init() file_latency_tracker_proto_init() + file_messages_proto_msgTypes[1].OneofWrappers = []any{ + (*ProtoMessage_MouseMove)(nil), + (*ProtoMessage_MouseMoveAbs)(nil), + (*ProtoMessage_MouseWheel)(nil), + (*ProtoMessage_MouseKeyDown)(nil), + (*ProtoMessage_MouseKeyUp)(nil), + (*ProtoMessage_KeyDown)(nil), + (*ProtoMessage_KeyUp)(nil), + (*ProtoMessage_ControllerAttach)(nil), + (*ProtoMessage_ControllerDetach)(nil), + (*ProtoMessage_ControllerButton)(nil), + (*ProtoMessage_ControllerTrigger)(nil), + (*ProtoMessage_ControllerStick)(nil), + (*ProtoMessage_ControllerAxis)(nil), + (*ProtoMessage_ControllerRumble)(nil), + (*ProtoMessage_Ice)(nil), + (*ProtoMessage_Sdp)(nil), + (*ProtoMessage_Raw)(nil), + (*ProtoMessage_ClientRequestRoomStream)(nil), + (*ProtoMessage_ClientDisconnected)(nil), + (*ProtoMessage_ServerPushStream)(nil), + } type x struct{} out := protoimpl.TypeBuilder{ File: protoimpl.DescBuilder{ diff --git a/packages/relay/internal/proto/types.pb.go b/packages/relay/internal/proto/types.pb.go index d4ecdecc..dae068ac 100644 --- a/packages/relay/internal/proto/types.pb.go +++ b/packages/relay/internal/proto/types.pb.go @@ -24,9 +24,8 @@ const ( // MouseMove message type ProtoMouseMove struct { state protoimpl.MessageState `protogen:"open.v1"` - Type string `protobuf:"bytes,1,opt,name=type,proto3" json:"type,omitempty"` // Fixed value "MouseMove" - X int32 `protobuf:"varint,2,opt,name=x,proto3" json:"x,omitempty"` - Y int32 `protobuf:"varint,3,opt,name=y,proto3" json:"y,omitempty"` + X int32 `protobuf:"varint,1,opt,name=x,proto3" json:"x,omitempty"` + Y int32 `protobuf:"varint,2,opt,name=y,proto3" json:"y,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -61,13 +60,6 @@ func (*ProtoMouseMove) Descriptor() ([]byte, []int) { return file_types_proto_rawDescGZIP(), []int{0} } -func (x *ProtoMouseMove) GetType() string { - if x != nil { - return x.Type - } - return "" -} - func (x *ProtoMouseMove) GetX() int32 { if x != nil { return x.X @@ -85,9 +77,8 @@ func (x *ProtoMouseMove) GetY() int32 { // MouseMoveAbs message type ProtoMouseMoveAbs struct { state protoimpl.MessageState `protogen:"open.v1"` - Type string `protobuf:"bytes,1,opt,name=type,proto3" json:"type,omitempty"` // Fixed value "MouseMoveAbs" - X int32 `protobuf:"varint,2,opt,name=x,proto3" json:"x,omitempty"` - Y int32 `protobuf:"varint,3,opt,name=y,proto3" json:"y,omitempty"` + X int32 `protobuf:"varint,1,opt,name=x,proto3" json:"x,omitempty"` + Y int32 `protobuf:"varint,2,opt,name=y,proto3" json:"y,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -122,13 +113,6 @@ func (*ProtoMouseMoveAbs) Descriptor() ([]byte, []int) { return file_types_proto_rawDescGZIP(), []int{1} } -func (x *ProtoMouseMoveAbs) GetType() string { - if x != nil { - return x.Type - } - return "" -} - func (x *ProtoMouseMoveAbs) GetX() int32 { if x != nil { return x.X @@ -146,9 +130,8 @@ func (x *ProtoMouseMoveAbs) GetY() int32 { // MouseWheel message type ProtoMouseWheel struct { state protoimpl.MessageState `protogen:"open.v1"` - Type string `protobuf:"bytes,1,opt,name=type,proto3" json:"type,omitempty"` // Fixed value "MouseWheel" - X int32 `protobuf:"varint,2,opt,name=x,proto3" json:"x,omitempty"` - Y int32 `protobuf:"varint,3,opt,name=y,proto3" json:"y,omitempty"` + X int32 `protobuf:"varint,1,opt,name=x,proto3" json:"x,omitempty"` + Y int32 `protobuf:"varint,2,opt,name=y,proto3" json:"y,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -183,13 +166,6 @@ func (*ProtoMouseWheel) Descriptor() ([]byte, []int) { return file_types_proto_rawDescGZIP(), []int{2} } -func (x *ProtoMouseWheel) GetType() string { - if x != nil { - return x.Type - } - return "" -} - func (x *ProtoMouseWheel) GetX() int32 { if x != nil { return x.X @@ -207,8 +183,7 @@ func (x *ProtoMouseWheel) GetY() int32 { // MouseKeyDown message type ProtoMouseKeyDown struct { state protoimpl.MessageState `protogen:"open.v1"` - Type string `protobuf:"bytes,1,opt,name=type,proto3" json:"type,omitempty"` // Fixed value "MouseKeyDown" - Key int32 `protobuf:"varint,2,opt,name=key,proto3" json:"key,omitempty"` + Key int32 `protobuf:"varint,1,opt,name=key,proto3" json:"key,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -243,13 +218,6 @@ func (*ProtoMouseKeyDown) Descriptor() ([]byte, []int) { return file_types_proto_rawDescGZIP(), []int{3} } -func (x *ProtoMouseKeyDown) GetType() string { - if x != nil { - return x.Type - } - return "" -} - func (x *ProtoMouseKeyDown) GetKey() int32 { if x != nil { return x.Key @@ -260,8 +228,7 @@ func (x *ProtoMouseKeyDown) GetKey() int32 { // MouseKeyUp message type ProtoMouseKeyUp struct { state protoimpl.MessageState `protogen:"open.v1"` - Type string `protobuf:"bytes,1,opt,name=type,proto3" json:"type,omitempty"` // Fixed value "MouseKeyUp" - Key int32 `protobuf:"varint,2,opt,name=key,proto3" json:"key,omitempty"` + Key int32 `protobuf:"varint,1,opt,name=key,proto3" json:"key,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -296,13 +263,6 @@ func (*ProtoMouseKeyUp) Descriptor() ([]byte, []int) { return file_types_proto_rawDescGZIP(), []int{4} } -func (x *ProtoMouseKeyUp) GetType() string { - if x != nil { - return x.Type - } - return "" -} - func (x *ProtoMouseKeyUp) GetKey() int32 { if x != nil { return x.Key @@ -313,8 +273,7 @@ func (x *ProtoMouseKeyUp) GetKey() int32 { // KeyDown message type ProtoKeyDown struct { state protoimpl.MessageState `protogen:"open.v1"` - Type string `protobuf:"bytes,1,opt,name=type,proto3" json:"type,omitempty"` // Fixed value "KeyDown" - Key int32 `protobuf:"varint,2,opt,name=key,proto3" json:"key,omitempty"` + Key int32 `protobuf:"varint,1,opt,name=key,proto3" json:"key,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -349,13 +308,6 @@ func (*ProtoKeyDown) Descriptor() ([]byte, []int) { return file_types_proto_rawDescGZIP(), []int{5} } -func (x *ProtoKeyDown) GetType() string { - if x != nil { - return x.Type - } - return "" -} - func (x *ProtoKeyDown) GetKey() int32 { if x != nil { return x.Key @@ -366,8 +318,7 @@ func (x *ProtoKeyDown) GetKey() int32 { // KeyUp message type ProtoKeyUp struct { state protoimpl.MessageState `protogen:"open.v1"` - Type string `protobuf:"bytes,1,opt,name=type,proto3" json:"type,omitempty"` // Fixed value "KeyUp" - Key int32 `protobuf:"varint,2,opt,name=key,proto3" json:"key,omitempty"` + Key int32 `protobuf:"varint,1,opt,name=key,proto3" json:"key,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -402,13 +353,6 @@ func (*ProtoKeyUp) Descriptor() ([]byte, []int) { return file_types_proto_rawDescGZIP(), []int{6} } -func (x *ProtoKeyUp) GetType() string { - if x != nil { - return x.Type - } - return "" -} - func (x *ProtoKeyUp) GetKey() int32 { if x != nil { return x.Key @@ -419,9 +363,9 @@ func (x *ProtoKeyUp) GetKey() int32 { // ControllerAttach message type ProtoControllerAttach struct { state protoimpl.MessageState `protogen:"open.v1"` - Type string `protobuf:"bytes,1,opt,name=type,proto3" json:"type,omitempty"` // Fixed value "ControllerAttach" - Id string `protobuf:"bytes,2,opt,name=id,proto3" json:"id,omitempty"` // One of the following enums: "ps", "xbox" or "switch" - Slot int32 `protobuf:"varint,3,opt,name=slot,proto3" json:"slot,omitempty"` // Slot number (0-3) + Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"` // One of the following enums: "ps", "xbox" or "switch" + Slot int32 `protobuf:"varint,2,opt,name=slot,proto3" json:"slot,omitempty"` // Slot number (0-3) + SessionId string `protobuf:"bytes,3,opt,name=session_id,json=sessionId,proto3" json:"session_id,omitempty"` // Session ID of the client attaching the controller unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -456,13 +400,6 @@ func (*ProtoControllerAttach) Descriptor() ([]byte, []int) { return file_types_proto_rawDescGZIP(), []int{7} } -func (x *ProtoControllerAttach) GetType() string { - if x != nil { - return x.Type - } - return "" -} - func (x *ProtoControllerAttach) GetId() string { if x != nil { return x.Id @@ -477,11 +414,17 @@ func (x *ProtoControllerAttach) GetSlot() int32 { return 0 } +func (x *ProtoControllerAttach) GetSessionId() string { + if x != nil { + return x.SessionId + } + return "" +} + // ControllerDetach message type ProtoControllerDetach struct { state protoimpl.MessageState `protogen:"open.v1"` - Type string `protobuf:"bytes,1,opt,name=type,proto3" json:"type,omitempty"` // Fixed value "ControllerDetach" - Slot int32 `protobuf:"varint,2,opt,name=slot,proto3" json:"slot,omitempty"` // Slot number (0-3) + Slot int32 `protobuf:"varint,1,opt,name=slot,proto3" json:"slot,omitempty"` // Slot number (0-3) unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -516,13 +459,6 @@ func (*ProtoControllerDetach) Descriptor() ([]byte, []int) { return file_types_proto_rawDescGZIP(), []int{8} } -func (x *ProtoControllerDetach) GetType() string { - if x != nil { - return x.Type - } - return "" -} - func (x *ProtoControllerDetach) GetSlot() int32 { if x != nil { return x.Slot @@ -533,10 +469,9 @@ func (x *ProtoControllerDetach) GetSlot() int32 { // ControllerButton message type ProtoControllerButton struct { state protoimpl.MessageState `protogen:"open.v1"` - Type string `protobuf:"bytes,1,opt,name=type,proto3" json:"type,omitempty"` // Fixed value "ControllerButtons" - Slot int32 `protobuf:"varint,2,opt,name=slot,proto3" json:"slot,omitempty"` // Slot number (0-3) - Button int32 `protobuf:"varint,3,opt,name=button,proto3" json:"button,omitempty"` // Button code (linux input event code) - Pressed bool `protobuf:"varint,4,opt,name=pressed,proto3" json:"pressed,omitempty"` // true if pressed, false if released + Slot int32 `protobuf:"varint,1,opt,name=slot,proto3" json:"slot,omitempty"` // Slot number (0-3) + Button int32 `protobuf:"varint,2,opt,name=button,proto3" json:"button,omitempty"` // Button code (linux input event code) + Pressed bool `protobuf:"varint,3,opt,name=pressed,proto3" json:"pressed,omitempty"` // true if pressed, false if released unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -571,13 +506,6 @@ func (*ProtoControllerButton) Descriptor() ([]byte, []int) { return file_types_proto_rawDescGZIP(), []int{9} } -func (x *ProtoControllerButton) GetType() string { - if x != nil { - return x.Type - } - return "" -} - func (x *ProtoControllerButton) GetSlot() int32 { if x != nil { return x.Slot @@ -602,10 +530,9 @@ func (x *ProtoControllerButton) GetPressed() bool { // ControllerTriggers message type ProtoControllerTrigger struct { state protoimpl.MessageState `protogen:"open.v1"` - Type string `protobuf:"bytes,1,opt,name=type,proto3" json:"type,omitempty"` // Fixed value "ControllerTriggers" - Slot int32 `protobuf:"varint,2,opt,name=slot,proto3" json:"slot,omitempty"` // Slot number (0-3) - Trigger int32 `protobuf:"varint,3,opt,name=trigger,proto3" json:"trigger,omitempty"` // Trigger number (0 for left, 1 for right) - Value int32 `protobuf:"varint,4,opt,name=value,proto3" json:"value,omitempty"` // trigger value (-32768 to 32767) + Slot int32 `protobuf:"varint,1,opt,name=slot,proto3" json:"slot,omitempty"` // Slot number (0-3) + Trigger int32 `protobuf:"varint,2,opt,name=trigger,proto3" json:"trigger,omitempty"` // Trigger number (0 for left, 1 for right) + Value int32 `protobuf:"varint,3,opt,name=value,proto3" json:"value,omitempty"` // trigger value (-32768 to 32767) unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -640,13 +567,6 @@ func (*ProtoControllerTrigger) Descriptor() ([]byte, []int) { return file_types_proto_rawDescGZIP(), []int{10} } -func (x *ProtoControllerTrigger) GetType() string { - if x != nil { - return x.Type - } - return "" -} - func (x *ProtoControllerTrigger) GetSlot() int32 { if x != nil { return x.Slot @@ -671,11 +591,10 @@ func (x *ProtoControllerTrigger) GetValue() int32 { // ControllerSticks message type ProtoControllerStick struct { state protoimpl.MessageState `protogen:"open.v1"` - Type string `protobuf:"bytes,1,opt,name=type,proto3" json:"type,omitempty"` // Fixed value "ControllerStick" - Slot int32 `protobuf:"varint,2,opt,name=slot,proto3" json:"slot,omitempty"` // Slot number (0-3) - Stick int32 `protobuf:"varint,3,opt,name=stick,proto3" json:"stick,omitempty"` // Stick number (0 for left, 1 for right) - X int32 `protobuf:"varint,4,opt,name=x,proto3" json:"x,omitempty"` // X axis value (-32768 to 32767) - Y int32 `protobuf:"varint,5,opt,name=y,proto3" json:"y,omitempty"` // Y axis value (-32768 to 32767) + Slot int32 `protobuf:"varint,1,opt,name=slot,proto3" json:"slot,omitempty"` // Slot number (0-3) + Stick int32 `protobuf:"varint,2,opt,name=stick,proto3" json:"stick,omitempty"` // Stick number (0 for left, 1 for right) + X int32 `protobuf:"varint,3,opt,name=x,proto3" json:"x,omitempty"` // X axis value (-32768 to 32767) + Y int32 `protobuf:"varint,4,opt,name=y,proto3" json:"y,omitempty"` // Y axis value (-32768 to 32767) unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -710,13 +629,6 @@ func (*ProtoControllerStick) Descriptor() ([]byte, []int) { return file_types_proto_rawDescGZIP(), []int{11} } -func (x *ProtoControllerStick) GetType() string { - if x != nil { - return x.Type - } - return "" -} - func (x *ProtoControllerStick) GetSlot() int32 { if x != nil { return x.Slot @@ -748,10 +660,9 @@ func (x *ProtoControllerStick) GetY() int32 { // ControllerAxis message type ProtoControllerAxis struct { state protoimpl.MessageState `protogen:"open.v1"` - Type string `protobuf:"bytes,1,opt,name=type,proto3" json:"type,omitempty"` // Fixed value "ControllerAxis" - Slot int32 `protobuf:"varint,2,opt,name=slot,proto3" json:"slot,omitempty"` // Slot number (0-3) - Axis int32 `protobuf:"varint,3,opt,name=axis,proto3" json:"axis,omitempty"` // Axis number (0 for d-pad horizontal, 1 for d-pad vertical) - Value int32 `protobuf:"varint,4,opt,name=value,proto3" json:"value,omitempty"` // axis value (-1 to 1) + Slot int32 `protobuf:"varint,1,opt,name=slot,proto3" json:"slot,omitempty"` // Slot number (0-3) + Axis int32 `protobuf:"varint,2,opt,name=axis,proto3" json:"axis,omitempty"` // Axis number (0 for d-pad horizontal, 1 for d-pad vertical) + Value int32 `protobuf:"varint,3,opt,name=value,proto3" json:"value,omitempty"` // axis value (-1 to 1) unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -786,13 +697,6 @@ func (*ProtoControllerAxis) Descriptor() ([]byte, []int) { return file_types_proto_rawDescGZIP(), []int{12} } -func (x *ProtoControllerAxis) GetType() string { - if x != nil { - return x.Type - } - return "" -} - func (x *ProtoControllerAxis) GetSlot() int32 { if x != nil { return x.Slot @@ -817,11 +721,10 @@ func (x *ProtoControllerAxis) GetValue() int32 { // ControllerRumble message type ProtoControllerRumble struct { state protoimpl.MessageState `protogen:"open.v1"` - Type string `protobuf:"bytes,1,opt,name=type,proto3" json:"type,omitempty"` // Fixed value "ControllerRumble" - Slot int32 `protobuf:"varint,2,opt,name=slot,proto3" json:"slot,omitempty"` // Slot number (0-3) - LowFrequency int32 `protobuf:"varint,3,opt,name=low_frequency,json=lowFrequency,proto3" json:"low_frequency,omitempty"` // Low frequency rumble (0-65535) - HighFrequency int32 `protobuf:"varint,4,opt,name=high_frequency,json=highFrequency,proto3" json:"high_frequency,omitempty"` // High frequency rumble (0-65535) - Duration int32 `protobuf:"varint,5,opt,name=duration,proto3" json:"duration,omitempty"` // Duration in milliseconds + Slot int32 `protobuf:"varint,1,opt,name=slot,proto3" json:"slot,omitempty"` // Slot number (0-3) + LowFrequency int32 `protobuf:"varint,2,opt,name=low_frequency,json=lowFrequency,proto3" json:"low_frequency,omitempty"` // Low frequency rumble (0-65535) + HighFrequency int32 `protobuf:"varint,3,opt,name=high_frequency,json=highFrequency,proto3" json:"high_frequency,omitempty"` // High frequency rumble (0-65535) + Duration int32 `protobuf:"varint,4,opt,name=duration,proto3" json:"duration,omitempty"` // Duration in milliseconds unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -856,13 +759,6 @@ func (*ProtoControllerRumble) Descriptor() ([]byte, []int) { return file_types_proto_rawDescGZIP(), []int{13} } -func (x *ProtoControllerRumble) GetType() string { - if x != nil { - return x.Type - } - return "" -} - func (x *ProtoControllerRumble) GetSlot() int32 { if x != nil { return x.Slot @@ -891,44 +787,30 @@ func (x *ProtoControllerRumble) GetDuration() int32 { return 0 } -// Union of all Input types -type ProtoInput struct { - state protoimpl.MessageState `protogen:"open.v1"` - // Types that are valid to be assigned to InputType: - // - // *ProtoInput_MouseMove - // *ProtoInput_MouseMoveAbs - // *ProtoInput_MouseWheel - // *ProtoInput_MouseKeyDown - // *ProtoInput_MouseKeyUp - // *ProtoInput_KeyDown - // *ProtoInput_KeyUp - // *ProtoInput_ControllerAttach - // *ProtoInput_ControllerDetach - // *ProtoInput_ControllerButton - // *ProtoInput_ControllerTrigger - // *ProtoInput_ControllerStick - // *ProtoInput_ControllerAxis - // *ProtoInput_ControllerRumble - InputType isProtoInput_InputType `protobuf_oneof:"input_type"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache +type RTCIceCandidateInit struct { + state protoimpl.MessageState `protogen:"open.v1"` + Candidate string `protobuf:"bytes,1,opt,name=candidate,proto3" json:"candidate,omitempty"` + SdpMLineIndex *uint32 `protobuf:"varint,2,opt,name=sdpMLineIndex,proto3,oneof" json:"sdpMLineIndex,omitempty"` + SdpMid *string `protobuf:"bytes,3,opt,name=sdpMid,proto3,oneof" json:"sdpMid,omitempty"` + UsernameFragment *string `protobuf:"bytes,4,opt,name=usernameFragment,proto3,oneof" json:"usernameFragment,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } -func (x *ProtoInput) Reset() { - *x = ProtoInput{} +func (x *RTCIceCandidateInit) Reset() { + *x = RTCIceCandidateInit{} mi := &file_types_proto_msgTypes[14] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } -func (x *ProtoInput) String() string { +func (x *RTCIceCandidateInit) String() string { return protoimpl.X.MessageStringOf(x) } -func (*ProtoInput) ProtoMessage() {} +func (*RTCIceCandidateInit) ProtoMessage() {} -func (x *ProtoInput) ProtoReflect() protoreflect.Message { +func (x *RTCIceCandidateInit) ProtoReflect() protoreflect.Message { mi := &file_types_proto_msgTypes[14] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) @@ -940,318 +822,456 @@ func (x *ProtoInput) ProtoReflect() protoreflect.Message { return mi.MessageOf(x) } -// Deprecated: Use ProtoInput.ProtoReflect.Descriptor instead. -func (*ProtoInput) Descriptor() ([]byte, []int) { +// Deprecated: Use RTCIceCandidateInit.ProtoReflect.Descriptor instead. +func (*RTCIceCandidateInit) Descriptor() ([]byte, []int) { return file_types_proto_rawDescGZIP(), []int{14} } -func (x *ProtoInput) GetInputType() isProtoInput_InputType { +func (x *RTCIceCandidateInit) GetCandidate() string { if x != nil { - return x.InputType + return x.Candidate } - return nil + return "" } -func (x *ProtoInput) GetMouseMove() *ProtoMouseMove { - if x != nil { - if x, ok := x.InputType.(*ProtoInput_MouseMove); ok { - return x.MouseMove - } +func (x *RTCIceCandidateInit) GetSdpMLineIndex() uint32 { + if x != nil && x.SdpMLineIndex != nil { + return *x.SdpMLineIndex } - return nil + return 0 } -func (x *ProtoInput) GetMouseMoveAbs() *ProtoMouseMoveAbs { - if x != nil { - if x, ok := x.InputType.(*ProtoInput_MouseMoveAbs); ok { - return x.MouseMoveAbs - } +func (x *RTCIceCandidateInit) GetSdpMid() string { + if x != nil && x.SdpMid != nil { + return *x.SdpMid } - return nil + return "" } -func (x *ProtoInput) GetMouseWheel() *ProtoMouseWheel { - if x != nil { - if x, ok := x.InputType.(*ProtoInput_MouseWheel); ok { - return x.MouseWheel - } +func (x *RTCIceCandidateInit) GetUsernameFragment() string { + if x != nil && x.UsernameFragment != nil { + return *x.UsernameFragment } - return nil + return "" } -func (x *ProtoInput) GetMouseKeyDown() *ProtoMouseKeyDown { - if x != nil { - if x, ok := x.InputType.(*ProtoInput_MouseKeyDown); ok { - return x.MouseKeyDown - } - } - return nil +type RTCSessionDescriptionInit struct { + state protoimpl.MessageState `protogen:"open.v1"` + Sdp string `protobuf:"bytes,1,opt,name=sdp,proto3" json:"sdp,omitempty"` + Type string `protobuf:"bytes,2,opt,name=type,proto3" json:"type,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } -func (x *ProtoInput) GetMouseKeyUp() *ProtoMouseKeyUp { - if x != nil { - if x, ok := x.InputType.(*ProtoInput_MouseKeyUp); ok { - return x.MouseKeyUp - } - } - return nil +func (x *RTCSessionDescriptionInit) Reset() { + *x = RTCSessionDescriptionInit{} + mi := &file_types_proto_msgTypes[15] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } -func (x *ProtoInput) GetKeyDown() *ProtoKeyDown { - if x != nil { - if x, ok := x.InputType.(*ProtoInput_KeyDown); ok { - return x.KeyDown - } - } - return nil +func (x *RTCSessionDescriptionInit) String() string { + return protoimpl.X.MessageStringOf(x) } -func (x *ProtoInput) GetKeyUp() *ProtoKeyUp { +func (*RTCSessionDescriptionInit) ProtoMessage() {} + +func (x *RTCSessionDescriptionInit) ProtoReflect() protoreflect.Message { + mi := &file_types_proto_msgTypes[15] if x != nil { - if x, ok := x.InputType.(*ProtoInput_KeyUp); ok { - return x.KeyUp + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) } + return ms } - return nil + return mi.MessageOf(x) } -func (x *ProtoInput) GetControllerAttach() *ProtoControllerAttach { - if x != nil { - if x, ok := x.InputType.(*ProtoInput_ControllerAttach); ok { - return x.ControllerAttach - } - } - return nil +// Deprecated: Use RTCSessionDescriptionInit.ProtoReflect.Descriptor instead. +func (*RTCSessionDescriptionInit) Descriptor() ([]byte, []int) { + return file_types_proto_rawDescGZIP(), []int{15} } -func (x *ProtoInput) GetControllerDetach() *ProtoControllerDetach { +func (x *RTCSessionDescriptionInit) GetSdp() string { if x != nil { - if x, ok := x.InputType.(*ProtoInput_ControllerDetach); ok { - return x.ControllerDetach - } + return x.Sdp } - return nil + return "" } -func (x *ProtoInput) GetControllerButton() *ProtoControllerButton { +func (x *RTCSessionDescriptionInit) GetType() string { if x != nil { - if x, ok := x.InputType.(*ProtoInput_ControllerButton); ok { - return x.ControllerButton - } + return x.Type } - return nil + return "" +} + +// ProtoICE message +type ProtoICE struct { + state protoimpl.MessageState `protogen:"open.v1"` + Candidate *RTCIceCandidateInit `protobuf:"bytes,1,opt,name=candidate,proto3" json:"candidate,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ProtoICE) Reset() { + *x = ProtoICE{} + mi := &file_types_proto_msgTypes[16] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ProtoICE) String() string { + return protoimpl.X.MessageStringOf(x) } -func (x *ProtoInput) GetControllerTrigger() *ProtoControllerTrigger { +func (*ProtoICE) ProtoMessage() {} + +func (x *ProtoICE) ProtoReflect() protoreflect.Message { + mi := &file_types_proto_msgTypes[16] if x != nil { - if x, ok := x.InputType.(*ProtoInput_ControllerTrigger); ok { - return x.ControllerTrigger + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) } + return ms } - return nil + return mi.MessageOf(x) +} + +// Deprecated: Use ProtoICE.ProtoReflect.Descriptor instead. +func (*ProtoICE) Descriptor() ([]byte, []int) { + return file_types_proto_rawDescGZIP(), []int{16} } -func (x *ProtoInput) GetControllerStick() *ProtoControllerStick { +func (x *ProtoICE) GetCandidate() *RTCIceCandidateInit { if x != nil { - if x, ok := x.InputType.(*ProtoInput_ControllerStick); ok { - return x.ControllerStick - } + return x.Candidate } return nil } -func (x *ProtoInput) GetControllerAxis() *ProtoControllerAxis { +// ProtoSDP message +type ProtoSDP struct { + state protoimpl.MessageState `protogen:"open.v1"` + Sdp *RTCSessionDescriptionInit `protobuf:"bytes,1,opt,name=sdp,proto3" json:"sdp,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ProtoSDP) Reset() { + *x = ProtoSDP{} + mi := &file_types_proto_msgTypes[17] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ProtoSDP) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ProtoSDP) ProtoMessage() {} + +func (x *ProtoSDP) ProtoReflect() protoreflect.Message { + mi := &file_types_proto_msgTypes[17] if x != nil { - if x, ok := x.InputType.(*ProtoInput_ControllerAxis); ok { - return x.ControllerAxis + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) } + return ms } - return nil + return mi.MessageOf(x) +} + +// Deprecated: Use ProtoSDP.ProtoReflect.Descriptor instead. +func (*ProtoSDP) Descriptor() ([]byte, []int) { + return file_types_proto_rawDescGZIP(), []int{17} } -func (x *ProtoInput) GetControllerRumble() *ProtoControllerRumble { +func (x *ProtoSDP) GetSdp() *RTCSessionDescriptionInit { if x != nil { - if x, ok := x.InputType.(*ProtoInput_ControllerRumble); ok { - return x.ControllerRumble - } + return x.Sdp } return nil } -type isProtoInput_InputType interface { - isProtoInput_InputType() +// ProtoRaw message +type ProtoRaw struct { + state protoimpl.MessageState `protogen:"open.v1"` + Data string `protobuf:"bytes,1,opt,name=data,proto3" json:"data,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } -type ProtoInput_MouseMove struct { - MouseMove *ProtoMouseMove `protobuf:"bytes,1,opt,name=mouse_move,json=mouseMove,proto3,oneof"` +func (x *ProtoRaw) Reset() { + *x = ProtoRaw{} + mi := &file_types_proto_msgTypes[18] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } -type ProtoInput_MouseMoveAbs struct { - MouseMoveAbs *ProtoMouseMoveAbs `protobuf:"bytes,2,opt,name=mouse_move_abs,json=mouseMoveAbs,proto3,oneof"` +func (x *ProtoRaw) String() string { + return protoimpl.X.MessageStringOf(x) } -type ProtoInput_MouseWheel struct { - MouseWheel *ProtoMouseWheel `protobuf:"bytes,3,opt,name=mouse_wheel,json=mouseWheel,proto3,oneof"` -} +func (*ProtoRaw) ProtoMessage() {} -type ProtoInput_MouseKeyDown struct { - MouseKeyDown *ProtoMouseKeyDown `protobuf:"bytes,4,opt,name=mouse_key_down,json=mouseKeyDown,proto3,oneof"` +func (x *ProtoRaw) ProtoReflect() protoreflect.Message { + mi := &file_types_proto_msgTypes[18] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) } -type ProtoInput_MouseKeyUp struct { - MouseKeyUp *ProtoMouseKeyUp `protobuf:"bytes,5,opt,name=mouse_key_up,json=mouseKeyUp,proto3,oneof"` +// Deprecated: Use ProtoRaw.ProtoReflect.Descriptor instead. +func (*ProtoRaw) Descriptor() ([]byte, []int) { + return file_types_proto_rawDescGZIP(), []int{18} } -type ProtoInput_KeyDown struct { - KeyDown *ProtoKeyDown `protobuf:"bytes,6,opt,name=key_down,json=keyDown,proto3,oneof"` +func (x *ProtoRaw) GetData() string { + if x != nil { + return x.Data + } + return "" } -type ProtoInput_KeyUp struct { - KeyUp *ProtoKeyUp `protobuf:"bytes,7,opt,name=key_up,json=keyUp,proto3,oneof"` +// ProtoClientRequestRoomStream message +type ProtoClientRequestRoomStream struct { + state protoimpl.MessageState `protogen:"open.v1"` + RoomName string `protobuf:"bytes,1,opt,name=room_name,json=roomName,proto3" json:"room_name,omitempty"` + SessionId string `protobuf:"bytes,2,opt,name=session_id,json=sessionId,proto3" json:"session_id,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } -type ProtoInput_ControllerAttach struct { - ControllerAttach *ProtoControllerAttach `protobuf:"bytes,8,opt,name=controller_attach,json=controllerAttach,proto3,oneof"` +func (x *ProtoClientRequestRoomStream) Reset() { + *x = ProtoClientRequestRoomStream{} + mi := &file_types_proto_msgTypes[19] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } -type ProtoInput_ControllerDetach struct { - ControllerDetach *ProtoControllerDetach `protobuf:"bytes,9,opt,name=controller_detach,json=controllerDetach,proto3,oneof"` +func (x *ProtoClientRequestRoomStream) String() string { + return protoimpl.X.MessageStringOf(x) } -type ProtoInput_ControllerButton struct { - ControllerButton *ProtoControllerButton `protobuf:"bytes,10,opt,name=controller_button,json=controllerButton,proto3,oneof"` +func (*ProtoClientRequestRoomStream) ProtoMessage() {} + +func (x *ProtoClientRequestRoomStream) ProtoReflect() protoreflect.Message { + mi := &file_types_proto_msgTypes[19] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) } -type ProtoInput_ControllerTrigger struct { - ControllerTrigger *ProtoControllerTrigger `protobuf:"bytes,11,opt,name=controller_trigger,json=controllerTrigger,proto3,oneof"` +// Deprecated: Use ProtoClientRequestRoomStream.ProtoReflect.Descriptor instead. +func (*ProtoClientRequestRoomStream) Descriptor() ([]byte, []int) { + return file_types_proto_rawDescGZIP(), []int{19} } -type ProtoInput_ControllerStick struct { - ControllerStick *ProtoControllerStick `protobuf:"bytes,12,opt,name=controller_stick,json=controllerStick,proto3,oneof"` +func (x *ProtoClientRequestRoomStream) GetRoomName() string { + if x != nil { + return x.RoomName + } + return "" } -type ProtoInput_ControllerAxis struct { - ControllerAxis *ProtoControllerAxis `protobuf:"bytes,13,opt,name=controller_axis,json=controllerAxis,proto3,oneof"` +func (x *ProtoClientRequestRoomStream) GetSessionId() string { + if x != nil { + return x.SessionId + } + return "" } -type ProtoInput_ControllerRumble struct { - ControllerRumble *ProtoControllerRumble `protobuf:"bytes,14,opt,name=controller_rumble,json=controllerRumble,proto3,oneof"` +// ProtoClientDisconnected message +type ProtoClientDisconnected struct { + state protoimpl.MessageState `protogen:"open.v1"` + SessionId string `protobuf:"bytes,1,opt,name=session_id,json=sessionId,proto3" json:"session_id,omitempty"` + ControllerSlots []int32 `protobuf:"varint,2,rep,packed,name=controller_slots,json=controllerSlots,proto3" json:"controller_slots,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } -func (*ProtoInput_MouseMove) isProtoInput_InputType() {} +func (x *ProtoClientDisconnected) Reset() { + *x = ProtoClientDisconnected{} + mi := &file_types_proto_msgTypes[20] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} -func (*ProtoInput_MouseMoveAbs) isProtoInput_InputType() {} +func (x *ProtoClientDisconnected) String() string { + return protoimpl.X.MessageStringOf(x) +} -func (*ProtoInput_MouseWheel) isProtoInput_InputType() {} +func (*ProtoClientDisconnected) ProtoMessage() {} -func (*ProtoInput_MouseKeyDown) isProtoInput_InputType() {} +func (x *ProtoClientDisconnected) ProtoReflect() protoreflect.Message { + mi := &file_types_proto_msgTypes[20] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} -func (*ProtoInput_MouseKeyUp) isProtoInput_InputType() {} +// Deprecated: Use ProtoClientDisconnected.ProtoReflect.Descriptor instead. +func (*ProtoClientDisconnected) Descriptor() ([]byte, []int) { + return file_types_proto_rawDescGZIP(), []int{20} +} -func (*ProtoInput_KeyDown) isProtoInput_InputType() {} +func (x *ProtoClientDisconnected) GetSessionId() string { + if x != nil { + return x.SessionId + } + return "" +} -func (*ProtoInput_KeyUp) isProtoInput_InputType() {} +func (x *ProtoClientDisconnected) GetControllerSlots() []int32 { + if x != nil { + return x.ControllerSlots + } + return nil +} -func (*ProtoInput_ControllerAttach) isProtoInput_InputType() {} +// ProtoServerPushStream message +type ProtoServerPushStream struct { + state protoimpl.MessageState `protogen:"open.v1"` + RoomName string `protobuf:"bytes,1,opt,name=room_name,json=roomName,proto3" json:"room_name,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} -func (*ProtoInput_ControllerDetach) isProtoInput_InputType() {} +func (x *ProtoServerPushStream) Reset() { + *x = ProtoServerPushStream{} + mi := &file_types_proto_msgTypes[21] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} -func (*ProtoInput_ControllerButton) isProtoInput_InputType() {} +func (x *ProtoServerPushStream) String() string { + return protoimpl.X.MessageStringOf(x) +} -func (*ProtoInput_ControllerTrigger) isProtoInput_InputType() {} +func (*ProtoServerPushStream) ProtoMessage() {} -func (*ProtoInput_ControllerStick) isProtoInput_InputType() {} +func (x *ProtoServerPushStream) ProtoReflect() protoreflect.Message { + mi := &file_types_proto_msgTypes[21] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} -func (*ProtoInput_ControllerAxis) isProtoInput_InputType() {} +// Deprecated: Use ProtoServerPushStream.ProtoReflect.Descriptor instead. +func (*ProtoServerPushStream) Descriptor() ([]byte, []int) { + return file_types_proto_rawDescGZIP(), []int{21} +} -func (*ProtoInput_ControllerRumble) isProtoInput_InputType() {} +func (x *ProtoServerPushStream) GetRoomName() string { + if x != nil { + return x.RoomName + } + return "" +} var File_types_proto protoreflect.FileDescriptor const file_types_proto_rawDesc = "" + "\n" + - "\vtypes.proto\x12\x05proto\"@\n" + - "\x0eProtoMouseMove\x12\x12\n" + - "\x04type\x18\x01 \x01(\tR\x04type\x12\f\n" + - "\x01x\x18\x02 \x01(\x05R\x01x\x12\f\n" + - "\x01y\x18\x03 \x01(\x05R\x01y\"C\n" + - "\x11ProtoMouseMoveAbs\x12\x12\n" + - "\x04type\x18\x01 \x01(\tR\x04type\x12\f\n" + - "\x01x\x18\x02 \x01(\x05R\x01x\x12\f\n" + - "\x01y\x18\x03 \x01(\x05R\x01y\"A\n" + - "\x0fProtoMouseWheel\x12\x12\n" + - "\x04type\x18\x01 \x01(\tR\x04type\x12\f\n" + - "\x01x\x18\x02 \x01(\x05R\x01x\x12\f\n" + - "\x01y\x18\x03 \x01(\x05R\x01y\"9\n" + - "\x11ProtoMouseKeyDown\x12\x12\n" + - "\x04type\x18\x01 \x01(\tR\x04type\x12\x10\n" + - "\x03key\x18\x02 \x01(\x05R\x03key\"7\n" + - "\x0fProtoMouseKeyUp\x12\x12\n" + - "\x04type\x18\x01 \x01(\tR\x04type\x12\x10\n" + - "\x03key\x18\x02 \x01(\x05R\x03key\"4\n" + - "\fProtoKeyDown\x12\x12\n" + - "\x04type\x18\x01 \x01(\tR\x04type\x12\x10\n" + - "\x03key\x18\x02 \x01(\x05R\x03key\"2\n" + + "\vtypes.proto\x12\x05proto\",\n" + + "\x0eProtoMouseMove\x12\f\n" + + "\x01x\x18\x01 \x01(\x05R\x01x\x12\f\n" + + "\x01y\x18\x02 \x01(\x05R\x01y\"/\n" + + "\x11ProtoMouseMoveAbs\x12\f\n" + + "\x01x\x18\x01 \x01(\x05R\x01x\x12\f\n" + + "\x01y\x18\x02 \x01(\x05R\x01y\"-\n" + + "\x0fProtoMouseWheel\x12\f\n" + + "\x01x\x18\x01 \x01(\x05R\x01x\x12\f\n" + + "\x01y\x18\x02 \x01(\x05R\x01y\"%\n" + + "\x11ProtoMouseKeyDown\x12\x10\n" + + "\x03key\x18\x01 \x01(\x05R\x03key\"#\n" + + "\x0fProtoMouseKeyUp\x12\x10\n" + + "\x03key\x18\x01 \x01(\x05R\x03key\" \n" + + "\fProtoKeyDown\x12\x10\n" + + "\x03key\x18\x01 \x01(\x05R\x03key\"\x1e\n" + + "\n" + + "ProtoKeyUp\x12\x10\n" + + "\x03key\x18\x01 \x01(\x05R\x03key\"Z\n" + + "\x15ProtoControllerAttach\x12\x0e\n" + + "\x02id\x18\x01 \x01(\tR\x02id\x12\x12\n" + + "\x04slot\x18\x02 \x01(\x05R\x04slot\x12\x1d\n" + "\n" + - "ProtoKeyUp\x12\x12\n" + - "\x04type\x18\x01 \x01(\tR\x04type\x12\x10\n" + - "\x03key\x18\x02 \x01(\x05R\x03key\"O\n" + - "\x15ProtoControllerAttach\x12\x12\n" + - "\x04type\x18\x01 \x01(\tR\x04type\x12\x0e\n" + - "\x02id\x18\x02 \x01(\tR\x02id\x12\x12\n" + - "\x04slot\x18\x03 \x01(\x05R\x04slot\"?\n" + + "session_id\x18\x03 \x01(\tR\tsessionId\"+\n" + "\x15ProtoControllerDetach\x12\x12\n" + - "\x04type\x18\x01 \x01(\tR\x04type\x12\x12\n" + - "\x04slot\x18\x02 \x01(\x05R\x04slot\"q\n" + + "\x04slot\x18\x01 \x01(\x05R\x04slot\"]\n" + "\x15ProtoControllerButton\x12\x12\n" + - "\x04type\x18\x01 \x01(\tR\x04type\x12\x12\n" + - "\x04slot\x18\x02 \x01(\x05R\x04slot\x12\x16\n" + - "\x06button\x18\x03 \x01(\x05R\x06button\x12\x18\n" + - "\apressed\x18\x04 \x01(\bR\apressed\"p\n" + + "\x04slot\x18\x01 \x01(\x05R\x04slot\x12\x16\n" + + "\x06button\x18\x02 \x01(\x05R\x06button\x12\x18\n" + + "\apressed\x18\x03 \x01(\bR\apressed\"\\\n" + "\x16ProtoControllerTrigger\x12\x12\n" + - "\x04type\x18\x01 \x01(\tR\x04type\x12\x12\n" + - "\x04slot\x18\x02 \x01(\x05R\x04slot\x12\x18\n" + - "\atrigger\x18\x03 \x01(\x05R\atrigger\x12\x14\n" + - "\x05value\x18\x04 \x01(\x05R\x05value\"p\n" + + "\x04slot\x18\x01 \x01(\x05R\x04slot\x12\x18\n" + + "\atrigger\x18\x02 \x01(\x05R\atrigger\x12\x14\n" + + "\x05value\x18\x03 \x01(\x05R\x05value\"\\\n" + "\x14ProtoControllerStick\x12\x12\n" + - "\x04type\x18\x01 \x01(\tR\x04type\x12\x12\n" + - "\x04slot\x18\x02 \x01(\x05R\x04slot\x12\x14\n" + - "\x05stick\x18\x03 \x01(\x05R\x05stick\x12\f\n" + - "\x01x\x18\x04 \x01(\x05R\x01x\x12\f\n" + - "\x01y\x18\x05 \x01(\x05R\x01y\"g\n" + + "\x04slot\x18\x01 \x01(\x05R\x04slot\x12\x14\n" + + "\x05stick\x18\x02 \x01(\x05R\x05stick\x12\f\n" + + "\x01x\x18\x03 \x01(\x05R\x01x\x12\f\n" + + "\x01y\x18\x04 \x01(\x05R\x01y\"S\n" + "\x13ProtoControllerAxis\x12\x12\n" + - "\x04type\x18\x01 \x01(\tR\x04type\x12\x12\n" + - "\x04slot\x18\x02 \x01(\x05R\x04slot\x12\x12\n" + - "\x04axis\x18\x03 \x01(\x05R\x04axis\x12\x14\n" + - "\x05value\x18\x04 \x01(\x05R\x05value\"\xa7\x01\n" + + "\x04slot\x18\x01 \x01(\x05R\x04slot\x12\x12\n" + + "\x04axis\x18\x02 \x01(\x05R\x04axis\x12\x14\n" + + "\x05value\x18\x03 \x01(\x05R\x05value\"\x93\x01\n" + "\x15ProtoControllerRumble\x12\x12\n" + - "\x04type\x18\x01 \x01(\tR\x04type\x12\x12\n" + - "\x04slot\x18\x02 \x01(\x05R\x04slot\x12#\n" + - "\rlow_frequency\x18\x03 \x01(\x05R\flowFrequency\x12%\n" + - "\x0ehigh_frequency\x18\x04 \x01(\x05R\rhighFrequency\x12\x1a\n" + - "\bduration\x18\x05 \x01(\x05R\bduration\"\xc0\a\n" + + "\x04slot\x18\x01 \x01(\x05R\x04slot\x12#\n" + + "\rlow_frequency\x18\x02 \x01(\x05R\flowFrequency\x12%\n" + + "\x0ehigh_frequency\x18\x03 \x01(\x05R\rhighFrequency\x12\x1a\n" + + "\bduration\x18\x04 \x01(\x05R\bduration\"\xde\x01\n" + + "\x13RTCIceCandidateInit\x12\x1c\n" + + "\tcandidate\x18\x01 \x01(\tR\tcandidate\x12)\n" + + "\rsdpMLineIndex\x18\x02 \x01(\rH\x00R\rsdpMLineIndex\x88\x01\x01\x12\x1b\n" + + "\x06sdpMid\x18\x03 \x01(\tH\x01R\x06sdpMid\x88\x01\x01\x12/\n" + + "\x10usernameFragment\x18\x04 \x01(\tH\x02R\x10usernameFragment\x88\x01\x01B\x10\n" + + "\x0e_sdpMLineIndexB\t\n" + + "\a_sdpMidB\x13\n" + + "\x11_usernameFragment\"A\n" + + "\x19RTCSessionDescriptionInit\x12\x10\n" + + "\x03sdp\x18\x01 \x01(\tR\x03sdp\x12\x12\n" + + "\x04type\x18\x02 \x01(\tR\x04type\"D\n" + + "\bProtoICE\x128\n" + + "\tcandidate\x18\x01 \x01(\v2\x1a.proto.RTCIceCandidateInitR\tcandidate\">\n" + + "\bProtoSDP\x122\n" + + "\x03sdp\x18\x01 \x01(\v2 .proto.RTCSessionDescriptionInitR\x03sdp\"\x1e\n" + + "\bProtoRaw\x12\x12\n" + + "\x04data\x18\x01 \x01(\tR\x04data\"Z\n" + + "\x1cProtoClientRequestRoomStream\x12\x1b\n" + + "\troom_name\x18\x01 \x01(\tR\broomName\x12\x1d\n" + "\n" + - "ProtoInput\x126\n" + + "session_id\x18\x02 \x01(\tR\tsessionId\"c\n" + + "\x17ProtoClientDisconnected\x12\x1d\n" + "\n" + - "mouse_move\x18\x01 \x01(\v2\x15.proto.ProtoMouseMoveH\x00R\tmouseMove\x12@\n" + - "\x0emouse_move_abs\x18\x02 \x01(\v2\x18.proto.ProtoMouseMoveAbsH\x00R\fmouseMoveAbs\x129\n" + - "\vmouse_wheel\x18\x03 \x01(\v2\x16.proto.ProtoMouseWheelH\x00R\n" + - "mouseWheel\x12@\n" + - "\x0emouse_key_down\x18\x04 \x01(\v2\x18.proto.ProtoMouseKeyDownH\x00R\fmouseKeyDown\x12:\n" + - "\fmouse_key_up\x18\x05 \x01(\v2\x16.proto.ProtoMouseKeyUpH\x00R\n" + - "mouseKeyUp\x120\n" + - "\bkey_down\x18\x06 \x01(\v2\x13.proto.ProtoKeyDownH\x00R\akeyDown\x12*\n" + - "\x06key_up\x18\a \x01(\v2\x11.proto.ProtoKeyUpH\x00R\x05keyUp\x12K\n" + - "\x11controller_attach\x18\b \x01(\v2\x1c.proto.ProtoControllerAttachH\x00R\x10controllerAttach\x12K\n" + - "\x11controller_detach\x18\t \x01(\v2\x1c.proto.ProtoControllerDetachH\x00R\x10controllerDetach\x12K\n" + - "\x11controller_button\x18\n" + - " \x01(\v2\x1c.proto.ProtoControllerButtonH\x00R\x10controllerButton\x12N\n" + - "\x12controller_trigger\x18\v \x01(\v2\x1d.proto.ProtoControllerTriggerH\x00R\x11controllerTrigger\x12H\n" + - "\x10controller_stick\x18\f \x01(\v2\x1b.proto.ProtoControllerStickH\x00R\x0fcontrollerStick\x12E\n" + - "\x0fcontroller_axis\x18\r \x01(\v2\x1a.proto.ProtoControllerAxisH\x00R\x0econtrollerAxis\x12K\n" + - "\x11controller_rumble\x18\x0e \x01(\v2\x1c.proto.ProtoControllerRumbleH\x00R\x10controllerRumbleB\f\n" + - "\n" + - "input_typeB\x16Z\x14relay/internal/protob\x06proto3" + "session_id\x18\x01 \x01(\tR\tsessionId\x12)\n" + + "\x10controller_slots\x18\x02 \x03(\x05R\x0fcontrollerSlots\"4\n" + + "\x15ProtoServerPushStream\x12\x1b\n" + + "\troom_name\x18\x01 \x01(\tR\broomNameB\x16Z\x14relay/internal/protob\x06proto3" var ( file_types_proto_rawDescOnce sync.Once @@ -1265,44 +1285,39 @@ func file_types_proto_rawDescGZIP() []byte { return file_types_proto_rawDescData } -var file_types_proto_msgTypes = make([]protoimpl.MessageInfo, 15) +var file_types_proto_msgTypes = make([]protoimpl.MessageInfo, 22) var file_types_proto_goTypes = []any{ - (*ProtoMouseMove)(nil), // 0: proto.ProtoMouseMove - (*ProtoMouseMoveAbs)(nil), // 1: proto.ProtoMouseMoveAbs - (*ProtoMouseWheel)(nil), // 2: proto.ProtoMouseWheel - (*ProtoMouseKeyDown)(nil), // 3: proto.ProtoMouseKeyDown - (*ProtoMouseKeyUp)(nil), // 4: proto.ProtoMouseKeyUp - (*ProtoKeyDown)(nil), // 5: proto.ProtoKeyDown - (*ProtoKeyUp)(nil), // 6: proto.ProtoKeyUp - (*ProtoControllerAttach)(nil), // 7: proto.ProtoControllerAttach - (*ProtoControllerDetach)(nil), // 8: proto.ProtoControllerDetach - (*ProtoControllerButton)(nil), // 9: proto.ProtoControllerButton - (*ProtoControllerTrigger)(nil), // 10: proto.ProtoControllerTrigger - (*ProtoControllerStick)(nil), // 11: proto.ProtoControllerStick - (*ProtoControllerAxis)(nil), // 12: proto.ProtoControllerAxis - (*ProtoControllerRumble)(nil), // 13: proto.ProtoControllerRumble - (*ProtoInput)(nil), // 14: proto.ProtoInput + (*ProtoMouseMove)(nil), // 0: proto.ProtoMouseMove + (*ProtoMouseMoveAbs)(nil), // 1: proto.ProtoMouseMoveAbs + (*ProtoMouseWheel)(nil), // 2: proto.ProtoMouseWheel + (*ProtoMouseKeyDown)(nil), // 3: proto.ProtoMouseKeyDown + (*ProtoMouseKeyUp)(nil), // 4: proto.ProtoMouseKeyUp + (*ProtoKeyDown)(nil), // 5: proto.ProtoKeyDown + (*ProtoKeyUp)(nil), // 6: proto.ProtoKeyUp + (*ProtoControllerAttach)(nil), // 7: proto.ProtoControllerAttach + (*ProtoControllerDetach)(nil), // 8: proto.ProtoControllerDetach + (*ProtoControllerButton)(nil), // 9: proto.ProtoControllerButton + (*ProtoControllerTrigger)(nil), // 10: proto.ProtoControllerTrigger + (*ProtoControllerStick)(nil), // 11: proto.ProtoControllerStick + (*ProtoControllerAxis)(nil), // 12: proto.ProtoControllerAxis + (*ProtoControllerRumble)(nil), // 13: proto.ProtoControllerRumble + (*RTCIceCandidateInit)(nil), // 14: proto.RTCIceCandidateInit + (*RTCSessionDescriptionInit)(nil), // 15: proto.RTCSessionDescriptionInit + (*ProtoICE)(nil), // 16: proto.ProtoICE + (*ProtoSDP)(nil), // 17: proto.ProtoSDP + (*ProtoRaw)(nil), // 18: proto.ProtoRaw + (*ProtoClientRequestRoomStream)(nil), // 19: proto.ProtoClientRequestRoomStream + (*ProtoClientDisconnected)(nil), // 20: proto.ProtoClientDisconnected + (*ProtoServerPushStream)(nil), // 21: proto.ProtoServerPushStream } var file_types_proto_depIdxs = []int32{ - 0, // 0: proto.ProtoInput.mouse_move:type_name -> proto.ProtoMouseMove - 1, // 1: proto.ProtoInput.mouse_move_abs:type_name -> proto.ProtoMouseMoveAbs - 2, // 2: proto.ProtoInput.mouse_wheel:type_name -> proto.ProtoMouseWheel - 3, // 3: proto.ProtoInput.mouse_key_down:type_name -> proto.ProtoMouseKeyDown - 4, // 4: proto.ProtoInput.mouse_key_up:type_name -> proto.ProtoMouseKeyUp - 5, // 5: proto.ProtoInput.key_down:type_name -> proto.ProtoKeyDown - 6, // 6: proto.ProtoInput.key_up:type_name -> proto.ProtoKeyUp - 7, // 7: proto.ProtoInput.controller_attach:type_name -> proto.ProtoControllerAttach - 8, // 8: proto.ProtoInput.controller_detach:type_name -> proto.ProtoControllerDetach - 9, // 9: proto.ProtoInput.controller_button:type_name -> proto.ProtoControllerButton - 10, // 10: proto.ProtoInput.controller_trigger:type_name -> proto.ProtoControllerTrigger - 11, // 11: proto.ProtoInput.controller_stick:type_name -> proto.ProtoControllerStick - 12, // 12: proto.ProtoInput.controller_axis:type_name -> proto.ProtoControllerAxis - 13, // 13: proto.ProtoInput.controller_rumble:type_name -> proto.ProtoControllerRumble - 14, // [14:14] is the sub-list for method output_type - 14, // [14:14] is the sub-list for method input_type - 14, // [14:14] is the sub-list for extension type_name - 14, // [14:14] is the sub-list for extension extendee - 0, // [0:14] is the sub-list for field type_name + 14, // 0: proto.ProtoICE.candidate:type_name -> proto.RTCIceCandidateInit + 15, // 1: proto.ProtoSDP.sdp:type_name -> proto.RTCSessionDescriptionInit + 2, // [2:2] is the sub-list for method output_type + 2, // [2:2] is the sub-list for method input_type + 2, // [2:2] is the sub-list for extension type_name + 2, // [2:2] is the sub-list for extension extendee + 0, // [0:2] is the sub-list for field type_name } func init() { file_types_proto_init() } @@ -1310,29 +1325,14 @@ func file_types_proto_init() { if File_types_proto != nil { return } - file_types_proto_msgTypes[14].OneofWrappers = []any{ - (*ProtoInput_MouseMove)(nil), - (*ProtoInput_MouseMoveAbs)(nil), - (*ProtoInput_MouseWheel)(nil), - (*ProtoInput_MouseKeyDown)(nil), - (*ProtoInput_MouseKeyUp)(nil), - (*ProtoInput_KeyDown)(nil), - (*ProtoInput_KeyUp)(nil), - (*ProtoInput_ControllerAttach)(nil), - (*ProtoInput_ControllerDetach)(nil), - (*ProtoInput_ControllerButton)(nil), - (*ProtoInput_ControllerTrigger)(nil), - (*ProtoInput_ControllerStick)(nil), - (*ProtoInput_ControllerAxis)(nil), - (*ProtoInput_ControllerRumble)(nil), - } + file_types_proto_msgTypes[14].OneofWrappers = []any{} type x struct{} out := protoimpl.TypeBuilder{ File: protoimpl.DescBuilder{ GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: unsafe.Slice(unsafe.StringData(file_types_proto_rawDesc), len(file_types_proto_rawDesc)), NumEnums: 0, - NumMessages: 15, + NumMessages: 22, NumExtensions: 0, NumServices: 0, }, diff --git a/packages/relay/internal/shared/participant.go b/packages/relay/internal/shared/participant.go index 08ea885b..6916be95 100644 --- a/packages/relay/internal/shared/participant.go +++ b/packages/relay/internal/shared/participant.go @@ -2,43 +2,59 @@ package shared import ( "fmt" + "log/slog" "relay/internal/common" "relay/internal/connections" + "github.com/libp2p/go-libp2p/core/peer" "github.com/oklog/ulid/v2" + "github.com/pion/rtp" "github.com/pion/webrtc/v4" ) type Participant struct { ID ulid.ULID + SessionID string // Track session for reconnection + PeerID peer.ID // libp2p peer ID PeerConnection *webrtc.PeerConnection DataChannel *connections.NestriDataChannel + + // Per-viewer tracks and channels + VideoTrack *webrtc.TrackLocalStaticRTP + AudioTrack *webrtc.TrackLocalStaticRTP + VideoChan chan *rtp.Packet + AudioChan chan *rtp.Packet } -func NewParticipant() (*Participant, error) { +func NewParticipant(sessionID string, peerID peer.ID) (*Participant, error) { id, err := common.NewULID() if err != nil { return nil, fmt.Errorf("failed to create ULID for Participant: %w", err) } return &Participant{ - ID: id, + ID: id, + SessionID: sessionID, + PeerID: peerID, + VideoChan: make(chan *rtp.Packet, 500), + AudioChan: make(chan *rtp.Packet, 100), }, nil } -func (p *Participant) addTrack(trackLocal *webrtc.TrackLocalStaticRTP) error { - rtpSender, err := p.PeerConnection.AddTrack(trackLocal) - if err != nil { - return err +// Close cleans up participant resources +func (p *Participant) Close() { + if p.VideoChan != nil { + close(p.VideoChan) + p.VideoChan = nil } - - go func() { - rtcpBuffer := make([]byte, 1400) - for { - if _, _, rtcpErr := rtpSender.Read(rtcpBuffer); rtcpErr != nil { - break - } + if p.AudioChan != nil { + close(p.AudioChan) + p.AudioChan = nil + } + if p.PeerConnection != nil { + err := p.PeerConnection.Close() + if err != nil { + slog.Error("Failed to close Participant PeerConnection", err) } - }() - - return nil + p.PeerConnection = nil + } } diff --git a/packages/relay/internal/shared/room.go b/packages/relay/internal/shared/room.go index e5bb75dd..5ef09361 100644 --- a/packages/relay/internal/shared/room.go +++ b/packages/relay/internal/shared/room.go @@ -4,9 +4,11 @@ import ( "log/slog" "relay/internal/common" "relay/internal/connections" + "time" "github.com/libp2p/go-libp2p/core/peer" "github.com/oklog/ulid/v2" + "github.com/pion/rtp" "github.com/pion/webrtc/v4" ) @@ -23,17 +25,31 @@ type Room struct { VideoTrack *webrtc.TrackLocalStaticRTP DataChannel *connections.NestriDataChannel Participants *common.SafeMap[ulid.ULID, *Participant] + + // Broadcast queues (unbuffered, fan-out happens async) + videoBroadcastChan chan *rtp.Packet + audioBroadcastChan chan *rtp.Packet + broadcastStop chan struct{} } func NewRoom(name string, roomID ulid.ULID, ownerID peer.ID) *Room { - return &Room{ + r := &Room{ RoomInfo: RoomInfo{ ID: roomID, Name: name, OwnerID: ownerID, }, - Participants: common.NewSafeMap[ulid.ULID, *Participant](), + Participants: common.NewSafeMap[ulid.ULID, *Participant](), + videoBroadcastChan: make(chan *rtp.Packet, 1000), // Large buffer for incoming packets + audioBroadcastChan: make(chan *rtp.Packet, 500), + broadcastStop: make(chan struct{}), } + + // Start async broadcasters + go r.videoBroadcaster() + go r.audioBroadcaster() + + return r } // AddParticipant adds a Participant to a Room @@ -42,8 +58,8 @@ func (r *Room) AddParticipant(participant *Participant) { r.Participants.Set(participant.ID, participant) } -// Removes a Participant from a Room by participant's ID -func (r *Room) removeParticipantByID(pID ulid.ULID) { +// RemoveParticipantByID removes a Participant from a Room by participant's ID +func (r *Room) RemoveParticipantByID(pID ulid.ULID) { if _, ok := r.Participants.Get(pID); ok { r.Participants.Delete(pID) } @@ -64,3 +80,92 @@ func (r *Room) SetTrack(trackType webrtc.RTPCodecType, track *webrtc.TrackLocalS slog.Warn("Unknown track type", "room", r.Name, "trackType", trackType) } } + +// BroadcastPacket enqueues packet for async broadcast (non-blocking) +func (r *Room) BroadcastPacket(kind webrtc.RTPCodecType, pkt *rtp.Packet) { + start := time.Now() + if kind == webrtc.RTPCodecTypeVideo { + select { + case r.videoBroadcastChan <- pkt: + duration := time.Since(start) + if duration > 10*time.Millisecond { + slog.Warn("Slow video broadcast enqueue", "duration", duration, "room", r.Name) + } + default: + // Broadcast queue full - system overload, drop packet globally + slog.Warn("Video broadcast queue full, dropping packet", "room", r.Name) + } + } else { + select { + case r.audioBroadcastChan <- pkt: + duration := time.Since(start) + if duration > 10*time.Millisecond { + slog.Warn("Slow audio broadcast enqueue", "duration", duration, "room", r.Name) + } + default: + slog.Warn("Audio broadcast queue full, dropping packet", "room", r.Name) + } + } +} + +// Close stops the broadcasters +func (r *Room) Close() { + close(r.broadcastStop) + close(r.videoBroadcastChan) + close(r.audioBroadcastChan) +} + +// videoBroadcaster runs async fan-out for video packets +func (r *Room) videoBroadcaster() { + for { + select { + case pkt := <-r.videoBroadcastChan: + // Fan out to all participants without blocking + r.Participants.Range(func(_ ulid.ULID, participant *Participant) bool { + if participant.VideoChan != nil { + // Clone packet for each participant to avoid shared pointer issues + clonedPkt := pkt.Clone() + select { + case participant.VideoChan <- clonedPkt: + // Sent + default: + // Participant slow, drop packet + slog.Debug("Dropped video packet for slow participant", + "room", r.Name, + "participant", participant.ID) + } + } + return true + }) + case <-r.broadcastStop: + return + } + } +} + +// audioBroadcaster runs async fan-out for audio packets +func (r *Room) audioBroadcaster() { + for { + select { + case pkt := <-r.audioBroadcastChan: + r.Participants.Range(func(_ ulid.ULID, participant *Participant) bool { + if participant.AudioChan != nil { + // Clone packet for each participant to avoid shared pointer issues + clonedPkt := pkt.Clone() + select { + case participant.AudioChan <- clonedPkt: + // Sent + default: + // Participant slow, drop packet + slog.Debug("Dropped audio packet for slow participant", + "room", r.Name, + "participant", participant.ID) + } + } + return true + }) + case <-r.broadcastStop: + return + } + } +} diff --git a/packages/server/Cargo.lock b/packages/server/Cargo.lock index b574c136..d8aec525 100644 --- a/packages/server/Cargo.lock +++ b/packages/server/Cargo.lock @@ -3151,6 +3151,7 @@ dependencies = [ "tokio-stream", "tracing", "tracing-subscriber", + "unsigned-varint 0.8.0", "vimputti", "webrtc", ] diff --git a/packages/server/Cargo.toml b/packages/server/Cargo.toml index 853844f0..ce6d11c6 100644 --- a/packages/server/Cargo.toml +++ b/packages/server/Cargo.toml @@ -40,3 +40,4 @@ libp2p-tcp = { version = "0.44", features = ["tokio"] } libp2p-websocket = "0.45" dashmap = "6.1" anyhow = "1.0" +unsigned-varint = "0.8" diff --git a/packages/server/src/input/controller.rs b/packages/server/src/input/controller.rs index 891a13d5..656c6d46 100644 --- a/packages/server/src/input/controller.rs +++ b/packages/server/src/input/controller.rs @@ -1,7 +1,5 @@ -use crate::proto::proto::proto_input::InputType::{ - ControllerAttach, ControllerAxis, ControllerButton, ControllerDetach, ControllerRumble, - ControllerStick, ControllerTrigger, -}; +use crate::proto::proto::ProtoControllerAttach; +use crate::proto::proto::proto_message::Payload; use anyhow::Result; use std::collections::HashMap; use std::sync::Arc; @@ -48,158 +46,235 @@ impl ControllerInput { pub struct ControllerManager { vimputti_client: Arc, - cmd_tx: mpsc::Sender, + cmd_tx: mpsc::Sender, rumble_tx: mpsc::Sender<(u32, u16, u16, u16)>, // (slot, strong, weak, duration_ms) + attach_tx: mpsc::Sender, } impl ControllerManager { pub fn new( vimputti_client: Arc, - ) -> Result<(Self, mpsc::Receiver<(u32, u16, u16, u16)>)> { - let (cmd_tx, cmd_rx) = mpsc::channel(100); - let (rumble_tx, rumble_rx) = mpsc::channel(100); + ) -> Result<( + Self, + mpsc::Receiver<(u32, u16, u16, u16)>, + mpsc::Receiver, + )> { + let (cmd_tx, cmd_rx) = mpsc::channel(512); + let (rumble_tx, rumble_rx) = mpsc::channel(256); + let (attach_tx, attach_rx) = mpsc::channel(64); tokio::spawn(command_loop( cmd_rx, vimputti_client.clone(), rumble_tx.clone(), + attach_tx.clone(), )); Ok(( Self { vimputti_client, cmd_tx, rumble_tx, + attach_tx, }, rumble_rx, + attach_rx, )) } - pub async fn send_command(&self, input: crate::proto::proto::ProtoInput) -> Result<()> { - self.cmd_tx.send(input).await?; + pub async fn send_command(&self, payload: Payload) -> Result<()> { + self.cmd_tx.send(payload).await?; Ok(()) } } +struct ControllerSlot { + controller: ControllerInput, + session_id: String, + last_activity: std::time::Instant, +} + +// Returns first free controller slot from 0-7 +fn get_free_slot(controllers: &HashMap) -> Option { + for slot in 0..8 { + if !controllers.contains_key(&slot) { + return Some(slot); + } + } + None +} + async fn command_loop( - mut cmd_rx: mpsc::Receiver, + mut cmd_rx: mpsc::Receiver, vimputti_client: Arc, rumble_tx: mpsc::Sender<(u32, u16, u16, u16)>, + attach_tx: mpsc::Sender, ) { - let mut controllers: HashMap = HashMap::new(); - while let Some(input) = cmd_rx.recv().await { - if let Some(input_type) = input.input_type { - match input_type { - ControllerAttach(data) => { - // Check if controller already exists in the slot, if so, ignore - if controllers.contains_key(&(data.slot as u32)) { - tracing::warn!( - "Controller slot {} already occupied, ignoring attach", - data.slot - ); - } else { - if let Ok(mut controller) = - ControllerInput::new(data.id.clone(), &vimputti_client).await - { - let slot = data.slot as u32; - let rumble_tx = rumble_tx.clone(); + let mut controllers: HashMap = HashMap::new(); + while let Some(payload) = cmd_rx.recv().await { + match payload { + Payload::ControllerAttach(data) => { + let session_id = data.session_id.clone(); + + // Check if this session already has a slot (reconnection) + let existing_slot = controllers + .iter() + .find(|(_, slot)| slot.session_id == session_id && !session_id.is_empty()) + .map(|(slot_num, _)| *slot_num); - controller - .device_mut() - .on_rumble(move |strong, weak, duration_ms| { - let _ = rumble_tx.try_send((slot, strong, weak, duration_ms)); - }) - .await - .map_err(|e| { - tracing::warn!( - "Failed to register rumble callback for slot {}: {}", - slot, - e - ); - }) - .ok(); + let slot = existing_slot.or_else(|| get_free_slot(&controllers)); - controllers.insert(data.slot as u32, controller); - tracing::info!("Controller {} attached to slot {}", data.id, data.slot); - } else { - tracing::error!( - "Failed to create controller of type {} for slot {}", - data.id, - data.slot - ); + if let Some(slot) = slot { + if let Ok(mut controller) = + ControllerInput::new(data.id.clone(), &vimputti_client).await + { + let rumble_tx = rumble_tx.clone(); + let attach_tx = attach_tx.clone(); + + controller + .device_mut() + .on_rumble(move |strong, weak, duration_ms| { + let _ = rumble_tx.try_send((slot, strong, weak, duration_ms)); + }) + .await + .map_err(|e| { + tracing::warn!( + "Failed to register rumble callback for slot {}: {}", + slot, + e + ); + }) + .ok(); + + // Return to attach_tx what slot was assigned + let attach_info = ProtoControllerAttach { + id: data.id.clone(), + slot: slot as i32, + session_id: session_id.clone(), + }; + + match attach_tx.send(attach_info).await { + Ok(_) => { + controllers.insert( + slot, + ControllerSlot { + controller, + session_id: session_id.clone(), + last_activity: std::time::Instant::now(), + }, + ); + tracing::info!( + "Controller {} attached to slot {} (session: {})", + data.id, + slot, + session_id + ); + } + Err(e) => { + tracing::error!( + "Failed to send attach info for slot {}: {}", + slot, + e + ); + } } - } - } - ControllerDetach(data) => { - if controllers.remove(&(data.slot as u32)).is_some() { - tracing::info!("Controller detached from slot {}", data.slot); } else { - tracing::warn!("No controller found in slot {} to detach", data.slot); + tracing::error!( + "Failed to create controller of type {} for slot {}", + data.id, + slot + ); } } - ControllerButton(data) => { - if let Some(controller) = controllers.get(&(data.slot as u32)) { - if let Some(button) = vimputti::Button::from_ev_code(data.button as u16) { - let device = controller.device(); - device.button(button, data.pressed); - device.sync(); - } - } else { - tracing::warn!("Controller slot {} not found for button event", data.slot); - } + } + Payload::ControllerDetach(data) => { + if controllers.remove(&(data.slot as u32)).is_some() { + tracing::info!("Controller detached from slot {}", data.slot); + } else { + tracing::warn!("No controller found in slot {} to detach", data.slot); } - ControllerStick(data) => { - if let Some(controller) = controllers.get(&(data.slot as u32)) { - let device = controller.device(); - if data.stick == 0 { - // Left stick - device.axis(vimputti::Axis::LeftStickX, data.x); - device.sync(); - device.axis(vimputti::Axis::LeftStickY, data.y); - } else if data.stick == 1 { - // Right stick - device.axis(vimputti::Axis::RightStickX, data.x); - device.sync(); - device.axis(vimputti::Axis::RightStickY, data.y); - } + } + Payload::ControllerButton(data) => { + if let Some(controller) = controllers.get(&(data.slot as u32)) { + if let Some(button) = vimputti::Button::from_ev_code(data.button as u16) { + let device = controller.controller.device(); + device.button(button, data.pressed); device.sync(); - } else { - tracing::warn!("Controller slot {} not found for stick event", data.slot); } + } else { + tracing::warn!("Controller slot {} not found for button event", data.slot); } - ControllerTrigger(data) => { - if let Some(controller) = controllers.get(&(data.slot as u32)) { - let device = controller.device(); - if data.trigger == 0 { - // Left trigger - device.axis(vimputti::Axis::LowerLeftTrigger, data.value); - } else if data.trigger == 1 { - // Right trigger - device.axis(vimputti::Axis::LowerRightTrigger, data.value); - } + } + Payload::ControllerStick(data) => { + if let Some(controller) = controllers.get(&(data.slot as u32)) { + let device = controller.controller.device(); + if data.stick == 0 { + // Left stick + device.axis(vimputti::Axis::LeftStickX, data.x); device.sync(); - } else { - tracing::warn!("Controller slot {} not found for trigger event", data.slot); + device.axis(vimputti::Axis::LeftStickY, data.y); + } else if data.stick == 1 { + // Right stick + device.axis(vimputti::Axis::RightStickX, data.x); + device.sync(); + device.axis(vimputti::Axis::RightStickY, data.y); } + device.sync(); + } else { + tracing::warn!("Controller slot {} not found for stick event", data.slot); } - ControllerAxis(data) => { - if let Some(controller) = controllers.get(&(data.slot as u32)) { - let device = controller.device(); - if data.axis == 0 { - // dpad x - device.axis(vimputti::Axis::DPadX, data.value); - } else if data.axis == 1 { - // dpad y - device.axis(vimputti::Axis::DPadY, data.value); - } - device.sync(); + } + Payload::ControllerTrigger(data) => { + if let Some(controller) = controllers.get(&(data.slot as u32)) { + let device = controller.controller.device(); + if data.trigger == 0 { + // Left trigger + device.axis(vimputti::Axis::LowerLeftTrigger, data.value); + } else if data.trigger == 1 { + // Right trigger + device.axis(vimputti::Axis::LowerRightTrigger, data.value); } + device.sync(); + } else { + tracing::warn!("Controller slot {} not found for trigger event", data.slot); } - // Rumble will be outgoing event.. - ControllerRumble(_) => { - //no-op + } + Payload::ControllerAxis(data) => { + if let Some(controller) = controllers.get(&(data.slot as u32)) { + let device = controller.controller.device(); + if data.axis == 0 { + // dpad x + device.axis(vimputti::Axis::DPadX, data.value); + } else if data.axis == 1 { + // dpad y + device.axis(vimputti::Axis::DPadY, data.value); + } + device.sync(); } - _ => { - //no-op + } + Payload::ClientDisconnected(data) => { + tracing::info!( + "Client disconnected, cleaning up controller slots: {:?} (client session: {})", + data.controller_slots, + data.session_id + ); + // Remove all controllers for the disconnected slots + for slot in &data.controller_slots { + if controllers.remove(&(*slot as u32)).is_some() { + tracing::info!( + "Removed controller from slot {} (client session: {})", + slot, + data.session_id + ); + } else { + tracing::warn!( + "No controller found in slot {} to cleanup (client session: {})", + slot, + data.session_id + ); + } } } + _ => { + //no-op + } } } } diff --git a/packages/server/src/main.rs b/packages/server/src/main.rs index c30b67ba..3427b71c 100644 --- a/packages/server/src/main.rs +++ b/packages/server/src/main.rs @@ -3,7 +3,6 @@ mod enc_helper; mod gpu; mod input; mod latency; -mod messages; mod nestrisink; mod p2p; mod proto; @@ -257,11 +256,15 @@ async fn main() -> Result<(), Box> { None } }; - let (controller_manager, rumble_rx) = if let Some(vclient) = vimputti_client { - let (controller_manager, rumble_rx) = ControllerManager::new(vclient)?; - (Some(Arc::new(controller_manager)), Some(rumble_rx)) + let (controller_manager, rumble_rx, attach_rx) = if let Some(vclient) = vimputti_client { + let (controller_manager, rumble_rx, attach_rx) = ControllerManager::new(vclient)?; + ( + Some(Arc::new(controller_manager)), + Some(rumble_rx), + Some(attach_rx), + ) } else { - (None, None) + (None, None, None) }; /*** PIPELINE CREATION ***/ @@ -416,6 +419,7 @@ async fn main() -> Result<(), Box> { video_source.clone(), controller_manager, rumble_rx, + attach_rx, ) .await?; let webrtcsink = BaseWebRTCSink::with_signaller(Signallable::from(signaller.clone())); @@ -550,7 +554,7 @@ async fn main() -> Result<(), Box> { } // Make sure QOS is disabled to avoid latency - video_encoder.set_property("qos", false); + video_encoder.set_property("qos", true); // Optimize latency of pipeline video_source diff --git a/packages/server/src/messages.rs b/packages/server/src/messages.rs deleted file mode 100644 index 21938bd3..00000000 --- a/packages/server/src/messages.rs +++ /dev/null @@ -1,50 +0,0 @@ -use crate::latency::LatencyTracker; -use serde::{Deserialize, Serialize}; -use webrtc::ice_transport::ice_candidate::RTCIceCandidateInit; -use webrtc::peer_connection::sdp::session_description::RTCSessionDescription; - -#[derive(Serialize, Deserialize, Debug)] -pub struct MessageBase { - pub payload_type: String, - pub latency: Option, -} - -#[derive(Serialize, Deserialize, Debug)] -pub struct MessageRaw { - #[serde(flatten)] - pub base: MessageBase, - pub data: serde_json::Value, -} - -#[derive(Serialize, Deserialize, Debug)] -pub struct MessageLog { - #[serde(flatten)] - pub base: MessageBase, - pub level: String, - pub message: String, - pub time: String, -} - -#[derive(Serialize, Deserialize, Debug)] -pub struct MessageMetrics { - #[serde(flatten)] - pub base: MessageBase, - pub usage_cpu: f64, - pub usage_memory: f64, - pub uptime: u64, - pub pipeline_latency: f64, -} - -#[derive(Serialize, Deserialize, Debug)] -pub struct MessageICE { - #[serde(flatten)] - pub base: MessageBase, - pub candidate: RTCIceCandidateInit, -} - -#[derive(Serialize, Deserialize, Debug)] -pub struct MessageSDP { - #[serde(flatten)] - pub base: MessageBase, - pub sdp: RTCSessionDescription, -} diff --git a/packages/server/src/nestrisink/imp.rs b/packages/server/src/nestrisink/imp.rs index 359a219c..f0da654c 100644 --- a/packages/server/src/nestrisink/imp.rs +++ b/packages/server/src/nestrisink/imp.rs @@ -1,11 +1,11 @@ use crate::input::controller::ControllerManager; -use crate::messages::{MessageBase, MessageICE, MessageRaw, MessageSDP}; use crate::p2p::p2p::NestriConnection; use crate::p2p::p2p_protocol_stream::NestriStreamProtocol; -use crate::proto::proto::proto_input::InputType::{ - KeyDown, KeyUp, MouseKeyDown, MouseKeyUp, MouseMove, MouseMoveAbs, MouseWheel, +use crate::proto::proto::proto_message::Payload; +use crate::proto::proto::{ + ProtoControllerAttach, ProtoControllerRumble, ProtoIce, ProtoMessage, ProtoSdp, + ProtoServerPushStream, RtcIceCandidateInit, RtcSessionDescriptionInit, }; -use crate::proto::proto::{ProtoInput, ProtoMessageInput}; use anyhow::Result; use glib::subclass::prelude::*; use gstreamer::glib; @@ -16,8 +16,6 @@ use parking_lot::RwLock as PLRwLock; use prost::Message; use std::sync::{Arc, LazyLock}; use tokio::sync::{Mutex, mpsc}; -use webrtc::ice_transport::ice_candidate::RTCIceCandidateInit; -use webrtc::peer_connection::sdp::session_description::RTCSessionDescription; pub struct Signaller { stream_room: PLRwLock>, @@ -26,6 +24,7 @@ pub struct Signaller { data_channel: PLRwLock>>, controller_manager: PLRwLock>>, rumble_rx: Mutex>>, + attach_rx: Mutex>>, } impl Default for Signaller { fn default() -> Self { @@ -36,6 +35,7 @@ impl Default for Signaller { data_channel: PLRwLock::new(None), controller_manager: PLRwLock::new(None), rumble_rx: Mutex::new(None), + attach_rx: Mutex::new(None), } } } @@ -74,11 +74,23 @@ impl Signaller { *self.rumble_rx.lock().await = Some(rumble_rx); } - // Change getter to take ownership: pub async fn take_rumble_rx(&self) -> Option> { self.rumble_rx.lock().await.take() } + pub async fn set_attach_rx( + &self, + attach_rx: mpsc::Receiver, + ) { + *self.attach_rx.lock().await = Some(attach_rx); + } + + pub async fn take_attach_rx( + &self, + ) -> Option> { + self.attach_rx.lock().await.take() + } + pub fn set_data_channel(&self, data_channel: gstreamer_webrtc::WebRTCDataChannel) { *self.data_channel.write() = Some(Arc::new(data_channel)); } @@ -95,68 +107,85 @@ impl Signaller { }; { let self_obj = self.obj().clone(); - stream_protocol.register_callback("answer", move |data| { - if let Ok(message) = serde_json::from_slice::(&data) { - let sdp = gst_sdp::SDPMessage::parse_buffer(message.sdp.sdp.as_bytes()) - .map_err(|e| anyhow::anyhow!("Invalid SDP in 'answer': {e:?}"))?; - let answer = WebRTCSessionDescription::new(WebRTCSDPType::Answer, sdp); - Ok(self_obj.emit_by_name::<()>( - "session-description", - &[&"unique-session-id", &answer], - )) + stream_protocol.register_callback("answer", move |msg| { + if let Some(payload) = msg.payload { + match payload { + Payload::Sdp(sdp) => { + if let Some(sdp) = sdp.sdp { + let sdp = gst_sdp::SDPMessage::parse_buffer(sdp.sdp.as_bytes()) + .map_err(|e| { + anyhow::anyhow!("Invalid SDP in 'answer': {e:?}") + })?; + let answer = + WebRTCSessionDescription::new(WebRTCSDPType::Answer, sdp); + return Ok(self_obj.emit_by_name::<()>( + "session-description", + &[&"unique-session-id", &answer], + )); + } + } + _ => { + tracing::warn!("Unexpected payload type for answer"); + return Ok(()); + } + } } else { - anyhow::bail!("Failed to decode SDP message"); + anyhow::bail!("Failed to decode answer message"); } + Ok(()) }); } { let self_obj = self.obj().clone(); - stream_protocol.register_callback("ice-candidate", move |data| { - if let Ok(message) = serde_json::from_slice::(&data) { - let candidate = message.candidate; - let sdp_m_line_index = candidate.sdp_mline_index.unwrap_or(0) as u32; - let sdp_mid = candidate.sdp_mid; - Ok(self_obj.emit_by_name::<()>( - "handle-ice", - &[ - &"unique-session-id", - &sdp_m_line_index, - &sdp_mid, - &candidate.candidate, - ], - )) + stream_protocol.register_callback("ice-candidate", move |msg| { + if let Some(payload) = msg.payload { + match payload { + Payload::Ice(ice) => { + if let Some(candidate) = ice.candidate { + let sdp_m_line_index = candidate.sdp_m_line_index.unwrap_or(0); + return Ok(self_obj.emit_by_name::<()>( + "handle-ice", + &[ + &"unique-session-id", + &sdp_m_line_index, + &candidate.sdp_mid, + &candidate.candidate, + ], + )); + } + } + _ => { + tracing::warn!("Unexpected payload type for ice-candidate"); + return Ok(()); + } + } } else { anyhow::bail!("Failed to decode ICE message"); } + Ok(()) }); } { let self_obj = self.obj().clone(); - stream_protocol.register_callback("push-stream-ok", move |data| { - if let Ok(answer) = serde_json::from_slice::(&data) { - // Decode room name string - if let Some(room_name) = answer.data.as_str() { - gstreamer::info!( - gstreamer::CAT_DEFAULT, - "Received OK answer for room: {}", - room_name - ); - } else { - gstreamer::error!( - gstreamer::CAT_DEFAULT, - "Failed to decode room name from answer" - ); - } - - // Send our SDP offer - Ok(self_obj.emit_by_name::<()>( - "session-requested", - &[ - &"unique-session-id", - &"consumer-identifier", - &None::, - ], - )) + stream_protocol.register_callback("push-stream-ok", move |msg| { + if let Some(payload) = msg.payload { + return match payload { + Payload::ServerPushStream(_res) => { + // Send our SDP offer + Ok(self_obj.emit_by_name::<()>( + "session-requested", + &[ + &"unique-session-id", + &"consumer-identifier", + &None::, + ], + )) + } + _ => { + tracing::warn!("Unexpected payload type for push-stream-ok"); + Ok(()) + } + }; } else { anyhow::bail!("Failed to decode answer"); } @@ -200,12 +229,14 @@ impl Signaller { // Spawn async task to take the receiver and set up tokio::spawn(async move { let rumble_rx = signaller.imp().take_rumble_rx().await; + let attach_rx = signaller.imp().take_attach_rx().await; let controller_manager = signaller.imp().get_controller_manager(); setup_data_channel( controller_manager, rumble_rx, + attach_rx, data_channel, &wayland_src, ); @@ -243,19 +274,18 @@ impl SignallableImpl for Signaller { return; }; - let push_msg = MessageRaw { - base: MessageBase { - payload_type: "push-stream-room".to_string(), - latency: None, - }, - data: serde_json::Value::from(stream_room), - }; - let Some(stream_protocol) = self.get_stream_protocol() else { gstreamer::error!(gstreamer::CAT_DEFAULT, "Stream protocol not set"); return; }; + let push_msg = crate::proto::create_message( + Payload::ServerPushStream(ProtoServerPushStream { + room_name: stream_room, + }), + "push-stream-room", + None, + ); if let Err(e) = stream_protocol.send_message(&push_msg) { tracing::error!("Failed to send push stream room message: {:?}", e); } @@ -266,20 +296,22 @@ impl SignallableImpl for Signaller { } fn send_sdp(&self, _session_id: &str, sdp: &WebRTCSessionDescription) { - let sdp_message = MessageSDP { - base: MessageBase { - payload_type: "offer".to_string(), - latency: None, - }, - sdp: RTCSessionDescription::offer(sdp.sdp().as_text().unwrap()).unwrap(), - }; - let Some(stream_protocol) = self.get_stream_protocol() else { gstreamer::error!(gstreamer::CAT_DEFAULT, "Stream protocol not set"); return; }; - if let Err(e) = stream_protocol.send_message(&sdp_message) { + let sdp_msg = crate::proto::create_message( + Payload::Sdp(ProtoSdp { + sdp: Some(RtcSessionDescriptionInit { + sdp: sdp.sdp().as_text().unwrap(), + r#type: "offer".to_string(), + }), + }), + "offer", + None, + ); + if let Err(e) = stream_protocol.send_message(&sdp_msg) { tracing::error!("Failed to send SDP message: {:?}", e); } } @@ -291,26 +323,25 @@ impl SignallableImpl for Signaller { sdp_m_line_index: u32, sdp_mid: Option, ) { - let candidate_init = RTCIceCandidateInit { - candidate: candidate.to_string(), - sdp_mid, - sdp_mline_index: Some(sdp_m_line_index as u16), - ..Default::default() - }; - let ice_message = MessageICE { - base: MessageBase { - payload_type: "ice-candidate".to_string(), - latency: None, - }, - candidate: candidate_init, - }; - let Some(stream_protocol) = self.get_stream_protocol() else { gstreamer::error!(gstreamer::CAT_DEFAULT, "Stream protocol not set"); return; }; - if let Err(e) = stream_protocol.send_message(&ice_message) { + let candidate_init = RtcIceCandidateInit { + candidate: candidate.to_string(), + sdp_mid, + sdp_m_line_index: Some(sdp_m_line_index), + ..Default::default() //username_fragment: Some(session_id.to_string()), TODO: required? + }; + let ice_msg = crate::proto::create_message( + Payload::Ice(ProtoIce { + candidate: Some(candidate_init), + }), + "ice-candidate", + None, + ); + if let Err(e) = stream_protocol.send_message(&ice_msg) { tracing::error!("Failed to send ICE candidate message: {:?}", e); } } @@ -352,6 +383,7 @@ impl ObjectImpl for Signaller { fn setup_data_channel( controller_manager: Option>, rumble_rx: Option>, // (slot, strong, weak, duration_ms) + attach_rx: Option>, data_channel: Arc, wayland_src: &gstreamer::Element, ) { @@ -361,11 +393,11 @@ fn setup_data_channel( // Spawn async processor tokio::spawn(async move { while let Some(data) = rx.recv().await { - match ProtoMessageInput::decode(data.as_slice()) { - Ok(message_input) => { - if let Some(message_base) = message_input.message_base { + match ProtoMessage::decode(data.as_slice()) { + Ok(msg_wrapper) => { + if let Some(message_base) = msg_wrapper.message_base { if message_base.payload_type == "input" { - if let Some(input_data) = message_input.data { + if let Some(input_data) = msg_wrapper.payload { if let Some(event) = handle_input_message(input_data) { // Send the event to wayland source, result bool is ignored let _ = wayland_src.send_event(event); @@ -373,7 +405,7 @@ fn setup_data_channel( } } else if message_base.payload_type == "controllerInput" { if let Some(controller_manager) = &controller_manager { - if let Some(input_data) = message_input.data { + if let Some(input_data) = msg_wrapper.payload { let _ = controller_manager.send_command(input_data).await; } } @@ -392,25 +424,16 @@ fn setup_data_channel( let data_channel_clone = data_channel.clone(); tokio::spawn(async move { while let Some((slot, strong, weak, duration_ms)) = rumble_rx.recv().await { - let rumble_msg = ProtoMessageInput { - message_base: Some(crate::proto::proto::ProtoMessageBase { - payload_type: "controllerInput".to_string(), - latency: None, - }), - data: Some(ProtoInput { - input_type: Some( - crate::proto::proto::proto_input::InputType::ControllerRumble( - crate::proto::proto::ProtoControllerRumble { - r#type: "ControllerRumble".to_string(), - slot: slot as i32, - low_frequency: weak as i32, - high_frequency: strong as i32, - duration: duration_ms as i32, - }, - ), - ), + let rumble_msg = crate::proto::create_message( + Payload::ControllerRumble(ProtoControllerRumble { + slot: slot as i32, + low_frequency: weak as i32, + high_frequency: strong as i32, + duration: duration_ms as i32, }), - }; + "controllerInput", + None, + ); let data = rumble_msg.encode_to_vec(); let bytes = glib::Bytes::from_owned(data); @@ -422,6 +445,27 @@ fn setup_data_channel( }); } + // Spawn attach sender + if let Some(mut attach_rx) = attach_rx { + let data_channel_clone = data_channel.clone(); + tokio::spawn(async move { + while let Some(attach_msg) = attach_rx.recv().await { + let proto_msg = crate::proto::create_message( + Payload::ControllerAttach(attach_msg), + "controllerInput", + None, + ); + + let data = proto_msg.encode_to_vec(); + let bytes = glib::Bytes::from_owned(data); + + if let Err(e) = data_channel_clone.send_data_full(Some(&bytes)) { + tracing::warn!("Failed to send controller attach data: {}", e); + } + } + }); + } + data_channel.connect_on_message_data(move |_data_channel, data| { if let Some(data) = data { let _ = tx.send(data.to_vec()); @@ -429,68 +473,64 @@ fn setup_data_channel( }); } -fn handle_input_message(input_msg: ProtoInput) -> Option { - if let Some(input_type) = input_msg.input_type { - match input_type { - MouseMove(data) => { - let structure = gstreamer::Structure::builder("MouseMoveRelative") - .field("pointer_x", data.x as f64) - .field("pointer_y", data.y as f64) - .build(); +fn handle_input_message(payload: Payload) -> Option { + match payload { + Payload::MouseMove(data) => { + let structure = gstreamer::Structure::builder("MouseMoveRelative") + .field("pointer_x", data.x as f64) + .field("pointer_y", data.y as f64) + .build(); - Some(gstreamer::event::CustomUpstream::new(structure)) - } - MouseMoveAbs(data) => { - let structure = gstreamer::Structure::builder("MouseMoveAbsolute") - .field("pointer_x", data.x as f64) - .field("pointer_y", data.y as f64) - .build(); + Some(gstreamer::event::CustomUpstream::new(structure)) + } + Payload::MouseMoveAbs(data) => { + let structure = gstreamer::Structure::builder("MouseMoveAbsolute") + .field("pointer_x", data.x as f64) + .field("pointer_y", data.y as f64) + .build(); - Some(gstreamer::event::CustomUpstream::new(structure)) - } - KeyDown(data) => { - let structure = gstreamer::Structure::builder("KeyboardKey") - .field("key", data.key as u32) - .field("pressed", true) - .build(); + Some(gstreamer::event::CustomUpstream::new(structure)) + } + Payload::KeyDown(data) => { + let structure = gstreamer::Structure::builder("KeyboardKey") + .field("key", data.key as u32) + .field("pressed", true) + .build(); - Some(gstreamer::event::CustomUpstream::new(structure)) - } - KeyUp(data) => { - let structure = gstreamer::Structure::builder("KeyboardKey") - .field("key", data.key as u32) - .field("pressed", false) - .build(); + Some(gstreamer::event::CustomUpstream::new(structure)) + } + Payload::KeyUp(data) => { + let structure = gstreamer::Structure::builder("KeyboardKey") + .field("key", data.key as u32) + .field("pressed", false) + .build(); - Some(gstreamer::event::CustomUpstream::new(structure)) - } - MouseWheel(data) => { - let structure = gstreamer::Structure::builder("MouseAxis") - .field("x", data.x as f64) - .field("y", data.y as f64) - .build(); + Some(gstreamer::event::CustomUpstream::new(structure)) + } + Payload::MouseWheel(data) => { + let structure = gstreamer::Structure::builder("MouseAxis") + .field("x", data.x as f64) + .field("y", data.y as f64) + .build(); - Some(gstreamer::event::CustomUpstream::new(structure)) - } - MouseKeyDown(data) => { - let structure = gstreamer::Structure::builder("MouseButton") - .field("button", data.key as u32) - .field("pressed", true) - .build(); + Some(gstreamer::event::CustomUpstream::new(structure)) + } + Payload::MouseKeyDown(data) => { + let structure = gstreamer::Structure::builder("MouseButton") + .field("button", data.key as u32) + .field("pressed", true) + .build(); - Some(gstreamer::event::CustomUpstream::new(structure)) - } - MouseKeyUp(data) => { - let structure = gstreamer::Structure::builder("MouseButton") - .field("button", data.key as u32) - .field("pressed", false) - .build(); + Some(gstreamer::event::CustomUpstream::new(structure)) + } + Payload::MouseKeyUp(data) => { + let structure = gstreamer::Structure::builder("MouseButton") + .field("button", data.key as u32) + .field("pressed", false) + .build(); - Some(gstreamer::event::CustomUpstream::new(structure)) - } - _ => None, + Some(gstreamer::event::CustomUpstream::new(structure)) } - } else { - None + _ => None, } } diff --git a/packages/server/src/nestrisink/mod.rs b/packages/server/src/nestrisink/mod.rs index ea9ac1b9..4de29964 100644 --- a/packages/server/src/nestrisink/mod.rs +++ b/packages/server/src/nestrisink/mod.rs @@ -19,6 +19,7 @@ impl NestriSignaller { wayland_src: Arc, controller_manager: Option>, rumble_rx: Option>, + attach_rx: Option>, ) -> Result> { let obj: Self = glib::Object::new(); obj.imp().set_stream_room(room); @@ -30,6 +31,9 @@ impl NestriSignaller { if let Some(rumble_rx) = rumble_rx { obj.imp().set_rumble_rx(rumble_rx).await; } + if let Some(attach_rx) = attach_rx { + obj.imp().set_attach_rx(attach_rx).await; + } Ok(obj) } } diff --git a/packages/server/src/p2p/p2p_protocol_stream.rs b/packages/server/src/p2p/p2p_protocol_stream.rs index 17265a7f..0f016925 100644 --- a/packages/server/src/p2p/p2p_protocol_stream.rs +++ b/packages/server/src/p2p/p2p_protocol_stream.rs @@ -3,21 +3,22 @@ use crate::p2p::p2p_safestream::SafeStream; use anyhow::Result; use dashmap::DashMap; use libp2p::StreamProtocol; +use prost::Message; use std::sync::Arc; use tokio::sync::mpsc; // Cloneable callback type -pub type CallbackInner = dyn Fn(Vec) -> Result<()> + Send + Sync + 'static; +pub type CallbackInner = dyn Fn(crate::proto::proto::ProtoMessage) -> Result<()> + Send + Sync + 'static; pub struct Callback(Arc); impl Callback { pub fn new(f: F) -> Self where - F: Fn(Vec) -> Result<()> + Send + Sync + 'static, + F: Fn(crate::proto::proto::ProtoMessage) -> Result<()> + Send + Sync + 'static, { Callback(Arc::new(f)) } - pub fn call(&self, data: Vec) -> Result<()> { + pub fn call(&self, data: crate::proto::proto::ProtoMessage) -> Result<()> { self.0(data) } } @@ -104,26 +105,31 @@ impl NestriStreamProtocol { } }; - match serde_json::from_slice::(&data) { - Ok(base_message) => { - let response_type = base_message.payload_type; - - // With DashMap, we don't need explicit locking - // we just get the callback directly if it exists - if let Some(callback) = callbacks.get(&response_type) { - // Execute the callback - if let Err(e) = callback.call(data.clone()) { - tracing::error!( - "Callback for response type '{}' errored: {:?}", - response_type, - e + match crate::proto::proto::ProtoMessage::decode(data.as_slice()) { + Ok(message) => { + if let Some(base_message) = &message.message_base { + let response_type = &base_message.payload_type; + let response_type = response_type.clone(); + + // With DashMap, we don't need explicit locking + // we just get the callback directly if it exists + if let Some(callback) = callbacks.get(&response_type) { + // Execute the callback + if let Err(e) = callback.call(message) { + tracing::error!( + "Callback for response type '{}' errored: {:?}", + response_type, + e + ); + } + } else { + tracing::warn!( + "No callback registered for response type: {}", + response_type ); } } else { - tracing::warn!( - "No callback registered for response type: {}", - response_type - ); + tracing::error!("No base message in decoded protobuf message",); } } Err(e) => { @@ -154,8 +160,9 @@ impl NestriStreamProtocol { }) } - pub fn send_message(&self, message: &M) -> Result<()> { - let json_data = serde_json::to_vec(message)?; + pub fn send_message(&self, message: &crate::proto::proto::ProtoMessage) -> Result<()> { + let mut buf = Vec::new(); + message.encode(&mut buf)?; let Some(tx) = &self.tx else { return Err(anyhow::Error::msg( if self.read_handle.is_none() && self.write_handle.is_none() { @@ -165,13 +172,13 @@ impl NestriStreamProtocol { }, )); }; - tx.try_send(json_data)?; + tx.try_send(buf)?; Ok(()) } pub fn register_callback(&self, response_type: &str, callback: F) where - F: Fn(Vec) -> Result<()> + Send + Sync + 'static, + F: Fn(crate::proto::proto::ProtoMessage) -> Result<()> + Send + Sync + 'static, { self.callbacks .insert(response_type.to_string(), Callback::new(callback)); diff --git a/packages/server/src/p2p/p2p_safestream.rs b/packages/server/src/p2p/p2p_safestream.rs index 05701fca..198aa50b 100644 --- a/packages/server/src/p2p/p2p_safestream.rs +++ b/packages/server/src/p2p/p2p_safestream.rs @@ -1,11 +1,9 @@ use anyhow::Result; -use byteorder::{BigEndian, ByteOrder}; use libp2p::futures::io::{ReadHalf, WriteHalf}; use libp2p::futures::{AsyncReadExt, AsyncWriteExt}; use std::sync::Arc; use tokio::sync::Mutex; - -const MAX_SIZE: usize = 1024 * 1024; // 1MB +use unsigned_varint::{decode, encode}; pub struct SafeStream { stream_read: Arc>>, @@ -29,34 +27,52 @@ impl SafeStream { } async fn send_with_length_prefix(&self, data: &[u8]) -> Result<()> { - if data.len() > MAX_SIZE { - anyhow::bail!("Data exceeds maximum size"); - } + let mut stream_write = self.stream_write.lock().await; - let mut buffer = Vec::with_capacity(4 + data.len()); - buffer.extend_from_slice(&(data.len() as u32).to_be_bytes()); // Length prefix - buffer.extend_from_slice(data); // Payload + // Encode length as varint + let mut length_buf = encode::usize_buffer(); + let length_bytes = encode::usize(data.len(), &mut length_buf); - let mut stream_write = self.stream_write.lock().await; - stream_write.write_all(&buffer).await?; // Single write + // Write varint length prefix + stream_write.write_all(length_bytes).await?; + + // Write payload + stream_write.write_all(data).await?; stream_write.flush().await?; + Ok(()) } async fn receive_with_length_prefix(&self) -> Result> { let mut stream_read = self.stream_read.lock().await; - // Read length prefix + data in one syscall - let mut length_prefix = [0u8; 4]; - stream_read.read_exact(&mut length_prefix).await?; - let length = BigEndian::read_u32(&length_prefix) as usize; + // Read varint length prefix (up to 10 bytes for u64) + let mut length_buf = Vec::new(); + let mut temp_byte = [0u8; 1]; + + loop { + stream_read.read_exact(&mut temp_byte).await?; + length_buf.push(temp_byte[0]); + + // Check if this is the last byte (MSB = 0) + if temp_byte[0] & 0x80 == 0 { + break; + } - if length > MAX_SIZE { - anyhow::bail!("Received data exceeds maximum size"); + // Protect against malicious infinite varints + if length_buf.len() > 10 { + anyhow::bail!("Invalid varint encoding"); + } } + // Decode the varint + let (length, _) = decode::usize(&length_buf) + .map_err(|e| anyhow::anyhow!("Failed to decode varint: {}", e))?; + + // Read payload let mut buffer = vec![0u8; length]; stream_read.read_exact(&mut buffer).await?; + Ok(buffer) } } diff --git a/packages/server/src/proto.rs b/packages/server/src/proto.rs index febacec6..57b0205c 100644 --- a/packages/server/src/proto.rs +++ b/packages/server/src/proto.rs @@ -1 +1,35 @@ pub mod proto; + +pub struct CreateMessageOptions { + pub sequence_id: Option, + pub latency: Option, +} + +pub fn create_message( + payload: proto::proto_message::Payload, + payload_type: impl Into, + options: Option, +) -> proto::ProtoMessage { + let opts = options.unwrap_or(CreateMessageOptions { + sequence_id: None, + latency: None, + }); + + let latency = opts.latency.or_else(|| { + opts.sequence_id.map(|seq_id| proto::ProtoLatencyTracker { + sequence_id: seq_id, + timestamps: vec![proto::ProtoTimestampEntry { + stage: "created".to_string(), + time: Some(prost_types::Timestamp::from(std::time::SystemTime::now())), + }], + }) + }); + + proto::ProtoMessage { + message_base: Some(proto::ProtoMessageBase { + payload_type: payload_type.into(), + latency, + }), + payload: Some(payload), + } +} diff --git a/packages/server/src/proto/gen.rs b/packages/server/src/proto/gen.rs deleted file mode 100644 index 04a9ab9d..00000000 --- a/packages/server/src/proto/gen.rs +++ /dev/null @@ -1,202 +0,0 @@ -// @generated -// This file is @generated by prost-build. -/// EntityState represents the state of an entity in the mesh (e.g., a room). -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct EntityState { - /// Type of entity (e.g., "room") - #[prost(string, tag="1")] - pub entity_type: ::prost::alloc::string::String, - /// Unique identifier (e.g., room name) - #[prost(string, tag="2")] - pub entity_id: ::prost::alloc::string::String, - /// Whether the entity is active - #[prost(bool, tag="3")] - pub active: bool, - /// Relay ID that owns this entity - #[prost(string, tag="4")] - pub owner_relay_id: ::prost::alloc::string::String, -} -/// MeshMessage is the top-level message for all relay-to-relay communication. -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct MeshMessage { - #[prost(oneof="mesh_message::Type", tags="1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13")] - pub r#type: ::core::option::Option, -} -/// Nested message and enum types in `MeshMessage`. -pub mod mesh_message { - #[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Oneof)] - pub enum Type { - /// Level 0 - #[prost(message, tag="1")] - StateUpdate(super::StateUpdate), - #[prost(message, tag="2")] - Ack(super::Ack), - #[prost(message, tag="3")] - RetransmissionRequest(super::RetransmissionRequest), - #[prost(message, tag="4")] - Retransmission(super::Retransmission), - #[prost(message, tag="5")] - Heartbeat(super::Heartbeat), - #[prost(message, tag="6")] - SuspectRelay(super::SuspectRelay), - #[prost(message, tag="7")] - Disconnect(super::Disconnect), - /// Level 1 - #[prost(message, tag="8")] - ForwardSdp(super::ForwardSdp), - #[prost(message, tag="9")] - ForwardIce(super::ForwardIce), - #[prost(message, tag="10")] - ForwardIngest(super::ForwardIngest), - #[prost(message, tag="11")] - StreamRequest(super::StreamRequest), - /// Level 2 - #[prost(message, tag="12")] - Handshake(super::Handshake), - #[prost(message, tag="13")] - HandshakeResponse(super::HandshakeResponse), - } -} -/// Handshake to inititiate new connection to mesh. -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct Handshake { - /// UUID of the relay - #[prost(string, tag="1")] - pub relay_id: ::prost::alloc::string::String, - /// base64 encoded Diffie-Hellman public key - #[prost(string, tag="2")] - pub dh_public_key: ::prost::alloc::string::String, -} -/// HandshakeResponse to respond to a mesh joiner. -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct HandshakeResponse { - #[prost(string, tag="1")] - pub relay_id: ::prost::alloc::string::String, - #[prost(string, tag="2")] - pub dh_public_key: ::prost::alloc::string::String, - /// relay id to signature - #[prost(map="string, string", tag="3")] - pub approvals: ::std::collections::HashMap<::prost::alloc::string::String, ::prost::alloc::string::String>, -} -/// Forwarded SDP from another relay. -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct ForwardSdp { - #[prost(string, tag="1")] - pub room_name: ::prost::alloc::string::String, - #[prost(string, tag="2")] - pub participant_id: ::prost::alloc::string::String, - #[prost(string, tag="3")] - pub sdp: ::prost::alloc::string::String, - /// "offer" or "answer" - #[prost(string, tag="4")] - pub r#type: ::prost::alloc::string::String, -} -/// Forwarded ICE candidate from another relay. -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct ForwardIce { - #[prost(string, tag="1")] - pub room_name: ::prost::alloc::string::String, - #[prost(string, tag="2")] - pub participant_id: ::prost::alloc::string::String, - #[prost(string, tag="3")] - pub candidate: ::prost::alloc::string::String, -} -/// Forwarded ingest room from another relay. -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct ForwardIngest { - #[prost(string, tag="1")] - pub room_name: ::prost::alloc::string::String, -} -/// Stream request from mesh. -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct StreamRequest { - #[prost(string, tag="1")] - pub room_name: ::prost::alloc::string::String, -} -/// StateUpdate propagates entity state changes across the mesh. -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct StateUpdate { - /// Unique sequence number for this update - #[prost(uint64, tag="1")] - pub sequence_number: u64, - /// Key: entity_id (e.g., room name), Value: EntityState - #[prost(map="string, message", tag="2")] - pub entities: ::std::collections::HashMap<::prost::alloc::string::String, EntityState>, -} -/// Ack acknowledges receipt of a StateUpdate. -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct Ack { - /// UUID of the acknowledging relay - #[prost(string, tag="1")] - pub relay_id: ::prost::alloc::string::String, - /// Sequence number being acknowledged - #[prost(uint64, tag="2")] - pub sequence_number: u64, -} -/// RetransmissionRequest requests a missed StateUpdate. -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct RetransmissionRequest { - /// UUID of the requesting relay - #[prost(string, tag="1")] - pub relay_id: ::prost::alloc::string::String, - /// Sequence number of the missed update - #[prost(uint64, tag="2")] - pub sequence_number: u64, -} -/// Retransmission resends a StateUpdate. -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct Retransmission { - /// UUID of the sending relay - #[prost(string, tag="1")] - pub relay_id: ::prost::alloc::string::String, - /// The retransmitted update - #[prost(message, optional, tag="2")] - pub state_update: ::core::option::Option, -} -/// Heartbeat signals relay liveness. -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct Heartbeat { - /// UUID of the sending relay - #[prost(string, tag="1")] - pub relay_id: ::prost::alloc::string::String, - /// Time of the heartbeat - #[prost(message, optional, tag="2")] - pub timestamp: ::core::option::Option<::prost_types::Timestamp>, -} -/// SuspectRelay marks a relay as potentially unresponsive. -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct SuspectRelay { - /// UUID of the suspected relay - #[prost(string, tag="1")] - pub relay_id: ::prost::alloc::string::String, - /// Reason for suspicion (e.g., "no heartbeat") - #[prost(string, tag="2")] - pub reason: ::prost::alloc::string::String, -} -/// Disconnect signals to remove a relay from the mesh. -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct Disconnect { - /// UUID of the relay to disconnect - #[prost(string, tag="1")] - pub relay_id: ::prost::alloc::string::String, - /// Reason for disconnection (e.g., "unresponsive") - #[prost(string, tag="2")] - pub reason: ::prost::alloc::string::String, -} -// @@protoc_insertion_point(module) diff --git a/packages/server/src/proto/proto.rs b/packages/server/src/proto/proto.rs index 9c148b00..a9ef7a42 100644 --- a/packages/server/src/proto/proto.rs +++ b/packages/server/src/proto/proto.rs @@ -20,80 +20,59 @@ pub struct ProtoLatencyTracker { /// MouseMove message #[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Clone, Copy, PartialEq, ::prost::Message)] pub struct ProtoMouseMove { - /// Fixed value "MouseMove" - #[prost(string, tag="1")] - pub r#type: ::prost::alloc::string::String, - #[prost(int32, tag="2")] + #[prost(int32, tag="1")] pub x: i32, - #[prost(int32, tag="3")] + #[prost(int32, tag="2")] pub y: i32, } /// MouseMoveAbs message #[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Clone, Copy, PartialEq, ::prost::Message)] pub struct ProtoMouseMoveAbs { - /// Fixed value "MouseMoveAbs" - #[prost(string, tag="1")] - pub r#type: ::prost::alloc::string::String, - #[prost(int32, tag="2")] + #[prost(int32, tag="1")] pub x: i32, - #[prost(int32, tag="3")] + #[prost(int32, tag="2")] pub y: i32, } /// MouseWheel message #[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Clone, Copy, PartialEq, ::prost::Message)] pub struct ProtoMouseWheel { - /// Fixed value "MouseWheel" - #[prost(string, tag="1")] - pub r#type: ::prost::alloc::string::String, - #[prost(int32, tag="2")] + #[prost(int32, tag="1")] pub x: i32, - #[prost(int32, tag="3")] + #[prost(int32, tag="2")] pub y: i32, } /// MouseKeyDown message #[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Clone, Copy, PartialEq, ::prost::Message)] pub struct ProtoMouseKeyDown { - /// Fixed value "MouseKeyDown" - #[prost(string, tag="1")] - pub r#type: ::prost::alloc::string::String, - #[prost(int32, tag="2")] + #[prost(int32, tag="1")] pub key: i32, } /// MouseKeyUp message #[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Clone, Copy, PartialEq, ::prost::Message)] pub struct ProtoMouseKeyUp { - /// Fixed value "MouseKeyUp" - #[prost(string, tag="1")] - pub r#type: ::prost::alloc::string::String, - #[prost(int32, tag="2")] + #[prost(int32, tag="1")] pub key: i32, } // Keyboard messages /// KeyDown message #[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Clone, Copy, PartialEq, ::prost::Message)] pub struct ProtoKeyDown { - /// Fixed value "KeyDown" - #[prost(string, tag="1")] - pub r#type: ::prost::alloc::string::String, - #[prost(int32, tag="2")] + #[prost(int32, tag="1")] pub key: i32, } /// KeyUp message #[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Clone, Copy, PartialEq, ::prost::Message)] pub struct ProtoKeyUp { - /// Fixed value "KeyUp" - #[prost(string, tag="1")] - pub r#type: ::prost::alloc::string::String, - #[prost(int32, tag="2")] + #[prost(int32, tag="1")] pub key: i32, } // Controller messages @@ -102,174 +81,231 @@ pub struct ProtoKeyUp { #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct ProtoControllerAttach { - /// Fixed value "ControllerAttach" - #[prost(string, tag="1")] - pub r#type: ::prost::alloc::string::String, /// One of the following enums: "ps", "xbox" or "switch" - #[prost(string, tag="2")] + #[prost(string, tag="1")] pub id: ::prost::alloc::string::String, /// Slot number (0-3) - #[prost(int32, tag="3")] + #[prost(int32, tag="2")] pub slot: i32, + /// Session ID of the client attaching the controller + #[prost(string, tag="3")] + pub session_id: ::prost::alloc::string::String, } /// ControllerDetach message #[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Clone, Copy, PartialEq, ::prost::Message)] pub struct ProtoControllerDetach { - /// Fixed value "ControllerDetach" - #[prost(string, tag="1")] - pub r#type: ::prost::alloc::string::String, /// Slot number (0-3) - #[prost(int32, tag="2")] + #[prost(int32, tag="1")] pub slot: i32, } /// ControllerButton message #[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Clone, Copy, PartialEq, ::prost::Message)] pub struct ProtoControllerButton { - /// Fixed value "ControllerButtons" - #[prost(string, tag="1")] - pub r#type: ::prost::alloc::string::String, /// Slot number (0-3) - #[prost(int32, tag="2")] + #[prost(int32, tag="1")] pub slot: i32, /// Button code (linux input event code) - #[prost(int32, tag="3")] + #[prost(int32, tag="2")] pub button: i32, /// true if pressed, false if released - #[prost(bool, tag="4")] + #[prost(bool, tag="3")] pub pressed: bool, } /// ControllerTriggers message #[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Clone, Copy, PartialEq, ::prost::Message)] pub struct ProtoControllerTrigger { - /// Fixed value "ControllerTriggers" - #[prost(string, tag="1")] - pub r#type: ::prost::alloc::string::String, /// Slot number (0-3) - #[prost(int32, tag="2")] + #[prost(int32, tag="1")] pub slot: i32, /// Trigger number (0 for left, 1 for right) - #[prost(int32, tag="3")] + #[prost(int32, tag="2")] pub trigger: i32, /// trigger value (-32768 to 32767) - #[prost(int32, tag="4")] + #[prost(int32, tag="3")] pub value: i32, } /// ControllerSticks message #[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Clone, Copy, PartialEq, ::prost::Message)] pub struct ProtoControllerStick { - /// Fixed value "ControllerStick" - #[prost(string, tag="1")] - pub r#type: ::prost::alloc::string::String, /// Slot number (0-3) - #[prost(int32, tag="2")] + #[prost(int32, tag="1")] pub slot: i32, /// Stick number (0 for left, 1 for right) - #[prost(int32, tag="3")] + #[prost(int32, tag="2")] pub stick: i32, /// X axis value (-32768 to 32767) - #[prost(int32, tag="4")] + #[prost(int32, tag="3")] pub x: i32, /// Y axis value (-32768 to 32767) - #[prost(int32, tag="5")] + #[prost(int32, tag="4")] pub y: i32, } /// ControllerAxis message #[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Clone, Copy, PartialEq, ::prost::Message)] pub struct ProtoControllerAxis { - /// Fixed value "ControllerAxis" - #[prost(string, tag="1")] - pub r#type: ::prost::alloc::string::String, /// Slot number (0-3) - #[prost(int32, tag="2")] + #[prost(int32, tag="1")] pub slot: i32, /// Axis number (0 for d-pad horizontal, 1 for d-pad vertical) - #[prost(int32, tag="3")] + #[prost(int32, tag="2")] pub axis: i32, /// axis value (-1 to 1) - #[prost(int32, tag="4")] + #[prost(int32, tag="3")] pub value: i32, } /// ControllerRumble message #[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Clone, Copy, PartialEq, ::prost::Message)] pub struct ProtoControllerRumble { - /// Fixed value "ControllerRumble" - #[prost(string, tag="1")] - pub r#type: ::prost::alloc::string::String, /// Slot number (0-3) - #[prost(int32, tag="2")] + #[prost(int32, tag="1")] pub slot: i32, /// Low frequency rumble (0-65535) - #[prost(int32, tag="3")] + #[prost(int32, tag="2")] pub low_frequency: i32, /// High frequency rumble (0-65535) - #[prost(int32, tag="4")] + #[prost(int32, tag="3")] pub high_frequency: i32, /// Duration in milliseconds - #[prost(int32, tag="5")] + #[prost(int32, tag="4")] pub duration: i32, } -/// Union of all Input types +// WebRTC + signaling + +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct RtcIceCandidateInit { + #[prost(string, tag="1")] + pub candidate: ::prost::alloc::string::String, + #[prost(uint32, optional, tag="2")] + pub sdp_m_line_index: ::core::option::Option, + #[prost(string, optional, tag="3")] + pub sdp_mid: ::core::option::Option<::prost::alloc::string::String>, + #[prost(string, optional, tag="4")] + pub username_fragment: ::core::option::Option<::prost::alloc::string::String>, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct RtcSessionDescriptionInit { + #[prost(string, tag="1")] + pub sdp: ::prost::alloc::string::String, + #[prost(string, tag="2")] + pub r#type: ::prost::alloc::string::String, +} +/// ProtoICE message +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct ProtoIce { + #[prost(message, optional, tag="1")] + pub candidate: ::core::option::Option, +} +/// ProtoSDP message #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] -pub struct ProtoInput { - #[prost(oneof="proto_input::InputType", tags="1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14")] - pub input_type: ::core::option::Option, +pub struct ProtoSdp { + #[prost(message, optional, tag="1")] + pub sdp: ::core::option::Option, } -/// Nested message and enum types in `ProtoInput`. -pub mod proto_input { +/// ProtoRaw message +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct ProtoRaw { + #[prost(string, tag="1")] + pub data: ::prost::alloc::string::String, +} +/// ProtoClientRequestRoomStream message +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct ProtoClientRequestRoomStream { + #[prost(string, tag="1")] + pub room_name: ::prost::alloc::string::String, + #[prost(string, tag="2")] + pub session_id: ::prost::alloc::string::String, +} +/// ProtoClientDisconnected message +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct ProtoClientDisconnected { + #[prost(string, tag="1")] + pub session_id: ::prost::alloc::string::String, + #[prost(int32, repeated, tag="2")] + pub controller_slots: ::prost::alloc::vec::Vec, +} +/// ProtoServerPushStream message +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct ProtoServerPushStream { + #[prost(string, tag="1")] + pub room_name: ::prost::alloc::string::String, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct ProtoMessageBase { + #[prost(string, tag="1")] + pub payload_type: ::prost::alloc::string::String, + #[prost(message, optional, tag="2")] + pub latency: ::core::option::Option, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct ProtoMessage { + #[prost(message, optional, tag="1")] + pub message_base: ::core::option::Option, + #[prost(oneof="proto_message::Payload", tags="2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 20, 21, 22, 23, 24, 25")] + pub payload: ::core::option::Option, +} +/// Nested message and enum types in `ProtoMessage`. +pub mod proto_message { #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Oneof)] - pub enum InputType { - #[prost(message, tag="1")] - MouseMove(super::ProtoMouseMove), + pub enum Payload { + /// Input types #[prost(message, tag="2")] - MouseMoveAbs(super::ProtoMouseMoveAbs), + MouseMove(super::ProtoMouseMove), #[prost(message, tag="3")] - MouseWheel(super::ProtoMouseWheel), + MouseMoveAbs(super::ProtoMouseMoveAbs), #[prost(message, tag="4")] - MouseKeyDown(super::ProtoMouseKeyDown), + MouseWheel(super::ProtoMouseWheel), #[prost(message, tag="5")] - MouseKeyUp(super::ProtoMouseKeyUp), + MouseKeyDown(super::ProtoMouseKeyDown), #[prost(message, tag="6")] - KeyDown(super::ProtoKeyDown), + MouseKeyUp(super::ProtoMouseKeyUp), #[prost(message, tag="7")] - KeyUp(super::ProtoKeyUp), + KeyDown(super::ProtoKeyDown), #[prost(message, tag="8")] - ControllerAttach(super::ProtoControllerAttach), + KeyUp(super::ProtoKeyUp), #[prost(message, tag="9")] - ControllerDetach(super::ProtoControllerDetach), + ControllerAttach(super::ProtoControllerAttach), #[prost(message, tag="10")] - ControllerButton(super::ProtoControllerButton), + ControllerDetach(super::ProtoControllerDetach), #[prost(message, tag="11")] - ControllerTrigger(super::ProtoControllerTrigger), + ControllerButton(super::ProtoControllerButton), #[prost(message, tag="12")] - ControllerStick(super::ProtoControllerStick), + ControllerTrigger(super::ProtoControllerTrigger), #[prost(message, tag="13")] - ControllerAxis(super::ProtoControllerAxis), + ControllerStick(super::ProtoControllerStick), #[prost(message, tag="14")] + ControllerAxis(super::ProtoControllerAxis), + #[prost(message, tag="15")] ControllerRumble(super::ProtoControllerRumble), + /// Signaling types + #[prost(message, tag="20")] + Ice(super::ProtoIce), + #[prost(message, tag="21")] + Sdp(super::ProtoSdp), + #[prost(message, tag="22")] + Raw(super::ProtoRaw), + #[prost(message, tag="23")] + ClientRequestRoomStream(super::ProtoClientRequestRoomStream), + #[prost(message, tag="24")] + ClientDisconnected(super::ProtoClientDisconnected), + #[prost(message, tag="25")] + ServerPushStream(super::ProtoServerPushStream), } } -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct ProtoMessageBase { - #[prost(string, tag="1")] - pub payload_type: ::prost::alloc::string::String, - #[prost(message, optional, tag="2")] - pub latency: ::core::option::Option, -} -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct ProtoMessageInput { - #[prost(message, optional, tag="1")] - pub message_base: ::core::option::Option, - #[prost(message, optional, tag="2")] - pub data: ::core::option::Option, -} // @@protoc_insertion_point(module) diff --git a/protobufs/messages.proto b/protobufs/messages.proto index 7617ccc0..bc14fd1b 100644 --- a/protobufs/messages.proto +++ b/protobufs/messages.proto @@ -12,7 +12,31 @@ message ProtoMessageBase { ProtoLatencyTracker latency = 2; } -message ProtoMessageInput { - ProtoMessageBase message_base = 1; - ProtoInput data = 2; +message ProtoMessage { + ProtoMessageBase message_base = 1; + oneof payload { + // Input types + ProtoMouseMove mouse_move = 2; + ProtoMouseMoveAbs mouse_move_abs = 3; + ProtoMouseWheel mouse_wheel = 4; + ProtoMouseKeyDown mouse_key_down = 5; + ProtoMouseKeyUp mouse_key_up = 6; + ProtoKeyDown key_down = 7; + ProtoKeyUp key_up = 8; + ProtoControllerAttach controller_attach = 9; + ProtoControllerDetach controller_detach = 10; + ProtoControllerButton controller_button = 11; + ProtoControllerTrigger controller_trigger = 12; + ProtoControllerStick controller_stick = 13; + ProtoControllerAxis controller_axis = 14; + ProtoControllerRumble controller_rumble = 15; + + // Signaling types + ProtoICE ice = 20; + ProtoSDP sdp = 21; + ProtoRaw raw = 22; + ProtoClientRequestRoomStream client_request_room_stream = 23; + ProtoClientDisconnected client_disconnected = 24; + ProtoServerPushStream server_push_stream = 25; + } } diff --git a/protobufs/types.proto b/protobufs/types.proto index af1d872a..037b05e9 100644 --- a/protobufs/types.proto +++ b/protobufs/types.proto @@ -8,124 +8,137 @@ package proto; // MouseMove message message ProtoMouseMove { - string type = 1; // Fixed value "MouseMove" - int32 x = 2; - int32 y = 3; + int32 x = 1; + int32 y = 2; } // MouseMoveAbs message message ProtoMouseMoveAbs { - string type = 1; // Fixed value "MouseMoveAbs" - int32 x = 2; - int32 y = 3; + int32 x = 1; + int32 y = 2; } // MouseWheel message message ProtoMouseWheel { - string type = 1; // Fixed value "MouseWheel" - int32 x = 2; - int32 y = 3; + int32 x = 1; + int32 y = 2; } // MouseKeyDown message message ProtoMouseKeyDown { - string type = 1; // Fixed value "MouseKeyDown" - int32 key = 2; + int32 key = 1; } // MouseKeyUp message message ProtoMouseKeyUp { - string type = 1; // Fixed value "MouseKeyUp" - int32 key = 2; + int32 key = 1; } /* Keyboard messages */ // KeyDown message message ProtoKeyDown { - string type = 1; // Fixed value "KeyDown" - int32 key = 2; + int32 key = 1; } // KeyUp message message ProtoKeyUp { - string type = 1; // Fixed value "KeyUp" - int32 key = 2; + int32 key = 1; } /* Controller messages */ // ControllerAttach message message ProtoControllerAttach { - string type = 1; // Fixed value "ControllerAttach" - string id = 2; // One of the following enums: "ps", "xbox" or "switch" - int32 slot = 3; // Slot number (0-3) + string id = 1; // One of the following enums: "ps", "xbox" or "switch" + int32 slot = 2; // Slot number (0-3) + string session_id = 3; // Session ID of the client attaching the controller } // ControllerDetach message message ProtoControllerDetach { - string type = 1; // Fixed value "ControllerDetach" - int32 slot = 2; // Slot number (0-3) + int32 slot = 1; // Slot number (0-3) } // ControllerButton message message ProtoControllerButton { - string type = 1; // Fixed value "ControllerButtons" - int32 slot = 2; // Slot number (0-3) - int32 button = 3; // Button code (linux input event code) - bool pressed = 4; // true if pressed, false if released + int32 slot = 1; // Slot number (0-3) + int32 button = 2; // Button code (linux input event code) + bool pressed = 3; // true if pressed, false if released } // ControllerTriggers message message ProtoControllerTrigger { - string type = 1; // Fixed value "ControllerTriggers" - int32 slot = 2; // Slot number (0-3) - int32 trigger = 3; // Trigger number (0 for left, 1 for right) - int32 value = 4; // trigger value (-32768 to 32767) + int32 slot = 1; // Slot number (0-3) + int32 trigger = 2; // Trigger number (0 for left, 1 for right) + int32 value = 3; // trigger value (-32768 to 32767) } // ControllerSticks message message ProtoControllerStick { - string type = 1; // Fixed value "ControllerStick" - int32 slot = 2; // Slot number (0-3) - int32 stick = 3; // Stick number (0 for left, 1 for right) - int32 x = 4; // X axis value (-32768 to 32767) - int32 y = 5; // Y axis value (-32768 to 32767) + int32 slot = 1; // Slot number (0-3) + int32 stick = 2; // Stick number (0 for left, 1 for right) + int32 x = 3; // X axis value (-32768 to 32767) + int32 y = 4; // Y axis value (-32768 to 32767) } // ControllerAxis message message ProtoControllerAxis { - string type = 1; // Fixed value "ControllerAxis" - int32 slot = 2; // Slot number (0-3) - int32 axis = 3; // Axis number (0 for d-pad horizontal, 1 for d-pad vertical) - int32 value = 4; // axis value (-1 to 1) + int32 slot = 1; // Slot number (0-3) + int32 axis = 2; // Axis number (0 for d-pad horizontal, 1 for d-pad vertical) + int32 value = 3; // axis value (-1 to 1) } // ControllerRumble message message ProtoControllerRumble { - string type = 1; // Fixed value "ControllerRumble" - int32 slot = 2; // Slot number (0-3) - int32 low_frequency = 3; // Low frequency rumble (0-65535) - int32 high_frequency = 4; // High frequency rumble (0-65535) - int32 duration = 5; // Duration in milliseconds -} - -// Union of all Input types -message ProtoInput { - oneof input_type { - ProtoMouseMove mouse_move = 1; - ProtoMouseMoveAbs mouse_move_abs = 2; - ProtoMouseWheel mouse_wheel = 3; - ProtoMouseKeyDown mouse_key_down = 4; - ProtoMouseKeyUp mouse_key_up = 5; - ProtoKeyDown key_down = 6; - ProtoKeyUp key_up = 7; - ProtoControllerAttach controller_attach = 8; - ProtoControllerDetach controller_detach = 9; - ProtoControllerButton controller_button = 10; - ProtoControllerTrigger controller_trigger = 11; - ProtoControllerStick controller_stick = 12; - ProtoControllerAxis controller_axis = 13; - ProtoControllerRumble controller_rumble = 14; - } + int32 slot = 1; // Slot number (0-3) + int32 low_frequency = 2; // Low frequency rumble (0-65535) + int32 high_frequency = 3; // High frequency rumble (0-65535) + int32 duration = 4; // Duration in milliseconds +} + +/* WebRTC + signaling */ + +message RTCIceCandidateInit { + string candidate = 1; + optional uint32 sdpMLineIndex = 2; + optional string sdpMid = 3; + optional string usernameFragment = 4; +} + +message RTCSessionDescriptionInit { + string sdp = 1; + string type = 2; +} + +// ProtoICE message +message ProtoICE { + RTCIceCandidateInit candidate = 1; +} + +// ProtoSDP message +message ProtoSDP { + RTCSessionDescriptionInit sdp = 1; +} + +// ProtoRaw message +message ProtoRaw { + string data = 1; +} + +// ProtoClientRequestRoomStream message +message ProtoClientRequestRoomStream { + string room_name = 1; + string session_id = 2; +} + +// ProtoClientDisconnected message +message ProtoClientDisconnected { + string session_id = 1; + repeated int32 controller_slots = 2; +} + +// ProtoServerPushStream message +message ProtoServerPushStream { + string room_name = 1; } From a54cf759fa63f75efee19695c1e774ef5e0c45b3 Mon Sep 17 00:00:00 2001 From: DatCaptainHorse Date: Sat, 25 Oct 2025 03:57:26 +0300 Subject: [PATCH 2/6] Fixed multi-controllers, optimize and improve code in relay and nestri-server --- containerfiles/runner-builder.Containerfile | 20 +- packages/input/src/controller.ts | 119 +++++++--- .../input/src/proto/latency_tracker_pb.ts | 2 +- packages/input/src/proto/messages_pb.ts | 2 +- packages/input/src/proto/types_pb.ts | 114 ++++++--- packages/input/src/webrtc-stream.ts | 17 +- .../play-standalone/src/pages/[room].astro | 6 +- packages/relay/internal/common/common.go | 2 +- .../relay/internal/core/protocol_stream.go | 213 ++++++++--------- packages/relay/internal/core/room.go | 2 +- packages/relay/internal/core/state.go | 6 +- packages/relay/internal/proto/types.pb.go | 186 ++++++++++----- packages/relay/internal/shared/participant.go | 110 +++++++-- packages/relay/internal/shared/room.go | 224 +++++++++--------- packages/scripts/entrypoint.sh | 28 ++- packages/scripts/entrypoint_nestri.sh | 2 +- packages/server/Cargo.lock | 128 +++++----- packages/server/Cargo.toml | 2 +- packages/server/src/args.rs | 8 + packages/server/src/args/app_args.rs | 8 + packages/server/src/enc_helper.rs | 62 +---- packages/server/src/input/controller.rs | 44 ++-- packages/server/src/main.rs | 29 +-- packages/server/src/nestrisink/imp.rs | 13 +- packages/server/src/nestrisink/mod.rs | 2 +- packages/server/src/proto/proto.rs | 84 ++++--- protobufs/types.proto | 46 ++-- 27 files changed, 836 insertions(+), 643 deletions(-) diff --git a/containerfiles/runner-builder.Containerfile b/containerfiles/runner-builder.Containerfile index 68528fdf..e6613bb4 100644 --- a/containerfiles/runner-builder.Containerfile +++ b/containerfiles/runner-builder.Containerfile @@ -41,7 +41,7 @@ RUN --mount=type=cache,target=/var/cache/pacman/pkg \ pacman -Sy --noconfirm lib32-gcc-libs # Clone repository -RUN git clone --depth 1 --rev "9e8bfd0217eeab011c5afc368d3ea67a4c239e81" https://github.com/DatCaptainHorse/vimputti.git +RUN git clone --depth 1 --rev "f2f21561ddcb814d74455311969d3e8934b052c6" https://github.com/DatCaptainHorse/vimputti.git #-------------------------------------------------------------------- FROM vimputti-manager-deps AS vimputti-manager-planner @@ -129,23 +129,8 @@ RUN --mount=type=cache,target=/var/cache/pacman/pkg \ RUN --mount=type=cache,target=${CARGO_HOME}/registry \ cargo install cargo-c -# Grab cudart from NVIDIA.. -RUN wget https://developer.download.nvidia.com/compute/cuda/redist/cuda_cudart/linux-x86_64/cuda_cudart-linux-x86_64-13.0.96-archive.tar.xz -O cuda_cudart.tar.xz && \ - mkdir cuda_cudart && tar -xf cuda_cudart.tar.xz -C cuda_cudart --strip-components=1 && \ - cp cuda_cudart/lib/libcudart.so cuda_cudart/lib/libcudart.so.* /usr/lib/ && \ - rm -r cuda_cudart && \ - rm cuda_cudart.tar.xz - -# Grab cuda lib from NVIDIA (it's in driver package of all things..) -RUN wget https://developer.download.nvidia.com/compute/cuda/redist/nvidia_driver/linux-x86_64/nvidia_driver-linux-x86_64-580.95.05-archive.tar.xz -O nvidia_driver.tar.xz && \ - mkdir nvidia_driver && tar -xf nvidia_driver.tar.xz -C nvidia_driver --strip-components=1 && \ - cp nvidia_driver/lib/libcuda.so.* /usr/lib/libcuda.so && \ - ln -s /usr/lib/libcuda.so /usr/lib/libcuda.so.1 && \ - rm -r nvidia_driver && \ - rm nvidia_driver.tar.xz - # Clone repository -RUN git clone --depth 1 --rev "afa853fa03e8403c83bbb3bc0cf39147ad46c266" https://github.com/games-on-whales/gst-wayland-display.git +RUN git clone --depth 1 --rev "a4abcfe2cffe2d33b564d1308b58504a5e3012b1" https://github.com/games-on-whales/gst-wayland-display.git #-------------------------------------------------------------------- FROM gst-wayland-deps AS gst-wayland-planner @@ -214,5 +199,4 @@ COPY --from=gst-wayland-cached-builder /artifacts/include/ /artifacts/include/ COPY --from=vimputti-manager-cached-builder /artifacts/vimputti-manager /artifacts/bin/ COPY --from=vimputti-manager-cached-builder /artifacts/libvimputti_shim_64.so /artifacts/lib64/libvimputti_shim.so COPY --from=vimputti-manager-cached-builder /artifacts/libvimputti_shim_32.so /artifacts/lib32/libvimputti_shim.so -COPY --from=gst-wayland-deps /usr/lib/libcuda.so /usr/lib/libcuda.so.* /artifacts/lib/ COPY --from=bubblewrap-builder /artifacts/bin/bwrap /artifacts/bin/ diff --git a/packages/input/src/controller.ts b/packages/input/src/controller.ts index d1c5316d..6742f6e5 100644 --- a/packages/input/src/controller.ts +++ b/packages/input/src/controller.ts @@ -32,7 +32,6 @@ interface GamepadState { export class Controller { protected wrtc: WebRTCStream; - protected slotMap: Map = new Map(); // local slot to server slot protected connected: boolean = false; protected gamepad: Gamepad | null = null; protected lastState: GamepadState = { @@ -50,6 +49,14 @@ export class Controller { protected stickDeadzone: number = 2048; // 2048 / 32768 = ~0.06 (6% of stick range) private updateInterval = 10.0; // 100 updates per second + private isIdle: boolean = true; + private lastInputTime: number = Date.now(); + private idleUpdateInterval: number = 150.0; // ~6-7 updates per second for keep-alive packets + private inputDetected: boolean = false; + private lastFullStateSend: number = Date.now(); + private fullStateSendInterval: number = 500.0; // send full state every 0.5 seconds (helps packet loss) + private forceFullStateSend: boolean = false; + private _dcHandler: ((data: ArrayBuffer) => void) | null = null; constructor({ webrtc, e }: Props) { @@ -79,9 +86,8 @@ export class Controller { const attachMsg = messageWrapper.payload.value; // Gamepad connected succesfully this.gamepad = e.gamepad; - this.slotMap.set(e.gamepad.index, attachMsg.slot); console.log( - `Gamepad connected: ${e.gamepad.id} assigned to slot ${attachMsg.slot} on server, local slot ${e.gamepad.index}`, + `Gamepad connected: ${e.gamepad.id}, local slot ${e.gamepad.index}, msg: ${attachMsg.sessionSlot}`, ); } } catch (err) { @@ -93,6 +99,7 @@ export class Controller { const attachMsg = createMessage( create(ProtoControllerAttachSchema, { id: this.vendor_id_to_controller(vendorId, productId), + sessionSlot: e.gamepad.index, sessionId: this.wrtc.getSessionID(), }), "controllerInput", @@ -102,6 +109,10 @@ export class Controller { this.run(); } + public getSlot(): number { + return this.gamepad.index; + } + // Maps vendor id and product id to supported controller type // Currently supported: Sony (ps4, ps5), Microsoft (xbox360, xboxone), Nintendo (switchpro) // Default fallback to xbox360 @@ -154,6 +165,13 @@ export class Controller { private pollGamepad() { // Get updated gamepad state const gamepads = navigator.getGamepads(); + + // Periodically force send full state to clear stuck inputs + if (Date.now() - this.lastFullStateSend > this.fullStateSendInterval) { + this.forceFullStateSend = true; + this.lastFullStateSend = Date.now(); + } + if (this.gamepad) { if (gamepads[this.gamepad.index]) { this.gamepad = gamepads[this.gamepad!.index]; @@ -164,7 +182,7 @@ export class Controller { // ignore trigger buttons (6-7) as we handle those as axis if (index === 6 || index === 7) return; // If state differs, send - if (button.pressed !== this.lastState.buttonState.get(index)) { + if (button.pressed !== this.lastState.buttonState.get(index) || this.forceFullStateSend) { const linuxCode = this.controllerButtonToVirtualKeyCode(index); if (linuxCode === undefined) { // Skip unmapped button index @@ -174,13 +192,15 @@ export class Controller { const buttonMessage = createMessage( create(ProtoControllerButtonSchema, { - slot: this.getServerSlot(), + sessionSlot: this.gamepad.index, + sessionId: this.wrtc.getSessionID(), button: linuxCode, pressed: button.pressed, }), "controllerInput", ); this.wrtc.sendBinary(toBinary(ProtoMessageSchema, buttonMessage)); + this.inputDetected = true; // Store button state this.lastState.buttonState.set(index, button.pressed); } @@ -198,16 +218,18 @@ export class Controller { ), ); // If state differs, send - if (leftTrigger !== this.lastState.leftTrigger) { + if (leftTrigger !== this.lastState.leftTrigger || this.forceFullStateSend) { const triggerMessage = createMessage( create(ProtoControllerTriggerSchema, { - slot: this.getServerSlot(), + sessionSlot: this.gamepad.index, + sessionId: this.wrtc.getSessionID(), trigger: 0, // 0 = left, 1 = right value: leftTrigger, }), "controllerInput", ); this.wrtc.sendBinary(toBinary(ProtoMessageSchema, triggerMessage)); + this.inputDetected = true; this.lastState.leftTrigger = leftTrigger; } const rightTrigger = Math.round( @@ -220,16 +242,18 @@ export class Controller { ), ); // If state differs, send - if (rightTrigger !== this.lastState.rightTrigger) { + if (rightTrigger !== this.lastState.rightTrigger || this.forceFullStateSend) { const triggerMessage = createMessage( create(ProtoControllerTriggerSchema, { - slot: this.getServerSlot(), + sessionSlot: this.gamepad.index, + sessionId: this.wrtc.getSessionID(), trigger: 1, // 0 = left, 1 = right value: rightTrigger, }), "controllerInput", ); this.wrtc.sendBinary(toBinary(ProtoMessageSchema, triggerMessage)); + this.inputDetected = true; this.lastState.rightTrigger = rightTrigger; } @@ -238,32 +262,36 @@ export class Controller { const dpadLeft = this.gamepad.buttons[14]?.pressed ? 1 : 0; const dpadRight = this.gamepad.buttons[15]?.pressed ? 1 : 0; const dpadX = dpadLeft ? -1 : dpadRight ? 1 : 0; - if (dpadX !== this.lastState.dpadX) { + if (dpadX !== this.lastState.dpadX || this.forceFullStateSend) { const dpadMessage = createMessage( create(ProtoControllerAxisSchema, { - slot: this.getServerSlot(), + sessionSlot: this.gamepad.index, + sessionId: this.wrtc.getSessionID(), axis: 0, // 0 = dpadX, 1 = dpadY value: dpadX, }), "controllerInput", ); - this.lastState.dpadX = dpadX; this.wrtc.sendBinary(toBinary(ProtoMessageSchema, dpadMessage)); + this.inputDetected = true; + this.lastState.dpadX = dpadX; } const dpadUp = this.gamepad.buttons[12]?.pressed ? 1 : 0; const dpadDown = this.gamepad.buttons[13]?.pressed ? 1 : 0; const dpadY = dpadUp ? -1 : dpadDown ? 1 : 0; - if (dpadY !== this.lastState.dpadY) { + if (dpadY !== this.lastState.dpadY || this.forceFullStateSend) { const dpadMessage = createMessage( create(ProtoControllerAxisSchema, { - slot: this.getServerSlot(), + sessionSlot: this.gamepad.index, + sessionId: this.wrtc.getSessionID(), axis: 1, // 0 = dpadX, 1 = dpadY value: dpadY, }), "controllerInput", ); this.wrtc.sendBinary(toBinary(ProtoMessageSchema, dpadMessage)); + this.inputDetected = true; this.lastState.dpadY = dpadY; } @@ -292,10 +320,12 @@ export class Controller { // if moves inside deadzone, zero it if not inside deadzone last time if ( sendLeftX !== this.lastState.leftX || - sendLeftY !== this.lastState.leftY + sendLeftY !== this.lastState.leftY || this.forceFullStateSend ) { const stickMessage = createMessage( create(ProtoControllerStickSchema, { + sessionSlot: this.gamepad.index, + sessionId: this.wrtc.getSessionID(), stick: 0, // 0 = left, 1 = right x: sendLeftX, y: sendLeftY, @@ -303,6 +333,7 @@ export class Controller { "controllerInput", ); this.wrtc.sendBinary(toBinary(ProtoMessageSchema, stickMessage)); + this.inputDetected = true; this.lastState.leftX = sendLeftX; this.lastState.leftY = sendLeftY; } @@ -328,10 +359,12 @@ export class Controller { Math.abs(rightY) > this.stickDeadzone ? Math.round(rightY) : 0; if ( sendRightX !== this.lastState.rightX || - sendRightY !== this.lastState.rightY + sendRightY !== this.lastState.rightY || this.forceFullStateSend ) { const stickMessage = createMessage( create(ProtoControllerStickSchema, { + sessionSlot: this.gamepad.index, + sessionId: this.wrtc.getSessionID(), stick: 1, // 0 = left, 1 = right x: sendRightX, y: sendRightY, @@ -339,11 +372,14 @@ export class Controller { "controllerInput", ); this.wrtc.sendBinary(toBinary(ProtoMessageSchema, stickMessage)); + this.inputDetected = true; this.lastState.rightX = sendRightX; this.lastState.rightY = sendRightY; } } } + + this.forceFullStateSend = false; } private loopInterval: any = null; @@ -352,10 +388,34 @@ export class Controller { if (this.connected) this.stop(); this.connected = true; - // Poll gamepads in setInterval loop + this.isIdle = true; + this.lastInputTime = Date.now(); + this.loopInterval = setInterval(() => { - if (this.connected) this.pollGamepad(); - }, this.updateInterval); + if (this.connected) { + this.inputDetected = false; // Reset before poll + this.pollGamepad(); + + // Switch polling rate based on input + if (this.inputDetected) { + this.lastInputTime = Date.now(); + if (this.isIdle) { + this.isIdle = false; + clearInterval(this.loopInterval); + this.loopInterval = setInterval(() => { + if (this.connected) this.pollGamepad(); + }, this.updateInterval); + } + } else if (!this.isIdle && Date.now() - this.lastInputTime > 200) { + // Switch to idle polling after 200ms of no input + this.isIdle = true; + clearInterval(this.loopInterval); + this.loopInterval = setInterval(() => { + if (this.connected) this.pollGamepad(); + }, this.idleUpdateInterval); + } + } + }, this.isIdle ? this.idleUpdateInterval : this.updateInterval); } public stop() { @@ -366,21 +426,6 @@ export class Controller { this.connected = false; } - public getLocalSlot(): number { - if (this.gamepad) { - return this.gamepad.index; - } - return -1; - } - - public getServerSlot(): number { - if (this.gamepad) { - const slot = this.slotMap.get(this.gamepad.index); - if (slot !== undefined) return slot; - } - return -1; - } - public dispose() { this.stop(); // Remove callback @@ -391,7 +436,7 @@ export class Controller { // Gamepad disconnected const detachMsg = createMessage( create(ProtoControllerDetachSchema, { - slot: this.getServerSlot(), + sessionSlot: this.gamepad.index, }), "controllerInput", ); @@ -407,7 +452,9 @@ export class Controller { if (!this.connected) return; // Check if aimed at this controller slot - if (rumbleMsg.slot !== this.getServerSlot()) return; + if (rumbleMsg.sessionId !== this.wrtc.getSessionID() && + rumbleMsg.sessionSlot !== this.gamepad.index) + return; // Trigger actual rumble // Need to remap from 0-65535 to 0.0-1.0 ranges diff --git a/packages/input/src/proto/latency_tracker_pb.ts b/packages/input/src/proto/latency_tracker_pb.ts index 27eefd2a..2d35d78e 100644 --- a/packages/input/src/proto/latency_tracker_pb.ts +++ b/packages/input/src/proto/latency_tracker_pb.ts @@ -1,4 +1,4 @@ -// @generated by protoc-gen-es v2.9.0 with parameter "target=ts" +// @generated by protoc-gen-es v2.10.0 with parameter "target=ts" // @generated from file latency_tracker.proto (package proto, syntax proto3) /* eslint-disable */ diff --git a/packages/input/src/proto/messages_pb.ts b/packages/input/src/proto/messages_pb.ts index 8c33ee30..4e8c70de 100644 --- a/packages/input/src/proto/messages_pb.ts +++ b/packages/input/src/proto/messages_pb.ts @@ -1,4 +1,4 @@ -// @generated by protoc-gen-es v2.9.0 with parameter "target=ts" +// @generated by protoc-gen-es v2.10.0 with parameter "target=ts" // @generated from file messages.proto (package proto, syntax proto3) /* eslint-disable */ diff --git a/packages/input/src/proto/types_pb.ts b/packages/input/src/proto/types_pb.ts index 6538b8f6..6da4bf1e 100644 --- a/packages/input/src/proto/types_pb.ts +++ b/packages/input/src/proto/types_pb.ts @@ -1,4 +1,4 @@ -// @generated by protoc-gen-es v2.9.0 with parameter "target=ts" +// @generated by protoc-gen-es v2.10.0 with parameter "target=ts" // @generated from file types.proto (package proto, syntax proto3) /* eslint-disable */ @@ -10,7 +10,7 @@ import type { Message } from "@bufbuild/protobuf"; * Describes the file types.proto. */ export const file_types: GenFile = /*@__PURE__*/ - fileDesc("Cgt0eXBlcy5wcm90bxIFcHJvdG8iJgoOUHJvdG9Nb3VzZU1vdmUSCQoBeBgBIAEoBRIJCgF5GAIgASgFIikKEVByb3RvTW91c2VNb3ZlQWJzEgkKAXgYASABKAUSCQoBeRgCIAEoBSInCg9Qcm90b01vdXNlV2hlZWwSCQoBeBgBIAEoBRIJCgF5GAIgASgFIiAKEVByb3RvTW91c2VLZXlEb3duEgsKA2tleRgBIAEoBSIeCg9Qcm90b01vdXNlS2V5VXASCwoDa2V5GAEgASgFIhsKDFByb3RvS2V5RG93bhILCgNrZXkYASABKAUiGQoKUHJvdG9LZXlVcBILCgNrZXkYASABKAUiRQoVUHJvdG9Db250cm9sbGVyQXR0YWNoEgoKAmlkGAEgASgJEgwKBHNsb3QYAiABKAUSEgoKc2Vzc2lvbl9pZBgDIAEoCSIlChVQcm90b0NvbnRyb2xsZXJEZXRhY2gSDAoEc2xvdBgBIAEoBSJGChVQcm90b0NvbnRyb2xsZXJCdXR0b24SDAoEc2xvdBgBIAEoBRIOCgZidXR0b24YAiABKAUSDwoHcHJlc3NlZBgDIAEoCCJGChZQcm90b0NvbnRyb2xsZXJUcmlnZ2VyEgwKBHNsb3QYASABKAUSDwoHdHJpZ2dlchgCIAEoBRINCgV2YWx1ZRgDIAEoBSJJChRQcm90b0NvbnRyb2xsZXJTdGljaxIMCgRzbG90GAEgASgFEg0KBXN0aWNrGAIgASgFEgkKAXgYAyABKAUSCQoBeRgEIAEoBSJAChNQcm90b0NvbnRyb2xsZXJBeGlzEgwKBHNsb3QYASABKAUSDAoEYXhpcxgCIAEoBRINCgV2YWx1ZRgDIAEoBSJmChVQcm90b0NvbnRyb2xsZXJSdW1ibGUSDAoEc2xvdBgBIAEoBRIVCg1sb3dfZnJlcXVlbmN5GAIgASgFEhYKDmhpZ2hfZnJlcXVlbmN5GAMgASgFEhAKCGR1cmF0aW9uGAQgASgFIqoBChNSVENJY2VDYW5kaWRhdGVJbml0EhEKCWNhbmRpZGF0ZRgBIAEoCRIaCg1zZHBNTGluZUluZGV4GAIgASgNSACIAQESEwoGc2RwTWlkGAMgASgJSAGIAQESHQoQdXNlcm5hbWVGcmFnbWVudBgEIAEoCUgCiAEBQhAKDl9zZHBNTGluZUluZGV4QgkKB19zZHBNaWRCEwoRX3VzZXJuYW1lRnJhZ21lbnQiNgoZUlRDU2Vzc2lvbkRlc2NyaXB0aW9uSW5pdBILCgNzZHAYASABKAkSDAoEdHlwZRgCIAEoCSI5CghQcm90b0lDRRItCgljYW5kaWRhdGUYASABKAsyGi5wcm90by5SVENJY2VDYW5kaWRhdGVJbml0IjkKCFByb3RvU0RQEi0KA3NkcBgBIAEoCzIgLnByb3RvLlJUQ1Nlc3Npb25EZXNjcmlwdGlvbkluaXQiGAoIUHJvdG9SYXcSDAoEZGF0YRgBIAEoCSJFChxQcm90b0NsaWVudFJlcXVlc3RSb29tU3RyZWFtEhEKCXJvb21fbmFtZRgBIAEoCRISCgpzZXNzaW9uX2lkGAIgASgJIkcKF1Byb3RvQ2xpZW50RGlzY29ubmVjdGVkEhIKCnNlc3Npb25faWQYASABKAkSGAoQY29udHJvbGxlcl9zbG90cxgCIAMoBSIqChVQcm90b1NlcnZlclB1c2hTdHJlYW0SEQoJcm9vbV9uYW1lGAEgASgJQhZaFHJlbGF5L2ludGVybmFsL3Byb3RvYgZwcm90bzM"); + fileDesc("Cgt0eXBlcy5wcm90bxIFcHJvdG8iJgoOUHJvdG9Nb3VzZU1vdmUSCQoBeBgBIAEoBRIJCgF5GAIgASgFIikKEVByb3RvTW91c2VNb3ZlQWJzEgkKAXgYASABKAUSCQoBeRgCIAEoBSInCg9Qcm90b01vdXNlV2hlZWwSCQoBeBgBIAEoBRIJCgF5GAIgASgFIiAKEVByb3RvTW91c2VLZXlEb3duEgsKA2tleRgBIAEoBSIeCg9Qcm90b01vdXNlS2V5VXASCwoDa2V5GAEgASgFIhsKDFByb3RvS2V5RG93bhILCgNrZXkYASABKAUiGQoKUHJvdG9LZXlVcBILCgNrZXkYASABKAUiTQoVUHJvdG9Db250cm9sbGVyQXR0YWNoEgoKAmlkGAEgASgJEhQKDHNlc3Npb25fc2xvdBgCIAEoBRISCgpzZXNzaW9uX2lkGAMgASgJIkEKFVByb3RvQ29udHJvbGxlckRldGFjaBIUCgxzZXNzaW9uX3Nsb3QYASABKAUSEgoKc2Vzc2lvbl9pZBgCIAEoCSJiChVQcm90b0NvbnRyb2xsZXJCdXR0b24SFAoMc2Vzc2lvbl9zbG90GAEgASgFEhIKCnNlc3Npb25faWQYAiABKAkSDgoGYnV0dG9uGAMgASgFEg8KB3ByZXNzZWQYBCABKAgiYgoWUHJvdG9Db250cm9sbGVyVHJpZ2dlchIUCgxzZXNzaW9uX3Nsb3QYASABKAUSEgoKc2Vzc2lvbl9pZBgCIAEoCRIPCgd0cmlnZ2VyGAMgASgFEg0KBXZhbHVlGAQgASgFImUKFFByb3RvQ29udHJvbGxlclN0aWNrEhQKDHNlc3Npb25fc2xvdBgBIAEoBRISCgpzZXNzaW9uX2lkGAIgASgJEg0KBXN0aWNrGAMgASgFEgkKAXgYBCABKAUSCQoBeRgFIAEoBSJcChNQcm90b0NvbnRyb2xsZXJBeGlzEhQKDHNlc3Npb25fc2xvdBgBIAEoBRISCgpzZXNzaW9uX2lkGAIgASgJEgwKBGF4aXMYAyABKAUSDQoFdmFsdWUYBCABKAUiggEKFVByb3RvQ29udHJvbGxlclJ1bWJsZRIUCgxzZXNzaW9uX3Nsb3QYASABKAUSEgoKc2Vzc2lvbl9pZBgCIAEoCRIVCg1sb3dfZnJlcXVlbmN5GAMgASgFEhYKDmhpZ2hfZnJlcXVlbmN5GAQgASgFEhAKCGR1cmF0aW9uGAUgASgFIqoBChNSVENJY2VDYW5kaWRhdGVJbml0EhEKCWNhbmRpZGF0ZRgBIAEoCRIaCg1zZHBNTGluZUluZGV4GAIgASgNSACIAQESEwoGc2RwTWlkGAMgASgJSAGIAQESHQoQdXNlcm5hbWVGcmFnbWVudBgEIAEoCUgCiAEBQhAKDl9zZHBNTGluZUluZGV4QgkKB19zZHBNaWRCEwoRX3VzZXJuYW1lRnJhZ21lbnQiNgoZUlRDU2Vzc2lvbkRlc2NyaXB0aW9uSW5pdBILCgNzZHAYASABKAkSDAoEdHlwZRgCIAEoCSI5CghQcm90b0lDRRItCgljYW5kaWRhdGUYASABKAsyGi5wcm90by5SVENJY2VDYW5kaWRhdGVJbml0IjkKCFByb3RvU0RQEi0KA3NkcBgBIAEoCzIgLnByb3RvLlJUQ1Nlc3Npb25EZXNjcmlwdGlvbkluaXQiGAoIUHJvdG9SYXcSDAoEZGF0YRgBIAEoCSJFChxQcm90b0NsaWVudFJlcXVlc3RSb29tU3RyZWFtEhEKCXJvb21fbmFtZRgBIAEoCRISCgpzZXNzaW9uX2lkGAIgASgJIkcKF1Byb3RvQ2xpZW50RGlzY29ubmVjdGVkEhIKCnNlc3Npb25faWQYASABKAkSGAoQY29udHJvbGxlcl9zbG90cxgCIAMoBSIqChVQcm90b1NlcnZlclB1c2hTdHJlYW0SEQoJcm9vbV9uYW1lGAEgASgJQhZaFHJlbGF5L2ludGVybmFsL3Byb3RvYgZwcm90bzM"); /** * MouseMove message @@ -174,14 +174,14 @@ export type ProtoControllerAttach = Message<"proto.ProtoControllerAttach"> & { id: string; /** - * Slot number (0-3) + * Session specific slot number (0-3) * - * @generated from field: int32 slot = 2; + * @generated from field: int32 session_slot = 2; */ - slot: number; + sessionSlot: number; /** - * Session ID of the client attaching the controller + * Session ID of the client * * @generated from field: string session_id = 3; */ @@ -202,11 +202,18 @@ export const ProtoControllerAttachSchema: GenMessage = /* */ export type ProtoControllerDetach = Message<"proto.ProtoControllerDetach"> & { /** - * Slot number (0-3) + * Session specific slot number (0-3) * - * @generated from field: int32 slot = 1; + * @generated from field: int32 session_slot = 1; */ - slot: number; + sessionSlot: number; + + /** + * Session ID of the client + * + * @generated from field: string session_id = 2; + */ + sessionId: string; }; /** @@ -223,23 +230,30 @@ export const ProtoControllerDetachSchema: GenMessage = /* */ export type ProtoControllerButton = Message<"proto.ProtoControllerButton"> & { /** - * Slot number (0-3) + * Session specific slot number (0-3) + * + * @generated from field: int32 session_slot = 1; + */ + sessionSlot: number; + + /** + * Session ID of the client * - * @generated from field: int32 slot = 1; + * @generated from field: string session_id = 2; */ - slot: number; + sessionId: string; /** * Button code (linux input event code) * - * @generated from field: int32 button = 2; + * @generated from field: int32 button = 3; */ button: number; /** * true if pressed, false if released * - * @generated from field: bool pressed = 3; + * @generated from field: bool pressed = 4; */ pressed: boolean; }; @@ -258,23 +272,30 @@ export const ProtoControllerButtonSchema: GenMessage = /* */ export type ProtoControllerTrigger = Message<"proto.ProtoControllerTrigger"> & { /** - * Slot number (0-3) + * Session specific slot number (0-3) * - * @generated from field: int32 slot = 1; + * @generated from field: int32 session_slot = 1; */ - slot: number; + sessionSlot: number; + + /** + * Session ID of the client + * + * @generated from field: string session_id = 2; + */ + sessionId: string; /** * Trigger number (0 for left, 1 for right) * - * @generated from field: int32 trigger = 2; + * @generated from field: int32 trigger = 3; */ trigger: number; /** * trigger value (-32768 to 32767) * - * @generated from field: int32 value = 3; + * @generated from field: int32 value = 4; */ value: number; }; @@ -293,30 +314,37 @@ export const ProtoControllerTriggerSchema: GenMessage = */ export type ProtoControllerStick = Message<"proto.ProtoControllerStick"> & { /** - * Slot number (0-3) + * Session specific slot number (0-3) * - * @generated from field: int32 slot = 1; + * @generated from field: int32 session_slot = 1; */ - slot: number; + sessionSlot: number; + + /** + * Session ID of the client + * + * @generated from field: string session_id = 2; + */ + sessionId: string; /** * Stick number (0 for left, 1 for right) * - * @generated from field: int32 stick = 2; + * @generated from field: int32 stick = 3; */ stick: number; /** * X axis value (-32768 to 32767) * - * @generated from field: int32 x = 3; + * @generated from field: int32 x = 4; */ x: number; /** * Y axis value (-32768 to 32767) * - * @generated from field: int32 y = 4; + * @generated from field: int32 y = 5; */ y: number; }; @@ -335,23 +363,30 @@ export const ProtoControllerStickSchema: GenMessage = /*@_ */ export type ProtoControllerAxis = Message<"proto.ProtoControllerAxis"> & { /** - * Slot number (0-3) + * Session specific slot number (0-3) + * + * @generated from field: int32 session_slot = 1; + */ + sessionSlot: number; + + /** + * Session ID of the client * - * @generated from field: int32 slot = 1; + * @generated from field: string session_id = 2; */ - slot: number; + sessionId: string; /** * Axis number (0 for d-pad horizontal, 1 for d-pad vertical) * - * @generated from field: int32 axis = 2; + * @generated from field: int32 axis = 3; */ axis: number; /** * axis value (-1 to 1) * - * @generated from field: int32 value = 3; + * @generated from field: int32 value = 4; */ value: number; }; @@ -370,30 +405,37 @@ export const ProtoControllerAxisSchema: GenMessage = /*@__P */ export type ProtoControllerRumble = Message<"proto.ProtoControllerRumble"> & { /** - * Slot number (0-3) + * Session specific slot number (0-3) * - * @generated from field: int32 slot = 1; + * @generated from field: int32 session_slot = 1; */ - slot: number; + sessionSlot: number; + + /** + * Session ID of the client + * + * @generated from field: string session_id = 2; + */ + sessionId: string; /** * Low frequency rumble (0-65535) * - * @generated from field: int32 low_frequency = 2; + * @generated from field: int32 low_frequency = 3; */ lowFrequency: number; /** * High frequency rumble (0-65535) * - * @generated from field: int32 high_frequency = 3; + * @generated from field: int32 high_frequency = 4; */ highFrequency: number; /** * Duration in milliseconds * - * @generated from field: int32 duration = 4; + * @generated from field: int32 duration = 5; */ duration: number; }; diff --git a/packages/input/src/webrtc-stream.ts b/packages/input/src/webrtc-stream.ts index b91e5b3c..8b43627b 100644 --- a/packages/input/src/webrtc-stream.ts +++ b/packages/input/src/webrtc-stream.ts @@ -22,6 +22,7 @@ import { P2PMessageStream } from "./streamwrapper"; const NESTRI_PROTOCOL_STREAM_REQUEST = "/nestri-relay/stream-request/1.0.0"; export class WebRTCStream { + private _sessionId: string | null = null; private _p2p: Libp2p | undefined = undefined; private _p2pConn: Connection | undefined = undefined; private _msgStream: P2PMessageStream | undefined = undefined; @@ -128,9 +129,9 @@ export class WebRTCStream { }); this._msgStream.on("session-assigned", (data: ProtoClientRequestRoomStream) => { - const sessionId = data.sessionId; - localStorage.setItem("nestri-session-id", sessionId); - console.log("Session ID assigned:", sessionId, "for room:", data.roomName); + this._sessionId = data.sessionId; + localStorage.setItem("nestri-session-id", this._sessionId); + console.log("Session ID assigned:", this._sessionId, "for room:", data.roomName); }); this._msgStream.on("offer", async (data: ProtoSDP) => { @@ -162,7 +163,7 @@ export class WebRTCStream { this._onConnected?.(null); }); - const clientId = localStorage.getItem("nestri-session-id"); + const clientId = this.getSessionID(); if (clientId) { console.debug("Using existing session ID:", clientId); } @@ -180,8 +181,10 @@ export class WebRTCStream { } } - public getSessionID(): string { - return localStorage.getItem("nestri-session-id") || ""; + public getSessionID(): string | null { + if (this._sessionId === null) + this._sessionId = localStorage.getItem("nestri-session-id"); + return this._sessionId; } // Forces opus to stereo in Chromium browsers, because of course @@ -298,7 +301,7 @@ export class WebRTCStream { // @ts-ignore receiver.jitterBufferTarget = receiver.jitterBufferDelayHint = receiver.playoutDelayHint = 0; } - }, 15); + }, 50); }); } } diff --git a/packages/play-standalone/src/pages/[room].astro b/packages/play-standalone/src/pages/[room].astro index 22e5305b..5556cab3 100644 --- a/packages/play-standalone/src/pages/[room].astro +++ b/packages/play-standalone/src/pages/[room].astro @@ -90,11 +90,7 @@ if (envs_map.size > 0) { let nestriControllers: Controller[] = []; window.addEventListener("gamepadconnected", (e) => { - // Ignore gamepads with id including "nestri" console.log("Gamepad connected:", e.gamepad); - if (e.gamepad.id.toLowerCase().includes("nestri")) - return; - const controller = new Controller({ webrtc: stream, e: e, @@ -106,7 +102,7 @@ if (envs_map.size > 0) { if (e.gamepad.id.toLowerCase().includes("nestri")) return; - let disconnected = nestriControllers.find((c) => c.getLocalSlot() === e.gamepad.index); + let disconnected = nestriControllers.find((c) => c.getSlot() === e.gamepad.index); if (disconnected) { disconnected.dispose(); nestriControllers = nestriControllers.filter((c) => c !== disconnected); diff --git a/packages/relay/internal/common/common.go b/packages/relay/internal/common/common.go index a1c86b77..0a462d56 100644 --- a/packages/relay/internal/common/common.go +++ b/packages/relay/internal/common/common.go @@ -26,7 +26,7 @@ func InitWebRTCAPI() error { mediaEngine := &webrtc.MediaEngine{} // Register our extensions - if err := RegisterExtensions(mediaEngine); err != nil { + if err = RegisterExtensions(mediaEngine); err != nil { return fmt.Errorf("failed to register extensions: %w", err) } diff --git a/packages/relay/internal/core/protocol_stream.go b/packages/relay/internal/core/protocol_stream.go index bff77ccc..19f3934f 100644 --- a/packages/relay/internal/core/protocol_stream.go +++ b/packages/relay/internal/core/protocol_stream.go @@ -7,6 +7,7 @@ import ( "fmt" "io" "log/slog" + "math" "relay/internal/common" "relay/internal/connections" "relay/internal/shared" @@ -176,7 +177,7 @@ func (sp *StreamProtocol) handleStreamRequest(stream network.Stream) { // Create participant for this viewer participant, err := shared.NewParticipant( - "", // session ID will be set if this is a client session + "", stream.Conn().RemotePeer(), ) if err != nil { @@ -189,102 +190,37 @@ func (sp *StreamProtocol) handleStreamRequest(stream network.Stream) { participant.SessionID = session.SessionID } + // Assign peer connection participant.PeerConnection = pc - // Create per-participant tracks - if room.VideoTrack != nil { - participant.VideoTrack, err = webrtc.NewTrackLocalStaticRTP( - room.VideoTrack.Codec(), - "video-"+participant.ID.String(), - "nestri-"+reqMsg.RoomName+"-video", + // Add audio/video tracks + { + localTrack, err := webrtc.NewTrackLocalStaticRTP( + room.AudioCodec, + "participant-"+participant.ID.String(), + "participant-"+participant.ID.String()+"-audio", ) if err != nil { - slog.Error("Failed to create participant video track", "room", reqMsg.RoomName, "err", err) - continue - } - - rtpSender, err := pc.AddTrack(participant.VideoTrack) - if err != nil { - slog.Error("Failed to add participant video track", "room", reqMsg.RoomName, "err", err) - continue + slog.Error("Failed to create track for stream request", "err", err) + return } - - slog.Info("Added video track for participant", - "room", reqMsg.RoomName, - "participant", participant.ID, - "sender_id", fmt.Sprintf("%p", rtpSender)) - - // Relay packets from channel to track (VIDEO) - go func() { - for pkt := range participant.VideoChan { - // Use a context with timeout - ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) - - done := make(chan error, 1) - go func() { - done <- participant.VideoTrack.WriteRTP(pkt) - }() - - select { - case err := <-done: - cancel() - if err != nil { - if !errors.Is(err, io.ErrClosedPipe) { - slog.Debug("Failed to write video", "room", reqMsg.RoomName, "err", err) - } - return - } - case <-ctx.Done(): - cancel() - slog.Error("WriteRTP BLOCKED for >100ms!", - "participant", participant.ID, - "room", reqMsg.RoomName) - // Don't return, continue processing - } - } - }() - } - if room.AudioTrack != nil { - participant.AudioTrack, err = webrtc.NewTrackLocalStaticRTP( - room.AudioTrack.Codec(), - "audio-"+participant.ID.String(), - "nestri-"+reqMsg.RoomName+"-audio", + participant.SetTrack(webrtc.RTPCodecTypeAudio, localTrack) + slog.Debug("Set audio track for requested stream", "room", room.Name) + } + { + localTrack, err := webrtc.NewTrackLocalStaticRTP( + room.VideoCodec, + "participant-"+participant.ID.String(), + "participant-"+participant.ID.String()+"-video", ) if err != nil { - slog.Error("Failed to create participant audio track", "room", reqMsg.RoomName, "err", err) - continue - } - - _, err := pc.AddTrack(participant.AudioTrack) - if err != nil { - slog.Error("Failed to add participant audio track", "room", reqMsg.RoomName, "err", err) - continue + slog.Error("Failed to create track for stream request", "err", err) + return } - - // Relay packets from channel to track (AUDIO) - go func() { - for pkt := range participant.AudioChan { - start := time.Now() - if err := participant.AudioTrack.WriteRTP(pkt); err != nil { - if !errors.Is(err, io.ErrClosedPipe) { - slog.Debug("Failed to write audio to participant", "room", reqMsg.RoomName, "err", err) - } - return - } - duration := time.Since(start) - if duration > 50*time.Millisecond { - slog.Warn("Slow audio WriteRTP detected", - "duration", duration, - "participant", participant.ID, - "room", reqMsg.RoomName) - } - } - }() + participant.SetTrack(webrtc.RTPCodecTypeVideo, localTrack) + slog.Debug("Set video track for requested stream", "room", room.Name) } - // Add participant to room - room.AddParticipant(participant) - // Cleanup on disconnect cleanupParticipantID := participant.ID pc.OnConnectionStateChange(func(state webrtc.PeerConnectionState) { @@ -294,6 +230,9 @@ func (sp *StreamProtocol) handleStreamRequest(stream network.Stream) { slog.Info("Participant disconnected from room", "room", reqMsg.RoomName, "participant", cleanupParticipantID) room.RemoveParticipantByID(cleanupParticipantID) participant.Close() + } else if state == webrtc.PeerConnectionStateConnected { + // Add participant to room when connection is established + room.AddParticipant(participant) } }) @@ -334,33 +273,33 @@ func (sp *StreamProtocol) handleStreamRequest(stream network.Stream) { peerID := stream.Conn().RemotePeer() // Check if it's a controller attach with assigned slot - if attach := msgWrapper.GetControllerAttach(); attach != nil && attach.Slot >= 0 { + if attach := msgWrapper.GetControllerAttach(); attach != nil && attach.SessionSlot >= 0 { if session, ok := sp.relay.ClientSessions.Get(peerID); ok { // Check if slot already tracked hasSlot := false for _, slot := range session.ControllerSlots { - if slot == attach.Slot { + if slot == attach.SessionSlot { hasSlot = true break } } if !hasSlot { - session.ControllerSlots = append(session.ControllerSlots, attach.Slot) + session.ControllerSlots = append(session.ControllerSlots, attach.SessionSlot) session.LastActivity = time.Now() slog.Info("Controller slot assigned to client session", "session", session.SessionID, - "slot", attach.Slot, + "slot", attach.SessionSlot, "total_slots", len(session.ControllerSlots)) } } } // Check if it's a controller detach - if detach := msgWrapper.GetControllerDetach(); detach != nil && detach.Slot >= 0 { + if detach := msgWrapper.GetControllerDetach(); detach != nil && detach.SessionSlot >= 0 { if session, ok := sp.relay.ClientSessions.Get(peerID); ok { newSlots := make([]int32, 0, len(session.ControllerSlots)) for _, slot := range session.ControllerSlots { - if slot != detach.Slot { + if slot != detach.SessionSlot { newSlots = append(newSlots, slot) } } @@ -368,7 +307,7 @@ func (sp *StreamProtocol) handleStreamRequest(stream network.Stream) { session.LastActivity = time.Now() slog.Info("Controller slot removed from client session", "session", session.SessionID, - "slot", detach.Slot, + "slot", detach.SessionSlot, "remaining_slots", len(session.ControllerSlots)) } } @@ -537,19 +476,25 @@ func (sp *StreamProtocol) handleStreamPush(stream network.Stream) { if err != nil { if errors.Is(err, io.EOF) || errors.Is(err, network.ErrReset) { slog.Debug("Stream push connection closed by peer", "peer", stream.Conn().RemotePeer(), "error", err) + if room != nil { + room.Close() + sp.incomingConns.Set(room.Name, nil) + } return } slog.Error("Failed to receive data for stream push", "err", err) _ = stream.Reset() - + if room != nil { + room.Close() + sp.incomingConns.Set(room.Name, nil) + } return } if msgWrapper.MessageBase == nil { slog.Error("No MessageBase in stream push") - _ = stream.Reset() - return + continue } switch msgWrapper.MessageBase.PayloadType { @@ -606,7 +551,7 @@ func (sp *StreamProtocol) handleStreamPush(stream network.Stream) { slog.Error("Failed to add ICE candidate for pushed stream", "err", err) } for _, heldIce := range iceHolder { - if err := conn.pc.AddICECandidate(heldIce); err != nil { + if err = conn.pc.AddICECandidate(heldIce); err != nil { slog.Error("Failed to add held ICE candidate for pushed stream", "err", err) } } @@ -645,6 +590,9 @@ func (sp *StreamProtocol) handleStreamPush(stream network.Stream) { continue } + // Assign room peer connection + room.PeerConnection = pc + pc.OnDataChannel(func(dc *webrtc.DataChannel) { // TODO: Is this the best way to handle DataChannel? Should we just use the map directly? room.DataChannel = connections.NewNestriDataChannel(dc) @@ -708,17 +656,6 @@ func (sp *StreamProtocol) handleStreamPush(stream network.Stream) { }) pc.OnTrack(func(remoteTrack *webrtc.TrackRemote, receiver *webrtc.RTPReceiver) { - localTrack, err := webrtc.NewTrackLocalStaticRTP(remoteTrack.Codec().RTPCodecCapability, remoteTrack.Kind().String(), fmt.Sprintf("nestri-%s-%s", room.Name, remoteTrack.Kind().String())) - if err != nil { - slog.Error("Failed to create local track for pushed stream", "room", room.Name, "track_kind", remoteTrack.Kind().String(), "err", err) - return - } - - slog.Debug("Received track for pushed stream", "room", room.Name, "track_kind", remoteTrack.Kind().String()) - - // Set track for Room - room.SetTrack(remoteTrack.Kind(), localTrack) - // Prepare PlayoutDelayExtension so we don't need to recreate it for each packet playoutExt := &rtp.PlayoutDelayExtension{ MinDelay: 0, @@ -730,6 +667,12 @@ func (sp *StreamProtocol) handleStreamPush(stream network.Stream) { return } + if remoteTrack.Kind() == webrtc.RTPCodecTypeAudio { + room.AudioCodec = remoteTrack.Codec().RTPCodecCapability + } else if remoteTrack.Kind() == webrtc.RTPCodecTypeVideo { + room.VideoCodec = remoteTrack.Codec().RTPCodecCapability + } + for { rtpPacket, _, err := remoteTrack.ReadRTP() if err != nil { @@ -741,19 +684,61 @@ func (sp *StreamProtocol) handleStreamPush(stream network.Stream) { // Use PlayoutDelayExtension for low latency, if set for this track kind if extID, ok := common.GetExtension(remoteTrack.Kind(), common.ExtensionPlayoutDelay); ok { - if err := rtpPacket.SetExtension(extID, playoutPayload); err != nil { + if err = rtpPacket.SetExtension(extID, playoutPayload); err != nil { slog.Error("Failed to set PlayoutDelayExtension for room", "room", room.Name, "err", err) continue } } - room.BroadcastPacket(remoteTrack.Kind(), rtpPacket) + // Calculate differences + var timeDiff int64 + var sequenceDiff int + + if remoteTrack.Kind() == webrtc.RTPCodecTypeVideo { + timeDiff = int64(rtpPacket.Timestamp) - int64(room.LastVideoTimestamp) + if !room.VideoTimestampSet { + timeDiff = 0 + room.VideoTimestampSet = true + } else if timeDiff < -(math.MaxUint32 / 10) { + timeDiff += math.MaxUint32 + 1 + } + + sequenceDiff = int(rtpPacket.SequenceNumber) - int(room.LastVideoSequenceNumber) + if !room.VideoSequenceSet { + sequenceDiff = 0 + room.VideoSequenceSet = true + } else if sequenceDiff < -(math.MaxUint16 / 10) { + sequenceDiff += math.MaxUint16 + 1 + } + + room.LastVideoTimestamp = rtpPacket.Timestamp + room.LastVideoSequenceNumber = rtpPacket.SequenceNumber + } else { // Audio + timeDiff = int64(rtpPacket.Timestamp) - int64(room.LastAudioTimestamp) + if !room.AudioTimestampSet { + timeDiff = 0 + room.AudioTimestampSet = true + } else if timeDiff < -(math.MaxUint32 / 10) { + timeDiff += math.MaxUint32 + 1 + } + + sequenceDiff = int(rtpPacket.SequenceNumber) - int(room.LastAudioSequenceNumber) + if !room.AudioSequenceSet { + sequenceDiff = 0 + room.AudioSequenceSet = true + } else if sequenceDiff < -(math.MaxUint16 / 10) { + sequenceDiff += math.MaxUint16 + 1 + } + + room.LastAudioTimestamp = rtpPacket.Timestamp + room.LastAudioSequenceNumber = rtpPacket.SequenceNumber + } + + // Broadcast with differences + room.BroadcastPacketRetimed(remoteTrack.Kind(), rtpPacket, timeDiff, sequenceDiff) } slog.Debug("Track closed for room", "room", room.Name, "track_kind", remoteTrack.Kind().String()) - - // Cleanup the track from the room - room.SetTrack(remoteTrack.Kind(), nil) }) // Set the remote description diff --git a/packages/relay/internal/core/room.go b/packages/relay/internal/core/room.go index 78e5e5c4..c4a5b2a9 100644 --- a/packages/relay/internal/core/room.go +++ b/packages/relay/internal/core/room.go @@ -45,7 +45,7 @@ func (r *Relay) DeleteRoomIfEmpty(room *shared.Room) { if room == nil { return } - if room.Participants.Len() == 0 && r.LocalRooms.Has(room.ID) { + if len(room.Participants) <= 0 && r.LocalRooms.Has(room.ID) { slog.Debug("Deleting empty room without participants", "room", room.Name) r.LocalRooms.Delete(room.ID) err := room.PeerConnection.Close() diff --git a/packages/relay/internal/core/state.go b/packages/relay/internal/core/state.go index d4d088c2..9323eadc 100644 --- a/packages/relay/internal/core/state.go +++ b/packages/relay/internal/core/state.go @@ -195,18 +195,18 @@ func (r *Relay) updateMeshRoomStates(peerID peer.ID, states []shared.RoomInfo) { } // If previously did not exist, but does now, request a connection if participants exist for our room - existed := r.Rooms.Has(state.ID.String()) + /*existed := r.Rooms.Has(state.ID.String()) if !existed { // Request connection to this peer if we have participants in our local room if room, ok := r.LocalRooms.Get(state.ID); ok { - if room.Participants.Len() > 0 { + if len(room.Participants) > 0 { slog.Debug("Got new remote room state, we locally have participants for, requesting stream", "room_name", room.Name, "peer", peerID) if err := r.StreamProtocol.RequestStream(context.Background(), room, peerID); err != nil { slog.Error("Failed to request stream for new remote room state", "room_name", room.Name, "peer", peerID, "err", err) } } } - } + }*/ r.Rooms.Set(state.ID.String(), state) } diff --git a/packages/relay/internal/proto/types.pb.go b/packages/relay/internal/proto/types.pb.go index dae068ac..e71fa071 100644 --- a/packages/relay/internal/proto/types.pb.go +++ b/packages/relay/internal/proto/types.pb.go @@ -363,9 +363,9 @@ func (x *ProtoKeyUp) GetKey() int32 { // ControllerAttach message type ProtoControllerAttach struct { state protoimpl.MessageState `protogen:"open.v1"` - Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"` // One of the following enums: "ps", "xbox" or "switch" - Slot int32 `protobuf:"varint,2,opt,name=slot,proto3" json:"slot,omitempty"` // Slot number (0-3) - SessionId string `protobuf:"bytes,3,opt,name=session_id,json=sessionId,proto3" json:"session_id,omitempty"` // Session ID of the client attaching the controller + Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"` // One of the following enums: "ps", "xbox" or "switch" + SessionSlot int32 `protobuf:"varint,2,opt,name=session_slot,json=sessionSlot,proto3" json:"session_slot,omitempty"` // Session specific slot number (0-3) + SessionId string `protobuf:"bytes,3,opt,name=session_id,json=sessionId,proto3" json:"session_id,omitempty"` // Session ID of the client unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -407,9 +407,9 @@ func (x *ProtoControllerAttach) GetId() string { return "" } -func (x *ProtoControllerAttach) GetSlot() int32 { +func (x *ProtoControllerAttach) GetSessionSlot() int32 { if x != nil { - return x.Slot + return x.SessionSlot } return 0 } @@ -424,7 +424,8 @@ func (x *ProtoControllerAttach) GetSessionId() string { // ControllerDetach message type ProtoControllerDetach struct { state protoimpl.MessageState `protogen:"open.v1"` - Slot int32 `protobuf:"varint,1,opt,name=slot,proto3" json:"slot,omitempty"` // Slot number (0-3) + SessionSlot int32 `protobuf:"varint,1,opt,name=session_slot,json=sessionSlot,proto3" json:"session_slot,omitempty"` // Session specific slot number (0-3) + SessionId string `protobuf:"bytes,2,opt,name=session_id,json=sessionId,proto3" json:"session_id,omitempty"` // Session ID of the client unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -459,19 +460,27 @@ func (*ProtoControllerDetach) Descriptor() ([]byte, []int) { return file_types_proto_rawDescGZIP(), []int{8} } -func (x *ProtoControllerDetach) GetSlot() int32 { +func (x *ProtoControllerDetach) GetSessionSlot() int32 { if x != nil { - return x.Slot + return x.SessionSlot } return 0 } +func (x *ProtoControllerDetach) GetSessionId() string { + if x != nil { + return x.SessionId + } + return "" +} + // ControllerButton message type ProtoControllerButton struct { state protoimpl.MessageState `protogen:"open.v1"` - Slot int32 `protobuf:"varint,1,opt,name=slot,proto3" json:"slot,omitempty"` // Slot number (0-3) - Button int32 `protobuf:"varint,2,opt,name=button,proto3" json:"button,omitempty"` // Button code (linux input event code) - Pressed bool `protobuf:"varint,3,opt,name=pressed,proto3" json:"pressed,omitempty"` // true if pressed, false if released + SessionSlot int32 `protobuf:"varint,1,opt,name=session_slot,json=sessionSlot,proto3" json:"session_slot,omitempty"` // Session specific slot number (0-3) + SessionId string `protobuf:"bytes,2,opt,name=session_id,json=sessionId,proto3" json:"session_id,omitempty"` // Session ID of the client + Button int32 `protobuf:"varint,3,opt,name=button,proto3" json:"button,omitempty"` // Button code (linux input event code) + Pressed bool `protobuf:"varint,4,opt,name=pressed,proto3" json:"pressed,omitempty"` // true if pressed, false if released unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -506,13 +515,20 @@ func (*ProtoControllerButton) Descriptor() ([]byte, []int) { return file_types_proto_rawDescGZIP(), []int{9} } -func (x *ProtoControllerButton) GetSlot() int32 { +func (x *ProtoControllerButton) GetSessionSlot() int32 { if x != nil { - return x.Slot + return x.SessionSlot } return 0 } +func (x *ProtoControllerButton) GetSessionId() string { + if x != nil { + return x.SessionId + } + return "" +} + func (x *ProtoControllerButton) GetButton() int32 { if x != nil { return x.Button @@ -530,9 +546,10 @@ func (x *ProtoControllerButton) GetPressed() bool { // ControllerTriggers message type ProtoControllerTrigger struct { state protoimpl.MessageState `protogen:"open.v1"` - Slot int32 `protobuf:"varint,1,opt,name=slot,proto3" json:"slot,omitempty"` // Slot number (0-3) - Trigger int32 `protobuf:"varint,2,opt,name=trigger,proto3" json:"trigger,omitempty"` // Trigger number (0 for left, 1 for right) - Value int32 `protobuf:"varint,3,opt,name=value,proto3" json:"value,omitempty"` // trigger value (-32768 to 32767) + SessionSlot int32 `protobuf:"varint,1,opt,name=session_slot,json=sessionSlot,proto3" json:"session_slot,omitempty"` // Session specific slot number (0-3) + SessionId string `protobuf:"bytes,2,opt,name=session_id,json=sessionId,proto3" json:"session_id,omitempty"` // Session ID of the client + Trigger int32 `protobuf:"varint,3,opt,name=trigger,proto3" json:"trigger,omitempty"` // Trigger number (0 for left, 1 for right) + Value int32 `protobuf:"varint,4,opt,name=value,proto3" json:"value,omitempty"` // trigger value (-32768 to 32767) unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -567,13 +584,20 @@ func (*ProtoControllerTrigger) Descriptor() ([]byte, []int) { return file_types_proto_rawDescGZIP(), []int{10} } -func (x *ProtoControllerTrigger) GetSlot() int32 { +func (x *ProtoControllerTrigger) GetSessionSlot() int32 { if x != nil { - return x.Slot + return x.SessionSlot } return 0 } +func (x *ProtoControllerTrigger) GetSessionId() string { + if x != nil { + return x.SessionId + } + return "" +} + func (x *ProtoControllerTrigger) GetTrigger() int32 { if x != nil { return x.Trigger @@ -591,10 +615,11 @@ func (x *ProtoControllerTrigger) GetValue() int32 { // ControllerSticks message type ProtoControllerStick struct { state protoimpl.MessageState `protogen:"open.v1"` - Slot int32 `protobuf:"varint,1,opt,name=slot,proto3" json:"slot,omitempty"` // Slot number (0-3) - Stick int32 `protobuf:"varint,2,opt,name=stick,proto3" json:"stick,omitempty"` // Stick number (0 for left, 1 for right) - X int32 `protobuf:"varint,3,opt,name=x,proto3" json:"x,omitempty"` // X axis value (-32768 to 32767) - Y int32 `protobuf:"varint,4,opt,name=y,proto3" json:"y,omitempty"` // Y axis value (-32768 to 32767) + SessionSlot int32 `protobuf:"varint,1,opt,name=session_slot,json=sessionSlot,proto3" json:"session_slot,omitempty"` // Session specific slot number (0-3) + SessionId string `protobuf:"bytes,2,opt,name=session_id,json=sessionId,proto3" json:"session_id,omitempty"` // Session ID of the client + Stick int32 `protobuf:"varint,3,opt,name=stick,proto3" json:"stick,omitempty"` // Stick number (0 for left, 1 for right) + X int32 `protobuf:"varint,4,opt,name=x,proto3" json:"x,omitempty"` // X axis value (-32768 to 32767) + Y int32 `protobuf:"varint,5,opt,name=y,proto3" json:"y,omitempty"` // Y axis value (-32768 to 32767) unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -629,13 +654,20 @@ func (*ProtoControllerStick) Descriptor() ([]byte, []int) { return file_types_proto_rawDescGZIP(), []int{11} } -func (x *ProtoControllerStick) GetSlot() int32 { +func (x *ProtoControllerStick) GetSessionSlot() int32 { if x != nil { - return x.Slot + return x.SessionSlot } return 0 } +func (x *ProtoControllerStick) GetSessionId() string { + if x != nil { + return x.SessionId + } + return "" +} + func (x *ProtoControllerStick) GetStick() int32 { if x != nil { return x.Stick @@ -660,9 +692,10 @@ func (x *ProtoControllerStick) GetY() int32 { // ControllerAxis message type ProtoControllerAxis struct { state protoimpl.MessageState `protogen:"open.v1"` - Slot int32 `protobuf:"varint,1,opt,name=slot,proto3" json:"slot,omitempty"` // Slot number (0-3) - Axis int32 `protobuf:"varint,2,opt,name=axis,proto3" json:"axis,omitempty"` // Axis number (0 for d-pad horizontal, 1 for d-pad vertical) - Value int32 `protobuf:"varint,3,opt,name=value,proto3" json:"value,omitempty"` // axis value (-1 to 1) + SessionSlot int32 `protobuf:"varint,1,opt,name=session_slot,json=sessionSlot,proto3" json:"session_slot,omitempty"` // Session specific slot number (0-3) + SessionId string `protobuf:"bytes,2,opt,name=session_id,json=sessionId,proto3" json:"session_id,omitempty"` // Session ID of the client + Axis int32 `protobuf:"varint,3,opt,name=axis,proto3" json:"axis,omitempty"` // Axis number (0 for d-pad horizontal, 1 for d-pad vertical) + Value int32 `protobuf:"varint,4,opt,name=value,proto3" json:"value,omitempty"` // axis value (-1 to 1) unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -697,13 +730,20 @@ func (*ProtoControllerAxis) Descriptor() ([]byte, []int) { return file_types_proto_rawDescGZIP(), []int{12} } -func (x *ProtoControllerAxis) GetSlot() int32 { +func (x *ProtoControllerAxis) GetSessionSlot() int32 { if x != nil { - return x.Slot + return x.SessionSlot } return 0 } +func (x *ProtoControllerAxis) GetSessionId() string { + if x != nil { + return x.SessionId + } + return "" +} + func (x *ProtoControllerAxis) GetAxis() int32 { if x != nil { return x.Axis @@ -721,10 +761,11 @@ func (x *ProtoControllerAxis) GetValue() int32 { // ControllerRumble message type ProtoControllerRumble struct { state protoimpl.MessageState `protogen:"open.v1"` - Slot int32 `protobuf:"varint,1,opt,name=slot,proto3" json:"slot,omitempty"` // Slot number (0-3) - LowFrequency int32 `protobuf:"varint,2,opt,name=low_frequency,json=lowFrequency,proto3" json:"low_frequency,omitempty"` // Low frequency rumble (0-65535) - HighFrequency int32 `protobuf:"varint,3,opt,name=high_frequency,json=highFrequency,proto3" json:"high_frequency,omitempty"` // High frequency rumble (0-65535) - Duration int32 `protobuf:"varint,4,opt,name=duration,proto3" json:"duration,omitempty"` // Duration in milliseconds + SessionSlot int32 `protobuf:"varint,1,opt,name=session_slot,json=sessionSlot,proto3" json:"session_slot,omitempty"` // Session specific slot number (0-3) + SessionId string `protobuf:"bytes,2,opt,name=session_id,json=sessionId,proto3" json:"session_id,omitempty"` // Session ID of the client + LowFrequency int32 `protobuf:"varint,3,opt,name=low_frequency,json=lowFrequency,proto3" json:"low_frequency,omitempty"` // Low frequency rumble (0-65535) + HighFrequency int32 `protobuf:"varint,4,opt,name=high_frequency,json=highFrequency,proto3" json:"high_frequency,omitempty"` // High frequency rumble (0-65535) + Duration int32 `protobuf:"varint,5,opt,name=duration,proto3" json:"duration,omitempty"` // Duration in milliseconds unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -759,13 +800,20 @@ func (*ProtoControllerRumble) Descriptor() ([]byte, []int) { return file_types_proto_rawDescGZIP(), []int{13} } -func (x *ProtoControllerRumble) GetSlot() int32 { +func (x *ProtoControllerRumble) GetSessionSlot() int32 { if x != nil { - return x.Slot + return x.SessionSlot } return 0 } +func (x *ProtoControllerRumble) GetSessionId() string { + if x != nil { + return x.SessionId + } + return "" +} + func (x *ProtoControllerRumble) GetLowFrequency() int32 { if x != nil { return x.LowFrequency @@ -1215,36 +1263,48 @@ const file_types_proto_rawDesc = "" + "\x03key\x18\x01 \x01(\x05R\x03key\"\x1e\n" + "\n" + "ProtoKeyUp\x12\x10\n" + - "\x03key\x18\x01 \x01(\x05R\x03key\"Z\n" + + "\x03key\x18\x01 \x01(\x05R\x03key\"i\n" + "\x15ProtoControllerAttach\x12\x0e\n" + - "\x02id\x18\x01 \x01(\tR\x02id\x12\x12\n" + - "\x04slot\x18\x02 \x01(\x05R\x04slot\x12\x1d\n" + + "\x02id\x18\x01 \x01(\tR\x02id\x12!\n" + + "\fsession_slot\x18\x02 \x01(\x05R\vsessionSlot\x12\x1d\n" + + "\n" + + "session_id\x18\x03 \x01(\tR\tsessionId\"Y\n" + + "\x15ProtoControllerDetach\x12!\n" + + "\fsession_slot\x18\x01 \x01(\x05R\vsessionSlot\x12\x1d\n" + + "\n" + + "session_id\x18\x02 \x01(\tR\tsessionId\"\x8b\x01\n" + + "\x15ProtoControllerButton\x12!\n" + + "\fsession_slot\x18\x01 \x01(\x05R\vsessionSlot\x12\x1d\n" + + "\n" + + "session_id\x18\x02 \x01(\tR\tsessionId\x12\x16\n" + + "\x06button\x18\x03 \x01(\x05R\x06button\x12\x18\n" + + "\apressed\x18\x04 \x01(\bR\apressed\"\x8a\x01\n" + + "\x16ProtoControllerTrigger\x12!\n" + + "\fsession_slot\x18\x01 \x01(\x05R\vsessionSlot\x12\x1d\n" + + "\n" + + "session_id\x18\x02 \x01(\tR\tsessionId\x12\x18\n" + + "\atrigger\x18\x03 \x01(\x05R\atrigger\x12\x14\n" + + "\x05value\x18\x04 \x01(\x05R\x05value\"\x8a\x01\n" + + "\x14ProtoControllerStick\x12!\n" + + "\fsession_slot\x18\x01 \x01(\x05R\vsessionSlot\x12\x1d\n" + + "\n" + + "session_id\x18\x02 \x01(\tR\tsessionId\x12\x14\n" + + "\x05stick\x18\x03 \x01(\x05R\x05stick\x12\f\n" + + "\x01x\x18\x04 \x01(\x05R\x01x\x12\f\n" + + "\x01y\x18\x05 \x01(\x05R\x01y\"\x81\x01\n" + + "\x13ProtoControllerAxis\x12!\n" + + "\fsession_slot\x18\x01 \x01(\x05R\vsessionSlot\x12\x1d\n" + + "\n" + + "session_id\x18\x02 \x01(\tR\tsessionId\x12\x12\n" + + "\x04axis\x18\x03 \x01(\x05R\x04axis\x12\x14\n" + + "\x05value\x18\x04 \x01(\x05R\x05value\"\xc1\x01\n" + + "\x15ProtoControllerRumble\x12!\n" + + "\fsession_slot\x18\x01 \x01(\x05R\vsessionSlot\x12\x1d\n" + "\n" + - "session_id\x18\x03 \x01(\tR\tsessionId\"+\n" + - "\x15ProtoControllerDetach\x12\x12\n" + - "\x04slot\x18\x01 \x01(\x05R\x04slot\"]\n" + - "\x15ProtoControllerButton\x12\x12\n" + - "\x04slot\x18\x01 \x01(\x05R\x04slot\x12\x16\n" + - "\x06button\x18\x02 \x01(\x05R\x06button\x12\x18\n" + - "\apressed\x18\x03 \x01(\bR\apressed\"\\\n" + - "\x16ProtoControllerTrigger\x12\x12\n" + - "\x04slot\x18\x01 \x01(\x05R\x04slot\x12\x18\n" + - "\atrigger\x18\x02 \x01(\x05R\atrigger\x12\x14\n" + - "\x05value\x18\x03 \x01(\x05R\x05value\"\\\n" + - "\x14ProtoControllerStick\x12\x12\n" + - "\x04slot\x18\x01 \x01(\x05R\x04slot\x12\x14\n" + - "\x05stick\x18\x02 \x01(\x05R\x05stick\x12\f\n" + - "\x01x\x18\x03 \x01(\x05R\x01x\x12\f\n" + - "\x01y\x18\x04 \x01(\x05R\x01y\"S\n" + - "\x13ProtoControllerAxis\x12\x12\n" + - "\x04slot\x18\x01 \x01(\x05R\x04slot\x12\x12\n" + - "\x04axis\x18\x02 \x01(\x05R\x04axis\x12\x14\n" + - "\x05value\x18\x03 \x01(\x05R\x05value\"\x93\x01\n" + - "\x15ProtoControllerRumble\x12\x12\n" + - "\x04slot\x18\x01 \x01(\x05R\x04slot\x12#\n" + - "\rlow_frequency\x18\x02 \x01(\x05R\flowFrequency\x12%\n" + - "\x0ehigh_frequency\x18\x03 \x01(\x05R\rhighFrequency\x12\x1a\n" + - "\bduration\x18\x04 \x01(\x05R\bduration\"\xde\x01\n" + + "session_id\x18\x02 \x01(\tR\tsessionId\x12#\n" + + "\rlow_frequency\x18\x03 \x01(\x05R\flowFrequency\x12%\n" + + "\x0ehigh_frequency\x18\x04 \x01(\x05R\rhighFrequency\x12\x1a\n" + + "\bduration\x18\x05 \x01(\x05R\bduration\"\xde\x01\n" + "\x13RTCIceCandidateInit\x12\x1c\n" + "\tcandidate\x18\x01 \x01(\tR\tcandidate\x12)\n" + "\rsdpMLineIndex\x18\x02 \x01(\rH\x00R\rsdpMLineIndex\x88\x01\x01\x12\x1b\n" + diff --git a/packages/relay/internal/shared/participant.go b/packages/relay/internal/shared/participant.go index 6916be95..74b87d60 100644 --- a/packages/relay/internal/shared/participant.go +++ b/packages/relay/internal/shared/participant.go @@ -1,14 +1,16 @@ package shared import ( + "errors" "fmt" + "io" "log/slog" "relay/internal/common" "relay/internal/connections" + "sync" "github.com/libp2p/go-libp2p/core/peer" "github.com/oklog/ulid/v2" - "github.com/pion/rtp" "github.com/pion/webrtc/v4" ) @@ -22,8 +24,15 @@ type Participant struct { // Per-viewer tracks and channels VideoTrack *webrtc.TrackLocalStaticRTP AudioTrack *webrtc.TrackLocalStaticRTP - VideoChan chan *rtp.Packet - AudioChan chan *rtp.Packet + + // Per-viewer RTP state for retiming + VideoSequenceNumber uint16 + VideoTimestamp uint32 + AudioSequenceNumber uint16 + AudioTimestamp uint32 + + packetQueue chan *participantPacket + closeOnce sync.Once } func NewParticipant(sessionID string, peerID peer.ID) (*Participant, error) { @@ -31,24 +40,50 @@ func NewParticipant(sessionID string, peerID peer.ID) (*Participant, error) { if err != nil { return nil, fmt.Errorf("failed to create ULID for Participant: %w", err) } - return &Participant{ - ID: id, - SessionID: sessionID, - PeerID: peerID, - VideoChan: make(chan *rtp.Packet, 500), - AudioChan: make(chan *rtp.Packet, 100), - }, nil + p := &Participant{ + ID: id, + SessionID: sessionID, + PeerID: peerID, + VideoSequenceNumber: 0, + VideoTimestamp: 0, + AudioSequenceNumber: 0, + AudioTimestamp: 0, + packetQueue: make(chan *participantPacket, 1000), + } + + go p.packetWriter() + + return p, nil +} + +// SetTrack sets audio/video track for Participant +func (p *Participant) SetTrack(trackType webrtc.RTPCodecType, track *webrtc.TrackLocalStaticRTP) { + switch trackType { + case webrtc.RTPCodecTypeAudio: + p.AudioTrack = track + _, err := p.PeerConnection.AddTrack(track) + if err != nil { + slog.Error("Failed to add Participant audio track", err) + } + case webrtc.RTPCodecTypeVideo: + p.VideoTrack = track + _, err := p.PeerConnection.AddTrack(track) + if err != nil { + slog.Error("Failed to add Participant video track", err) + } + default: + slog.Warn("Unknown track type", "participant", p.ID, "trackType", trackType) + } } // Close cleans up participant resources func (p *Participant) Close() { - if p.VideoChan != nil { - close(p.VideoChan) - p.VideoChan = nil - } - if p.AudioChan != nil { - close(p.AudioChan) - p.AudioChan = nil + if p.DataChannel != nil { + err := p.DataChannel.Close() + if err != nil { + slog.Error("Failed to close Participant DataChannel", err) + } + p.DataChannel = nil } if p.PeerConnection != nil { err := p.PeerConnection.Close() @@ -57,4 +92,45 @@ func (p *Participant) Close() { } p.PeerConnection = nil } + if p.VideoTrack != nil { + p.VideoTrack = nil + } + if p.AudioTrack != nil { + p.AudioTrack = nil + } +} + +func (p *Participant) packetWriter() { + for pkt := range p.packetQueue { + var track *webrtc.TrackLocalStaticRTP + var sequenceNumber uint16 + var timestamp uint32 + + // No mutex needed - only this goroutine modifies these + if pkt.kind == webrtc.RTPCodecTypeAudio { + track = p.AudioTrack + p.AudioSequenceNumber = uint16(int(p.AudioSequenceNumber) + pkt.sequenceDiff) + p.AudioTimestamp = uint32(int64(p.AudioTimestamp) + pkt.timeDiff) + sequenceNumber = p.AudioSequenceNumber + timestamp = p.AudioTimestamp + } else { + track = p.VideoTrack + p.VideoSequenceNumber = uint16(int(p.VideoSequenceNumber) + pkt.sequenceDiff) + p.VideoTimestamp = uint32(int64(p.VideoTimestamp) + pkt.timeDiff) + sequenceNumber = p.VideoSequenceNumber + timestamp = p.VideoTimestamp + } + + if track != nil { + pkt.packet.SequenceNumber = sequenceNumber + pkt.packet.Timestamp = timestamp + + if err := track.WriteRTP(pkt.packet); err != nil && !errors.Is(err, io.ErrClosedPipe) { + slog.Error("WriteRTP failed", "participant", p.ID, "kind", pkt.kind, "err", err) + } + } + + // Return packet struct to pool + participantPacketPool.Put(pkt) + } } diff --git a/packages/relay/internal/shared/room.go b/packages/relay/internal/shared/room.go index 5ef09361..516fdc92 100644 --- a/packages/relay/internal/shared/room.go +++ b/packages/relay/internal/shared/room.go @@ -2,9 +2,9 @@ package shared import ( "log/slog" - "relay/internal/common" "relay/internal/connections" - "time" + "sync" + "sync/atomic" "github.com/libp2p/go-libp2p/core/peer" "github.com/oklog/ulid/v2" @@ -12,6 +12,19 @@ import ( "github.com/pion/webrtc/v4" ) +var participantPacketPool = sync.Pool{ + New: func() interface{} { + return &participantPacket{} + }, +} + +type participantPacket struct { + kind webrtc.RTPCodecType + packet *rtp.Packet + timeDiff int64 + sequenceDiff int +} + type RoomInfo struct { ID ulid.ULID `json:"id"` Name string `json:"name"` @@ -20,16 +33,27 @@ type RoomInfo struct { type Room struct { RoomInfo + AudioCodec webrtc.RTPCodecCapability + VideoCodec webrtc.RTPCodecCapability PeerConnection *webrtc.PeerConnection - AudioTrack *webrtc.TrackLocalStaticRTP - VideoTrack *webrtc.TrackLocalStaticRTP DataChannel *connections.NestriDataChannel - Participants *common.SafeMap[ulid.ULID, *Participant] - // Broadcast queues (unbuffered, fan-out happens async) - videoBroadcastChan chan *rtp.Packet - audioBroadcastChan chan *rtp.Packet - broadcastStop chan struct{} + // Atomic pointer to slice of participant channels + participantChannels atomic.Pointer[[]chan<- *participantPacket] + participantsMtx sync.Mutex // Use only for add/remove + + Participants map[ulid.ULID]*Participant // Keep general track of Participant(s) + + // Track last seen values to calculate diffs + LastVideoTimestamp uint32 + LastVideoSequenceNumber uint16 + LastAudioTimestamp uint32 + LastAudioSequenceNumber uint16 + + VideoTimestampSet bool + VideoSequenceSet bool + AudioTimestampSet bool + AudioSequenceSet bool } func NewRoom(name string, roomID ulid.ULID, ownerID peer.ID) *Room { @@ -39,133 +63,109 @@ func NewRoom(name string, roomID ulid.ULID, ownerID peer.ID) *Room { Name: name, OwnerID: ownerID, }, - Participants: common.NewSafeMap[ulid.ULID, *Participant](), - videoBroadcastChan: make(chan *rtp.Packet, 1000), // Large buffer for incoming packets - audioBroadcastChan: make(chan *rtp.Packet, 500), - broadcastStop: make(chan struct{}), + PeerConnection: nil, + DataChannel: nil, + Participants: make(map[ulid.ULID]*Participant), } - // Start async broadcasters - go r.videoBroadcaster() - go r.audioBroadcaster() + emptyChannels := make([]chan<- *participantPacket, 0) + r.participantChannels.Store(&emptyChannels) return r } +// Close closes up Room (stream ended) +func (r *Room) Close() { + if r.DataChannel != nil { + err := r.DataChannel.Close() + if err != nil { + slog.Error("Failed to close Room DataChannel", err) + } + r.DataChannel = nil + } + if r.PeerConnection != nil { + err := r.PeerConnection.Close() + if err != nil { + slog.Error("Failed to close Room PeerConnection", err) + } + r.PeerConnection = nil + } +} + // AddParticipant adds a Participant to a Room func (r *Room) AddParticipant(participant *Participant) { - slog.Debug("Adding participant to room", "participant", participant.ID, "room", r.Name) - r.Participants.Set(participant.ID, participant) + r.participantsMtx.Lock() + defer r.participantsMtx.Unlock() + + r.Participants[participant.ID] = participant + + // Update channel slice atomically + current := r.participantChannels.Load() + newChannels := make([]chan<- *participantPacket, len(*current)+1) + copy(newChannels, *current) + newChannels[len(*current)] = participant.packetQueue + + r.participantChannels.Store(&newChannels) + + slog.Debug("Added participant", "participant", participant.ID, "room", r.Name) } // RemoveParticipantByID removes a Participant from a Room by participant's ID func (r *Room) RemoveParticipantByID(pID ulid.ULID) { - if _, ok := r.Participants.Get(pID); ok { - r.Participants.Delete(pID) - } -} + r.participantsMtx.Lock() + defer r.participantsMtx.Unlock() -// IsOnline checks if the room is online (has both audio and video tracks) -func (r *Room) IsOnline() bool { - return r.AudioTrack != nil && r.VideoTrack != nil -} - -func (r *Room) SetTrack(trackType webrtc.RTPCodecType, track *webrtc.TrackLocalStaticRTP) { - switch trackType { - case webrtc.RTPCodecTypeAudio: - r.AudioTrack = track - case webrtc.RTPCodecTypeVideo: - r.VideoTrack = track - default: - slog.Warn("Unknown track type", "room", r.Name, "trackType", trackType) + participant, ok := r.Participants[pID] + if !ok { + return } -} -// BroadcastPacket enqueues packet for async broadcast (non-blocking) -func (r *Room) BroadcastPacket(kind webrtc.RTPCodecType, pkt *rtp.Packet) { - start := time.Now() - if kind == webrtc.RTPCodecTypeVideo { - select { - case r.videoBroadcastChan <- pkt: - duration := time.Since(start) - if duration > 10*time.Millisecond { - slog.Warn("Slow video broadcast enqueue", "duration", duration, "room", r.Name) - } - default: - // Broadcast queue full - system overload, drop packet globally - slog.Warn("Video broadcast queue full, dropping packet", "room", r.Name) - } - } else { - select { - case r.audioBroadcastChan <- pkt: - duration := time.Since(start) - if duration > 10*time.Millisecond { - slog.Warn("Slow audio broadcast enqueue", "duration", duration, "room", r.Name) - } - default: - slog.Warn("Audio broadcast queue full, dropping packet", "room", r.Name) + delete(r.Participants, pID) + + // Update channel slice + current := r.participantChannels.Load() + newChannels := make([]chan<- *participantPacket, 0, len(*current)-1) + for _, ch := range *current { + if ch != participant.packetQueue { + newChannels = append(newChannels, ch) } } + + r.participantChannels.Store(&newChannels) + + slog.Debug("Removed participant", "participant", pID, "room", r.Name) } -// Close stops the broadcasters -func (r *Room) Close() { - close(r.broadcastStop) - close(r.videoBroadcastChan) - close(r.audioBroadcastChan) +// IsOnline checks if the room is online +func (r *Room) IsOnline() bool { + return r.PeerConnection != nil } -// videoBroadcaster runs async fan-out for video packets -func (r *Room) videoBroadcaster() { - for { - select { - case pkt := <-r.videoBroadcastChan: - // Fan out to all participants without blocking - r.Participants.Range(func(_ ulid.ULID, participant *Participant) bool { - if participant.VideoChan != nil { - // Clone packet for each participant to avoid shared pointer issues - clonedPkt := pkt.Clone() - select { - case participant.VideoChan <- clonedPkt: - // Sent - default: - // Participant slow, drop packet - slog.Debug("Dropped video packet for slow participant", - "room", r.Name, - "participant", participant.ID) - } - } - return true - }) - case <-r.broadcastStop: - return - } +func (r *Room) BroadcastPacketRetimed(kind webrtc.RTPCodecType, pkt *rtp.Packet, timeDiff int64, sequenceDiff int) { + // Lock-free load of channel slice + channels := r.participantChannels.Load() + + // no participants.. + if len(*channels) == 0 { + return } -} -// audioBroadcaster runs async fan-out for audio packets -func (r *Room) audioBroadcaster() { - for { + // Send to each participant channel (non-blocking) + for i, ch := range *channels { + // Get packet struct from pool + pp := participantPacketPool.Get().(*participantPacket) + pp.kind = kind + pp.packet = pkt.Clone() + pp.timeDiff = timeDiff + pp.sequenceDiff = sequenceDiff + select { - case pkt := <-r.audioBroadcastChan: - r.Participants.Range(func(_ ulid.ULID, participant *Participant) bool { - if participant.AudioChan != nil { - // Clone packet for each participant to avoid shared pointer issues - clonedPkt := pkt.Clone() - select { - case participant.AudioChan <- clonedPkt: - // Sent - default: - // Participant slow, drop packet - slog.Debug("Dropped audio packet for slow participant", - "room", r.Name, - "participant", participant.ID) - } - } - return true - }) - case <-r.broadcastStop: - return + case ch <- pp: + // Sent successfully + default: + // Channel full, drop packet, log? + slog.Warn("Channel full, dropping packet", "channel_index", i) + participantPacketPool.Put(pp) } } } diff --git a/packages/scripts/entrypoint.sh b/packages/scripts/entrypoint.sh index 8e753d42..3342035d 100644 --- a/packages/scripts/entrypoint.sh +++ b/packages/scripts/entrypoint.sh @@ -15,13 +15,13 @@ NVIDIA_INSTALLER_DIR="/tmp" TIMEOUT_SECONDS=10 ENTCMD_PREFIX="" -# Ensures user directory ownership -chown_user_directory() { +# Ensures user ownership across directories +handle_user_permissions() { if ! $ENTCMD_PREFIX chown "${NESTRI_USER}:${NESTRI_USER}" "${NESTRI_HOME}" 2>/dev/null; then echo "Error: Failed to change ownership of ${NESTRI_HOME} to ${NESTRI_USER}:${NESTRI_USER}" >&2 return 1 fi - # Also apply to .cache separately + # Also apply to .cache if [[ -d "${NESTRI_HOME}/.cache" ]]; then if ! $ENTCMD_PREFIX chown "${NESTRI_USER}:${NESTRI_USER}" "${NESTRI_HOME}/.cache" 2>/dev/null; then echo "Error: Failed to change ownership of ${NESTRI_HOME}/.cache to ${NESTRI_USER}:${NESTRI_USER}" >&2 @@ -324,9 +324,23 @@ main() { log "Skipping CAP_SYS_NICE for gamescope, capability not available" fi - # Handle user directory permissions - log "Ensuring user directory permissions..." - chown_user_directory || exit 1 + # Make sure /tmp/.X11-unix exists.. + if [[ ! -d "/tmp/.X11-unix" ]]; then + log "Creating /tmp/.X11-unix directory.." + $ENTCMD_PREFIX mkdir -p /tmp/.X11-unix || { + log "Error: Failed to create /tmp/.X11-unix directory" + exit 1 + } + # Set required perms.. + $ENTCMD_PREFIX chmod 1777 /tmp/.X11-unix || { + log "Error: Failed to chmod /tmp/.X11-unix to 1777" + exit 1 + } + fi + + # Handle user permissions + log "Ensuring user permissions..." + handle_user_permissions || exit 1 # Setup namespaceless env if needed for container runtime if [[ "$container_runtime" != "podman" ]]; then @@ -336,7 +350,7 @@ main() { # Make sure /run/udev/ directory exists with /run/udev/control, needed for virtual controller support if [[ ! -d "/run/udev" || ! -e "/run/udev/control" ]]; then - log "Creating /run/udev directory and control file..." + log "Creating /run/udev directory and control file.." $ENTCMD_PREFIX mkdir -p /run/udev || { log "Error: Failed to create /run/udev directory" exit 1 diff --git a/packages/scripts/entrypoint_nestri.sh b/packages/scripts/entrypoint_nestri.sh index fb5e127c..abbb95fd 100644 --- a/packages/scripts/entrypoint_nestri.sh +++ b/packages/scripts/entrypoint_nestri.sh @@ -187,7 +187,7 @@ start_compositor() { if [[ -n "${NESTRI_LAUNCH_CMD}" ]]; then log "Starting application: $NESTRI_LAUNCH_CMD" - WAYLAND_DISPLAY=wayland-0 /bin/bash -c "$NESTRI_LAUNCH_CMD" & + WAYLAND_DISPLAY="$COMPOSITOR_SOCKET" /bin/bash -c "$NESTRI_LAUNCH_CMD" & APP_PID=$! fi else diff --git a/packages/server/Cargo.lock b/packages/server/Cargo.lock index d8aec525..ef4f4763 100644 --- a/packages/server/Cargo.lock +++ b/packages/server/Cargo.lock @@ -181,7 +181,7 @@ checksum = "965c2d33e53cb6b267e148a4cb0760bc01f4904c1cd4bb4002a085bb016d1490" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.108", "synstructure", ] @@ -193,7 +193,7 @@ checksum = "3109e49b1e4909e9db6515a30c633684d68cdeaa252f215214cb4fa1a5bfee2c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.108", "synstructure", ] @@ -205,7 +205,7 @@ checksum = "7b18050c2cd6fe86c3a76584ef5e0baf286d038cda203eb6223df2cc413565f7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.108", ] [[package]] @@ -246,7 +246,7 @@ checksum = "3b43422f69d8ff38f95f1b2bb76517c91589a924d1559a0e935d7c8ce0274c11" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.108", ] [[package]] @@ -257,7 +257,7 @@ checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.108", ] [[package]] @@ -395,7 +395,7 @@ version = "0.72.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "993776b509cfb49c750f11b8f07a46fa23e0a1386ffc01fb1e7d343efc387895" dependencies = [ - "bitflags 2.9.4", + "bitflags 2.10.0", "cexpr", "clang-sys", "itertools 0.13.0", @@ -406,7 +406,7 @@ dependencies = [ "regex", "rustc-hash", "shlex", - "syn 2.0.106", + "syn 2.0.108", ] [[package]] @@ -417,9 +417,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" -version = "2.9.4" +version = "2.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2261d10cca569e4643e526d8dc2e62e433cc8aba21ab764233731f8d369bf394" +checksum = "812e12b5285cc515a9c72a5c1d3b6d46a19dac5acfef5265968c166106e31dd3" [[package]] name = "blake2" @@ -603,9 +603,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.5.49" +version = "4.5.50" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f4512b90fa68d3a9932cea5184017c5d200f5921df706d45e853537dea51508f" +checksum = "0c2cfd7bf8a6017ddaa4e32ffe7403d547790db06bd171c1c53926faab501623" dependencies = [ "clap_builder", "clap_derive", @@ -613,9 +613,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.49" +version = "4.5.50" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0025e98baa12e766c67ba13ff4695a887a1eba19569aad00a472546795bd6730" +checksum = "0a4c05b9e80c5ccd3a7ef080ad7b6ba7d6fc00a985b8b157197075677c82c7a0" dependencies = [ "anstream", "anstyle", @@ -632,7 +632,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.108", ] [[package]] @@ -839,7 +839,7 @@ checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.108", ] [[package]] @@ -879,7 +879,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8d162beedaa69905488a8da94f5ac3edb4dd4788b732fadb7bd120b2625c1976" dependencies = [ "data-encoding", - "syn 2.0.106", + "syn 2.0.108", ] [[package]] @@ -956,7 +956,7 @@ checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.108", ] [[package]] @@ -1090,7 +1090,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.108", ] [[package]] @@ -1262,7 +1262,7 @@ checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.108", ] [[package]] @@ -1396,7 +1396,7 @@ version = "0.21.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e1f2cbc4577536c849335878552f42086bfd25a8dcd6f54a18655cf818b20c8f" dependencies = [ - "bitflags 2.9.4", + "bitflags 2.10.0", "futures-channel", "futures-core", "futures-executor", @@ -1421,7 +1421,7 @@ dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.108", ] [[package]] @@ -2299,9 +2299,9 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.11.4" +version = "2.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b0f83760fb341a774ed326568e19f5a863af4a952def8c39f9ab92fd95b88e5" +checksum = "6717a8d2a5a929a1a2eb43a12812498ed141a0bcfb7e8f7844fbdbe4303bba9f" dependencies = [ "equivalent", "hashbrown 0.16.0", @@ -2368,9 +2368,9 @@ dependencies = [ [[package]] name = "is_terminal_polyfill" -version = "1.70.1" +version = "1.70.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf" +checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" [[package]] name = "itertools" @@ -2828,7 +2828,7 @@ checksum = "dd297cf53f0cb3dee4d2620bb319ae47ef27c702684309f682bdb7e55a18ae9c" dependencies = [ "heck", "quote", - "syn 2.0.106", + "syn 2.0.108", ] [[package]] @@ -3239,7 +3239,7 @@ version = "0.30.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "74523f3a35e05aba87a1d978330aef40f67b0304ac79c1c00b294c9830543db6" dependencies = [ - "bitflags 2.9.4", + "bitflags 2.10.0", "cfg-if", "cfg_aliases", "libc", @@ -3355,9 +3355,9 @@ dependencies = [ [[package]] name = "once_cell_polyfill" -version = "1.70.1" +version = "1.70.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4895175b425cb1f87721b59f0f286c2092bd4af812243672510e1ac53e2e0ad" +checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" [[package]] name = "opaque-debug" @@ -3499,7 +3499,7 @@ checksum = "6e918e4ff8c4549eb882f14b3a4bc8c8bc93de829416eacf579f1207a8fbf861" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.108", ] [[package]] @@ -3604,7 +3604,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" dependencies = [ "proc-macro2", - "syn 2.0.106", + "syn 2.0.108", ] [[package]] @@ -3627,9 +3627,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.101" +version = "1.0.102" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89ae43fd86e4158d6db51ad8e2b80f313af9cc74f5c0e03ccb87de09998732de" +checksum = "8e0f6df8eaa422d97d72edcd152e1451618fed47fabbdbd5a8864167b1d4aff7" dependencies = [ "unicode-ident", ] @@ -3654,7 +3654,7 @@ checksum = "440f724eba9f6996b75d63681b0a92b06947f1457076d503a4d2e2c8f56442b8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.108", ] [[package]] @@ -3677,7 +3677,7 @@ dependencies = [ "itertools 0.14.0", "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.108", ] [[package]] @@ -3861,7 +3861,7 @@ version = "0.5.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" dependencies = [ - "bitflags 2.9.4", + "bitflags 2.10.0", ] [[package]] @@ -4038,7 +4038,7 @@ version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cd15f8a2c5551a84d56efdc1cd049089e409ac19a3072d5037a17fd70719ff3e" dependencies = [ - "bitflags 2.9.4", + "bitflags 2.10.0", "errno", "libc", "linux-raw-sys", @@ -4047,9 +4047,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.33" +version = "0.23.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "751e04a496ca00bb97a5e043158d23d66b5aabf2e1d5aa2a0aaebb1aafe6f82c" +checksum = "6a9586e9ee2b4f8fab52a0048ca7334d7024eef48e2cb9407e3497bb7cab7fa7" dependencies = [ "aws-lc-rs", "log", @@ -4180,7 +4180,7 @@ version = "3.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b3297343eaf830f66ede390ea39da1d462b6b0c1b000f420d0a83f898bbbe6ef" dependencies = [ - "bitflags 2.9.4", + "bitflags 2.10.0", "core-foundation 0.10.1", "core-foundation-sys", "libc", @@ -4240,7 +4240,7 @@ checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.108", ] [[package]] @@ -4481,9 +4481,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.106" +version = "2.0.108" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ede7c438028d4436d71104916910f5bb611972c5cfd7f89b8300a8186e6fada6" +checksum = "da58917d35242480a05c2897064da0a80589a2a0476c9a3f2fdc83b53502e917" dependencies = [ "proc-macro2", "quote", @@ -4507,7 +4507,7 @@ checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.108", ] [[package]] @@ -4516,7 +4516,7 @@ version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3c879d448e9d986b661742763247d3693ed13609438cf3d006f51f5368a5ba6b" dependencies = [ - "bitflags 2.9.4", + "bitflags 2.10.0", "core-foundation 0.9.4", "system-configuration-sys", ] @@ -4574,7 +4574,7 @@ checksum = "451b374529930d7601b1eef8d32bc79ae870b6079b069401709c2a8bf9e75f36" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.108", ] [[package]] @@ -4603,7 +4603,7 @@ checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.108", ] [[package]] @@ -4614,7 +4614,7 @@ checksum = "3ff15c8ecd7de3849db632e14d18d2571fa09dfc5ed93479bc4485c7a517c913" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.108", ] [[package]] @@ -4707,7 +4707,7 @@ checksum = "af407857209536a95c8e56f8231ef2c2e2aff839b22e07a1ffcbc617e9db9fa5" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.108", ] [[package]] @@ -4817,7 +4817,7 @@ version = "0.6.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "adc82fd73de2a9722ac5da747f12383d2bfdb93591ee6c58486e0097890f05f2" dependencies = [ - "bitflags 2.9.4", + "bitflags 2.10.0", "bytes", "futures-util", "http", @@ -4861,7 +4861,7 @@ checksum = "81383ab64e72a7a8b8e13130c49e3dab29def6d0c7d76a03087b3cf71c5c6903" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.108", ] [[package]] @@ -4986,9 +4986,9 @@ checksum = "75b844d17643ee918803943289730bec8aac480150456169e647ed0b576ba539" [[package]] name = "unicode-ident" -version = "1.0.19" +version = "1.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f63a545481291138910575129486daeaf8ac54aee4387fe7906919f7830c7d9d" +checksum = "462eeb75aeb73aea900253ce739c8e18a67423fadf006037cd3ff27e82748a06" [[package]] name = "universal-hash" @@ -5079,9 +5079,9 @@ checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" [[package]] name = "vimputti" -version = "0.1.3" +version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4a5839a89185ccec572f746ccc02e37702cc6c0b62a6aa0d9bcd6e5921edba12" +checksum = "ffb370ee43e3ee4ca5329886e64dc5b27c83dc8cced5a63c2418777dac9a41a8" dependencies = [ "anyhow", "libc", @@ -5177,7 +5177,7 @@ dependencies = [ "log", "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.108", "wasm-bindgen-shared", ] @@ -5212,7 +5212,7 @@ checksum = "9f07d2f20d4da7b26400c9f4a0511e6e0345b040694e8a75bd41d578fa4421d7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.108", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -5502,7 +5502,7 @@ checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.108", ] [[package]] @@ -5513,7 +5513,7 @@ checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.108", ] [[package]] @@ -5959,7 +5959,7 @@ checksum = "38da3c9736e16c5d3c8c597a9aaa5d1fa565d0532ae05e27c24aa62fb32c0ab6" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.108", "synstructure", ] @@ -5980,7 +5980,7 @@ checksum = "88d2b8d9c68ad2b9e4340d7832716a4d21a22a1154777ad56ea55c51a9cf3831" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.108", ] [[package]] @@ -6000,7 +6000,7 @@ checksum = "d71e5d6e06ab090c67b5e44993ec16b72dcbaabc526db883a360057678b48502" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.108", "synstructure", ] @@ -6021,7 +6021,7 @@ checksum = "ce36e65b0d2999d2aafac989fb249189a141aee1f53c612c1f37d72631959f69" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.108", ] [[package]] @@ -6054,5 +6054,5 @@ checksum = "5b96237efa0c878c64bd89c436f661be4e46b2f3eff1ebb976f7ef2321d2f58f" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.108", ] diff --git a/packages/server/Cargo.toml b/packages/server/Cargo.toml index ce6d11c6..9da600f4 100644 --- a/packages/server/Cargo.toml +++ b/packages/server/Cargo.toml @@ -22,7 +22,7 @@ rand = "0.9" rustls = { version = "0.23", features = ["ring"] } tracing = "0.1" tracing-subscriber = { version = "0.3", features = ["env-filter"] } -vimputti = "0.1.3" +vimputti = "0.1.4" chrono = "0.4" prost = "0.14" prost-types = "0.14" diff --git a/packages/server/src/args.rs b/packages/server/src/args.rs index bbb5c3fa..5432c96c 100644 --- a/packages/server/src/args.rs +++ b/packages/server/src/args.rs @@ -211,6 +211,14 @@ impl Args { .value_parser(value_parser!(u32).range(1..)) .default_value("192"), ) + .arg( + Arg::new("software-render") + .long("software-render") + .env("SOFTWARE_RENDER") + .help("Use software rendering for wayland") + .value_parser(BoolishValueParser::new()) + .default_value("false"), + ) .arg( Arg::new("zero-copy") .long("zero-copy") diff --git a/packages/server/src/args/app_args.rs b/packages/server/src/args/app_args.rs index 44fa0633..d313b8c9 100644 --- a/packages/server/src/args/app_args.rs +++ b/packages/server/src/args/app_args.rs @@ -15,6 +15,9 @@ pub struct AppArgs { /// vimputti socket path pub vimputti_path: Option, + /// Use software rendering for wayland display + pub software_render: bool, + /// Experimental zero-copy pipeline support /// TODO: Move to video encoding flags pub zero_copy: bool, @@ -51,6 +54,10 @@ impl AppArgs { vimputti_path: matches .get_one::("vimputti-path") .map(|s| s.clone()), + software_render: matches + .get_one::("software-render") + .unwrap_or(&false) + .clone(), zero_copy: matches .get_one::("zero-copy") .unwrap_or(&false) @@ -73,6 +80,7 @@ impl AppArgs { "> vimputti_path: '{}'", self.vimputti_path.as_ref().map_or("None", |s| s.as_str()) ); + tracing::info!("> software_render: {}", self.software_render); tracing::info!("> zero_copy: {}", self.zero_copy); } } diff --git a/packages/server/src/enc_helper.rs b/packages/server/src/enc_helper.rs index a97aca32..1888a265 100644 --- a/packages/server/src/enc_helper.rs +++ b/packages/server/src/enc_helper.rs @@ -585,7 +585,6 @@ pub fn get_best_working_encoder( encoders: &Vec, codec: &Codec, encoder_type: &EncoderType, - zero_copy: bool, ) -> Result> { let mut candidates = get_encoders_by_videocodec( encoders, @@ -601,7 +600,7 @@ pub fn get_best_working_encoder( while !candidates.is_empty() { let best = get_best_compatible_encoder(&candidates, codec, encoder_type)?; tracing::info!("Testing encoder: {}", best.name,); - if test_encoder(&best, zero_copy).is_ok() { + if test_encoder(&best).is_ok() { return Ok(best); } else { // Remove this encoder and try next best @@ -613,25 +612,10 @@ pub fn get_best_working_encoder( } /// Test if a pipeline with the given encoder can be created and set to Playing -pub fn test_encoder(encoder: &VideoEncoderInfo, zero_copy: bool) -> Result<(), Box> { - let src = gstreamer::ElementFactory::make("waylanddisplaysrc").build()?; - if let Some(gpu_info) = &encoder.gpu_info { - src.set_property_from_str("render-node", gpu_info.render_path()); - } +pub fn test_encoder(encoder: &VideoEncoderInfo) -> Result<(), Box> { + let src = gstreamer::ElementFactory::make("videotestsrc").build()?; let caps_filter = gstreamer::ElementFactory::make("capsfilter").build()?; - let caps = gstreamer::Caps::from_str(&format!( - "{},width=1280,height=720,framerate=30/1{}", - if zero_copy { - if encoder.encoder_api == EncoderAPI::NVENC { - "video/x-raw(memory:CUDAMemory)" - } else { - "video/x-raw(memory:DMABuf)" - } - } else { - "video/x-raw" - }, - if zero_copy { "" } else { ",format=RGBx" } - ))?; + let caps = gstreamer::Caps::from_str("video/x-raw,width=1280,height=720,framerate=30/1")?; caps_filter.set_property("caps", &caps); let enc = gstreamer::ElementFactory::make(&encoder.name).build()?; @@ -642,41 +626,9 @@ pub fn test_encoder(encoder: &VideoEncoderInfo, zero_copy: bool) -> Result<(), B // Create pipeline and link elements let pipeline = gstreamer::Pipeline::new(); - if zero_copy { - if encoder.encoder_api == EncoderAPI::NVENC { - // NVENC zero-copy path - pipeline.add_many(&[&src, &caps_filter, &enc, &sink])?; - gstreamer::Element::link_many(&[&src, &caps_filter, &enc, &sink])?; - } else { - // VA-API/QSV zero-copy path - let vapostproc = gstreamer::ElementFactory::make("vapostproc").build()?; - let va_caps_filter = gstreamer::ElementFactory::make("capsfilter").build()?; - let va_caps = gstreamer::Caps::from_str("video/x-raw(memory:VAMemory),format=NV12")?; - va_caps_filter.set_property("caps", &va_caps); - - pipeline.add_many(&[ - &src, - &caps_filter, - &vapostproc, - &va_caps_filter, - &enc, - &sink, - ])?; - gstreamer::Element::link_many(&[ - &src, - &caps_filter, - &vapostproc, - &va_caps_filter, - &enc, - &sink, - ])?; - } - } else { - // Non-zero-copy path for all encoders - needs videoconvert - let videoconvert = gstreamer::ElementFactory::make("videoconvert").build()?; - pipeline.add_many(&[&src, &caps_filter, &videoconvert, &enc, &sink])?; - gstreamer::Element::link_many(&[&src, &caps_filter, &videoconvert, &enc, &sink])?; - } + let videoconvert = gstreamer::ElementFactory::make("videoconvert").build()?; + pipeline.add_many(&[&src, &caps_filter, &videoconvert, &enc, &sink])?; + gstreamer::Element::link_many(&[&src, &caps_filter, &videoconvert, &enc, &sink])?; let bus = pipeline.bus().ok_or("Pipeline has no bus")?; pipeline.set_state(gstreamer::State::Playing)?; diff --git a/packages/server/src/input/controller.rs b/packages/server/src/input/controller.rs index 656c6d46..d34e8600 100644 --- a/packages/server/src/input/controller.rs +++ b/packages/server/src/input/controller.rs @@ -47,7 +47,7 @@ impl ControllerInput { pub struct ControllerManager { vimputti_client: Arc, cmd_tx: mpsc::Sender, - rumble_tx: mpsc::Sender<(u32, u16, u16, u16)>, // (slot, strong, weak, duration_ms) + rumble_tx: mpsc::Sender<(u32, u16, u16, u16, String)>, // (slot, strong, weak, duration_ms, session_id) attach_tx: mpsc::Sender, } impl ControllerManager { @@ -55,7 +55,7 @@ impl ControllerManager { vimputti_client: Arc, ) -> Result<( Self, - mpsc::Receiver<(u32, u16, u16, u16)>, + mpsc::Receiver<(u32, u16, u16, u16, String)>, mpsc::Receiver, )> { let (cmd_tx, cmd_rx) = mpsc::channel(512); @@ -88,12 +88,12 @@ impl ControllerManager { struct ControllerSlot { controller: ControllerInput, session_id: String, - last_activity: std::time::Instant, + session_slot: u32, } -// Returns first free controller slot from 0-7 +// Returns first free controller slot from 0-16 fn get_free_slot(controllers: &HashMap) -> Option { - for slot in 0..8 { + for slot in 0..17 { if !controllers.contains_key(&slot) { return Some(slot); } @@ -104,7 +104,7 @@ fn get_free_slot(controllers: &HashMap) -> Option { async fn command_loop( mut cmd_rx: mpsc::Receiver, vimputti_client: Arc, - rumble_tx: mpsc::Sender<(u32, u16, u16, u16)>, + rumble_tx: mpsc::Sender<(u32, u16, u16, u16, String)>, attach_tx: mpsc::Sender, ) { let mut controllers: HashMap = HashMap::new(); @@ -112,13 +112,15 @@ async fn command_loop( match payload { Payload::ControllerAttach(data) => { let session_id = data.session_id.clone(); + let session_slot = data.session_slot.clone(); // Check if this session already has a slot (reconnection) let existing_slot = controllers .iter() - .find(|(_, slot)| slot.session_id == session_id && !session_id.is_empty()) + .find(|(_, slot)| { + slot.session_id == session_id && slot.session_slot == session_slot as u32 + }) .map(|(slot_num, _)| *slot_num); - let slot = existing_slot.or_else(|| get_free_slot(&controllers)); if let Some(slot) = slot { @@ -131,7 +133,7 @@ async fn command_loop( controller .device_mut() .on_rumble(move |strong, weak, duration_ms| { - let _ = rumble_tx.try_send((slot, strong, weak, duration_ms)); + let _ = rumble_tx.try_send((slot, strong, weak, duration_ms, data.session_id.clone())); }) .await .map_err(|e| { @@ -146,7 +148,7 @@ async fn command_loop( // Return to attach_tx what slot was assigned let attach_info = ProtoControllerAttach { id: data.id.clone(), - slot: slot as i32, + session_slot: slot as i32, session_id: session_id.clone(), }; @@ -157,7 +159,7 @@ async fn command_loop( ControllerSlot { controller, session_id: session_id.clone(), - last_activity: std::time::Instant::now(), + session_slot: session_slot.clone() as u32, }, ); tracing::info!( @@ -185,25 +187,25 @@ async fn command_loop( } } Payload::ControllerDetach(data) => { - if controllers.remove(&(data.slot as u32)).is_some() { - tracing::info!("Controller detached from slot {}", data.slot); + if controllers.remove(&(data.session_slot as u32)).is_some() { + tracing::info!("Controller detached from slot {}", data.session_slot); } else { - tracing::warn!("No controller found in slot {} to detach", data.slot); + tracing::warn!("No controller found in slot {} to detach", data.session_slot); } } Payload::ControllerButton(data) => { - if let Some(controller) = controllers.get(&(data.slot as u32)) { + if let Some(controller) = controllers.get(&(data.session_slot as u32)) { if let Some(button) = vimputti::Button::from_ev_code(data.button as u16) { let device = controller.controller.device(); device.button(button, data.pressed); device.sync(); } } else { - tracing::warn!("Controller slot {} not found for button event", data.slot); + tracing::warn!("Controller slot {} not found for button event", data.session_slot); } } Payload::ControllerStick(data) => { - if let Some(controller) = controllers.get(&(data.slot as u32)) { + if let Some(controller) = controllers.get(&(data.session_slot as u32)) { let device = controller.controller.device(); if data.stick == 0 { // Left stick @@ -218,11 +220,11 @@ async fn command_loop( } device.sync(); } else { - tracing::warn!("Controller slot {} not found for stick event", data.slot); + tracing::warn!("Controller slot {} not found for stick event", data.session_slot); } } Payload::ControllerTrigger(data) => { - if let Some(controller) = controllers.get(&(data.slot as u32)) { + if let Some(controller) = controllers.get(&(data.session_slot as u32)) { let device = controller.controller.device(); if data.trigger == 0 { // Left trigger @@ -233,11 +235,11 @@ async fn command_loop( } device.sync(); } else { - tracing::warn!("Controller slot {} not found for trigger event", data.slot); + tracing::warn!("Controller slot {} not found for trigger event", data.session_slot); } } Payload::ControllerAxis(data) => { - if let Some(controller) = controllers.get(&(data.slot as u32)) { + if let Some(controller) = controllers.get(&(data.session_slot as u32)) { let device = controller.controller.device(); if data.axis == 0 { // dpad x diff --git a/packages/server/src/main.rs b/packages/server/src/main.rs index 3427b71c..e4a211c5 100644 --- a/packages/server/src/main.rs +++ b/packages/server/src/main.rs @@ -24,7 +24,7 @@ use tracing_subscriber::EnvFilter; use tracing_subscriber::filter::LevelFilter; // Handles gathering GPU information and selecting the most suitable GPU -fn handle_gpus(args: &args::Args) -> Result, Box> { +fn handle_gpus(args: &args::Args) -> Result, Box> { tracing::info!("Gathering GPU information.."); let mut gpus = gpu::get_gpus()?; if gpus.is_empty() { @@ -119,7 +119,6 @@ fn handle_encoder_video( &video_encoders, &args.encoding.video.codec, &args.encoding.video.encoder_type, - args.app.zero_copy, )?; } tracing::info!("Selected video encoder: '{}'", video_encoder.name); @@ -323,7 +322,9 @@ async fn main() -> Result<(), Box> { /* Video */ // Video Source Element let video_source = Arc::new(gstreamer::ElementFactory::make("waylanddisplaysrc").build()?); - if let Some(gpu_info) = &video_encoder_info.gpu_info { + if args.app.software_render { + video_source.set_property_from_str("render-node", "software"); + } else if let Some(gpu_info) = &video_encoder_info.gpu_info { video_source.set_property_from_str("render-node", gpu_info.render_path()); } @@ -428,20 +429,16 @@ async fn main() -> Result<(), Box> { webrtcsink.set_property("do-retransmission", false); /* Queues */ - let video_source_queue = gstreamer::ElementFactory::make("queue") - .property("max-size-buffers", 5u32) - .build()?; - - let audio_source_queue = gstreamer::ElementFactory::make("queue") - .property("max-size-buffers", 5u32) - .build()?; - let video_queue = gstreamer::ElementFactory::make("queue") - .property("max-size-buffers", 5u32) + .property("max-size-buffers", 2u32) + .property("max-size-time", 0u64) + .property("max-size-bytes", 0u32) .build()?; let audio_queue = gstreamer::ElementFactory::make("queue") - .property("max-size-buffers", 5u32) + .property("max-size-buffers", 2u32) + .property("max-size-time", 0u64) + .property("max-size-bytes", 0u32) .build()?; /* Clock Sync */ @@ -460,7 +457,6 @@ async fn main() -> Result<(), Box> { &caps_filter, &video_queue, &video_clocksync, - &video_source_queue, &video_source, &audio_encoder, &audio_capsfilter, @@ -468,7 +464,6 @@ async fn main() -> Result<(), Box> { &audio_clocksync, &audio_rate, &audio_converter, - &audio_source_queue, &audio_source, ])?; @@ -495,7 +490,6 @@ async fn main() -> Result<(), Box> { // Link main audio branch gstreamer::Element::link_many(&[ &audio_source, - &audio_source_queue, &audio_converter, &audio_rate, &audio_capsfilter, @@ -517,7 +511,6 @@ async fn main() -> Result<(), Box> { if let (Some(vapostproc), Some(va_caps_filter)) = (&vapostproc, &va_caps_filter) { gstreamer::Element::link_many(&[ &video_source, - &video_source_queue, &caps_filter, &video_queue, &video_clocksync, @@ -529,7 +522,6 @@ async fn main() -> Result<(), Box> { // NVENC pipeline gstreamer::Element::link_many(&[ &video_source, - &video_source_queue, &caps_filter, &video_encoder, ])?; @@ -537,7 +529,6 @@ async fn main() -> Result<(), Box> { } else { gstreamer::Element::link_many(&[ &video_source, - &video_source_queue, &caps_filter, &video_queue, &video_clocksync, diff --git a/packages/server/src/nestrisink/imp.rs b/packages/server/src/nestrisink/imp.rs index f0da654c..88937411 100644 --- a/packages/server/src/nestrisink/imp.rs +++ b/packages/server/src/nestrisink/imp.rs @@ -23,7 +23,7 @@ pub struct Signaller { wayland_src: PLRwLock>>, data_channel: PLRwLock>>, controller_manager: PLRwLock>>, - rumble_rx: Mutex>>, + rumble_rx: Mutex>>, attach_rx: Mutex>>, } impl Default for Signaller { @@ -70,11 +70,11 @@ impl Signaller { self.controller_manager.read().clone() } - pub async fn set_rumble_rx(&self, rumble_rx: mpsc::Receiver<(u32, u16, u16, u16)>) { + pub async fn set_rumble_rx(&self, rumble_rx: mpsc::Receiver<(u32, u16, u16, u16, String)>) { *self.rumble_rx.lock().await = Some(rumble_rx); } - pub async fn take_rumble_rx(&self) -> Option> { + pub async fn take_rumble_rx(&self) -> Option> { self.rumble_rx.lock().await.take() } @@ -382,7 +382,7 @@ impl ObjectImpl for Signaller { fn setup_data_channel( controller_manager: Option>, - rumble_rx: Option>, // (slot, strong, weak, duration_ms) + rumble_rx: Option>, // (slot, strong, weak, duration_ms, session_id) attach_rx: Option>, data_channel: Arc, wayland_src: &gstreamer::Element, @@ -423,10 +423,11 @@ fn setup_data_channel( if let Some(mut rumble_rx) = rumble_rx { let data_channel_clone = data_channel.clone(); tokio::spawn(async move { - while let Some((slot, strong, weak, duration_ms)) = rumble_rx.recv().await { + while let Some((slot, strong, weak, duration_ms, session_id)) = rumble_rx.recv().await { let rumble_msg = crate::proto::create_message( Payload::ControllerRumble(ProtoControllerRumble { - slot: slot as i32, + session_slot: slot as i32, + session_id: session_id, low_frequency: weak as i32, high_frequency: strong as i32, duration: duration_ms as i32, diff --git a/packages/server/src/nestrisink/mod.rs b/packages/server/src/nestrisink/mod.rs index 4de29964..a37b03d1 100644 --- a/packages/server/src/nestrisink/mod.rs +++ b/packages/server/src/nestrisink/mod.rs @@ -18,7 +18,7 @@ impl NestriSignaller { nestri_conn: NestriConnection, wayland_src: Arc, controller_manager: Option>, - rumble_rx: Option>, + rumble_rx: Option>, attach_rx: Option>, ) -> Result> { let obj: Self = glib::Object::new(); diff --git a/packages/server/src/proto/proto.rs b/packages/server/src/proto/proto.rs index a9ef7a42..635b8f3e 100644 --- a/packages/server/src/proto/proto.rs +++ b/packages/server/src/proto/proto.rs @@ -84,95 +84,113 @@ pub struct ProtoControllerAttach { /// One of the following enums: "ps", "xbox" or "switch" #[prost(string, tag="1")] pub id: ::prost::alloc::string::String, - /// Slot number (0-3) + /// Session specific slot number (0-3) #[prost(int32, tag="2")] - pub slot: i32, - /// Session ID of the client attaching the controller + pub session_slot: i32, + /// Session ID of the client #[prost(string, tag="3")] pub session_id: ::prost::alloc::string::String, } /// ControllerDetach message #[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, Copy, PartialEq, ::prost::Message)] +#[derive(Clone, PartialEq, ::prost::Message)] pub struct ProtoControllerDetach { - /// Slot number (0-3) + /// Session specific slot number (0-3) #[prost(int32, tag="1")] - pub slot: i32, + pub session_slot: i32, + /// Session ID of the client + #[prost(string, tag="2")] + pub session_id: ::prost::alloc::string::String, } /// ControllerButton message #[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, Copy, PartialEq, ::prost::Message)] +#[derive(Clone, PartialEq, ::prost::Message)] pub struct ProtoControllerButton { - /// Slot number (0-3) + /// Session specific slot number (0-3) #[prost(int32, tag="1")] - pub slot: i32, + pub session_slot: i32, + /// Session ID of the client + #[prost(string, tag="2")] + pub session_id: ::prost::alloc::string::String, /// Button code (linux input event code) - #[prost(int32, tag="2")] + #[prost(int32, tag="3")] pub button: i32, /// true if pressed, false if released - #[prost(bool, tag="3")] + #[prost(bool, tag="4")] pub pressed: bool, } /// ControllerTriggers message #[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, Copy, PartialEq, ::prost::Message)] +#[derive(Clone, PartialEq, ::prost::Message)] pub struct ProtoControllerTrigger { - /// Slot number (0-3) + /// Session specific slot number (0-3) #[prost(int32, tag="1")] - pub slot: i32, + pub session_slot: i32, + /// Session ID of the client + #[prost(string, tag="2")] + pub session_id: ::prost::alloc::string::String, /// Trigger number (0 for left, 1 for right) - #[prost(int32, tag="2")] + #[prost(int32, tag="3")] pub trigger: i32, /// trigger value (-32768 to 32767) - #[prost(int32, tag="3")] + #[prost(int32, tag="4")] pub value: i32, } /// ControllerSticks message #[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, Copy, PartialEq, ::prost::Message)] +#[derive(Clone, PartialEq, ::prost::Message)] pub struct ProtoControllerStick { - /// Slot number (0-3) + /// Session specific slot number (0-3) #[prost(int32, tag="1")] - pub slot: i32, + pub session_slot: i32, + /// Session ID of the client + #[prost(string, tag="2")] + pub session_id: ::prost::alloc::string::String, /// Stick number (0 for left, 1 for right) - #[prost(int32, tag="2")] + #[prost(int32, tag="3")] pub stick: i32, /// X axis value (-32768 to 32767) - #[prost(int32, tag="3")] + #[prost(int32, tag="4")] pub x: i32, /// Y axis value (-32768 to 32767) - #[prost(int32, tag="4")] + #[prost(int32, tag="5")] pub y: i32, } /// ControllerAxis message #[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, Copy, PartialEq, ::prost::Message)] +#[derive(Clone, PartialEq, ::prost::Message)] pub struct ProtoControllerAxis { - /// Slot number (0-3) + /// Session specific slot number (0-3) #[prost(int32, tag="1")] - pub slot: i32, + pub session_slot: i32, + /// Session ID of the client + #[prost(string, tag="2")] + pub session_id: ::prost::alloc::string::String, /// Axis number (0 for d-pad horizontal, 1 for d-pad vertical) - #[prost(int32, tag="2")] + #[prost(int32, tag="3")] pub axis: i32, /// axis value (-1 to 1) - #[prost(int32, tag="3")] + #[prost(int32, tag="4")] pub value: i32, } /// ControllerRumble message #[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, Copy, PartialEq, ::prost::Message)] +#[derive(Clone, PartialEq, ::prost::Message)] pub struct ProtoControllerRumble { - /// Slot number (0-3) + /// Session specific slot number (0-3) #[prost(int32, tag="1")] - pub slot: i32, + pub session_slot: i32, + /// Session ID of the client + #[prost(string, tag="2")] + pub session_id: ::prost::alloc::string::String, /// Low frequency rumble (0-65535) - #[prost(int32, tag="2")] + #[prost(int32, tag="3")] pub low_frequency: i32, /// High frequency rumble (0-65535) - #[prost(int32, tag="3")] + #[prost(int32, tag="4")] pub high_frequency: i32, /// Duration in milliseconds - #[prost(int32, tag="4")] + #[prost(int32, tag="5")] pub duration: i32, } // WebRTC + signaling diff --git a/protobufs/types.proto b/protobufs/types.proto index 037b05e9..1f951f16 100644 --- a/protobufs/types.proto +++ b/protobufs/types.proto @@ -51,50 +51,56 @@ message ProtoKeyUp { // ControllerAttach message message ProtoControllerAttach { string id = 1; // One of the following enums: "ps", "xbox" or "switch" - int32 slot = 2; // Slot number (0-3) - string session_id = 3; // Session ID of the client attaching the controller + int32 session_slot = 2; // Session specific slot number (0-3) + string session_id = 3; // Session ID of the client } // ControllerDetach message message ProtoControllerDetach { - int32 slot = 1; // Slot number (0-3) + int32 session_slot = 1; // Session specific slot number (0-3) + string session_id = 2; // Session ID of the client } // ControllerButton message message ProtoControllerButton { - int32 slot = 1; // Slot number (0-3) - int32 button = 2; // Button code (linux input event code) - bool pressed = 3; // true if pressed, false if released + int32 session_slot = 1; // Session specific slot number (0-3) + string session_id = 2; // Session ID of the client + int32 button = 3; // Button code (linux input event code) + bool pressed = 4; // true if pressed, false if released } // ControllerTriggers message message ProtoControllerTrigger { - int32 slot = 1; // Slot number (0-3) - int32 trigger = 2; // Trigger number (0 for left, 1 for right) - int32 value = 3; // trigger value (-32768 to 32767) + int32 session_slot = 1; // Session specific slot number (0-3) + string session_id = 2; // Session ID of the client + int32 trigger = 3; // Trigger number (0 for left, 1 for right) + int32 value = 4; // trigger value (-32768 to 32767) } // ControllerSticks message message ProtoControllerStick { - int32 slot = 1; // Slot number (0-3) - int32 stick = 2; // Stick number (0 for left, 1 for right) - int32 x = 3; // X axis value (-32768 to 32767) - int32 y = 4; // Y axis value (-32768 to 32767) + int32 session_slot = 1; // Session specific slot number (0-3) + string session_id = 2; // Session ID of the client + int32 stick = 3; // Stick number (0 for left, 1 for right) + int32 x = 4; // X axis value (-32768 to 32767) + int32 y = 5; // Y axis value (-32768 to 32767) } // ControllerAxis message message ProtoControllerAxis { - int32 slot = 1; // Slot number (0-3) - int32 axis = 2; // Axis number (0 for d-pad horizontal, 1 for d-pad vertical) - int32 value = 3; // axis value (-1 to 1) + int32 session_slot = 1; // Session specific slot number (0-3) + string session_id = 2; // Session ID of the client + int32 axis = 3; // Axis number (0 for d-pad horizontal, 1 for d-pad vertical) + int32 value = 4; // axis value (-1 to 1) } // ControllerRumble message message ProtoControllerRumble { - int32 slot = 1; // Slot number (0-3) - int32 low_frequency = 2; // Low frequency rumble (0-65535) - int32 high_frequency = 3; // High frequency rumble (0-65535) - int32 duration = 4; // Duration in milliseconds + int32 session_slot = 1; // Session specific slot number (0-3) + string session_id = 2; // Session ID of the client + int32 low_frequency = 3; // Low frequency rumble (0-65535) + int32 high_frequency = 4; // High frequency rumble (0-65535) + int32 duration = 5; // Duration in milliseconds } /* WebRTC + signaling */ From 1d88a03b933fb29f706bad3fd931058eee5e0d9a Mon Sep 17 00:00:00 2001 From: DatCaptainHorse Date: Sat, 1 Nov 2025 00:53:15 +0200 Subject: [PATCH 3/6] More multi-controller fixes, better controller polling logic, clean up dead relay code --- containerfiles/runner-builder.Containerfile | 2 +- packages/input/src/controller.ts | 550 ++++++++++-------- packages/input/src/proto/messages_pb.ts | 36 +- packages/input/src/proto/types_pb.ts | 217 +++---- packages/play-standalone/package.json | 4 +- packages/relay/go.mod | 18 +- packages/relay/go.sum | 36 +- packages/relay/internal/core/core.go | 13 - .../relay/internal/core/protocol_stream.go | 78 +-- packages/relay/internal/core/state.go | 45 -- packages/relay/internal/proto/messages.pb.go | 138 ++--- packages/relay/internal/proto/types.pb.go | 546 ++++++++--------- packages/server/Cargo.lock | 4 +- packages/server/Cargo.toml | 2 +- packages/server/src/input/controller.rs | 251 +++++--- packages/server/src/proto/proto.rs | 160 ++--- protobufs/messages.proto | 9 +- protobufs/types.proto | 65 +-- 18 files changed, 1017 insertions(+), 1157 deletions(-) diff --git a/containerfiles/runner-builder.Containerfile b/containerfiles/runner-builder.Containerfile index e6613bb4..d18d94cd 100644 --- a/containerfiles/runner-builder.Containerfile +++ b/containerfiles/runner-builder.Containerfile @@ -41,7 +41,7 @@ RUN --mount=type=cache,target=/var/cache/pacman/pkg \ pacman -Sy --noconfirm lib32-gcc-libs # Clone repository -RUN git clone --depth 1 --rev "f2f21561ddcb814d74455311969d3e8934b052c6" https://github.com/DatCaptainHorse/vimputti.git +RUN git clone --depth 1 --rev "2fde5376b6b9a38cdbd94ccc6a80c9d29a81a417" https://github.com/DatCaptainHorse/vimputti.git #-------------------------------------------------------------------- FROM vimputti-manager-deps AS vimputti-manager-planner diff --git a/packages/input/src/controller.ts b/packages/input/src/controller.ts index 6742f6e5..ba71c69d 100644 --- a/packages/input/src/controller.ts +++ b/packages/input/src/controller.ts @@ -3,10 +3,8 @@ import { WebRTCStream } from "./webrtc-stream"; import { ProtoControllerAttachSchema, ProtoControllerDetachSchema, - ProtoControllerButtonSchema, - ProtoControllerTriggerSchema, - ProtoControllerAxisSchema, - ProtoControllerStickSchema, + ProtoControllerStateBatchSchema, + ProtoControllerStateBatch, ProtoControllerRumble, } from "./proto/types_pb"; import { create, toBinary, fromBinary } from "@bufbuild/protobuf"; @@ -19,6 +17,7 @@ interface Props { } interface GamepadState { + previousButtonState: Map; buttonState: Map; leftTrigger: number; rightTrigger: number; @@ -30,11 +29,17 @@ interface GamepadState { dpadY: number; } +enum PollState { + IDLE, + RUNNING, +} + export class Controller { protected wrtc: WebRTCStream; protected connected: boolean = false; protected gamepad: Gamepad | null = null; - protected lastState: GamepadState = { + protected state: GamepadState = { + previousButtonState: new Map(), buttonState: new Map(), leftTrigger: 0, rightTrigger: 0, @@ -48,22 +53,34 @@ export class Controller { // TODO: As user configurable, set quite low now for decent controllers (not Nintendo ones :P) protected stickDeadzone: number = 2048; // 2048 / 32768 = ~0.06 (6% of stick range) - private updateInterval = 10.0; // 100 updates per second - private isIdle: boolean = true; + // Polling configuration + private readonly FULL_RATE_MS = 10; // 100 UPS + private readonly IDLE_THRESHOLD = 100; // ms before considering idle/hands off controller + private readonly FULL_INTERVAL= 250; // ms before sending full state occassionally, to verify inputs are synced + + // Polling state + private pollingState: PollState = PollState.IDLE; private lastInputTime: number = Date.now(); - private idleUpdateInterval: number = 150.0; // ~6-7 updates per second for keep-alive packets - private inputDetected: boolean = false; - private lastFullStateSend: number = Date.now(); - private fullStateSendInterval: number = 500.0; // send full state every 0.5 seconds (helps packet loss) - private forceFullStateSend: boolean = false; + private lastFullTime: number = Date.now(); + private pollInterval: any = null; + + // Controller batch vars + private sequence: number = 0; + private readonly CHANGED_BUTTONS_STATE = 1 << 0; + private readonly CHANGED_LEFT_STICK_X = 1 << 1; + private readonly CHANGED_LEFT_STICK_Y = 1 << 2; + private readonly CHANGED_RIGHT_STICK_X = 1 << 3; + private readonly CHANGED_RIGHT_STICK_Y = 1 << 4; + private readonly CHANGED_LEFT_TRIGGER = 1 << 5; + private readonly CHANGED_RIGHT_TRIGGER = 1 << 6; + private readonly CHANGED_DPAD_X = 1 << 7; + private readonly CHANGED_DPAD_Y = 1 << 8; private _dcHandler: ((data: ArrayBuffer) => void) | null = null; constructor({ webrtc, e }: Props) { this.wrtc = webrtc; - this.updateInterval = 1000 / webrtc.currentFrameRate; - // Get vendor of gamepad from id string (i.e. "... Vendor: 054c Product: 09cc") const vendorMatch = e.gamepad.id.match(/Vendor:\s?([0-9a-fA-F]{4})/); const vendorId = vendorMatch ? vendorMatch[1].toLowerCase() : "unknown"; @@ -89,6 +106,7 @@ export class Controller { console.log( `Gamepad connected: ${e.gamepad.id}, local slot ${e.gamepad.index}, msg: ${attachMsg.sessionSlot}`, ); + this.run(); } } catch (err) { console.error("Error decoding datachannel message:", err); @@ -162,266 +180,283 @@ export class Controller { return ((value - fromMin) * (toMax - toMin)) / (fromMax - fromMin) + toMin; } + private restartPolling() { + // Clear existing interval + if (this.pollInterval) { + clearInterval(this.pollInterval); + this.pollInterval = null; + } + + // Restart with active polling + this.pollingState = PollState.RUNNING; + this.lastInputTime = Date.now(); + + // Start interval + this.pollInterval = setInterval( + () => this.pollGamepad(), + this.FULL_RATE_MS, + ); + } + private pollGamepad() { - // Get updated gamepad state + if (!this.connected || !this.gamepad) return; + const gamepads = navigator.getGamepads(); + if (!gamepads[this.gamepad.index]) return; + + this.gamepad = gamepads[this.gamepad.index]; + + // Collect state changes + const changedFields = this.collectStateChanges(); + + // Send batched changes update if there's changes + if (changedFields > 0) { + let send_type = 1; + const timeSinceFull = Date.now() - this.lastFullTime; + if (timeSinceFull > this.FULL_INTERVAL) { + send_type = 0; + this.lastFullTime = Date.now(); + } - // Periodically force send full state to clear stuck inputs - if (Date.now() - this.lastFullStateSend > this.fullStateSendInterval) { - this.forceFullStateSend = true; - this.lastFullStateSend = Date.now(); + this.sendBatchedState(changedFields, send_type); + this.lastInputTime = Date.now(); + if (this.pollingState !== PollState.RUNNING) { + this.pollingState = PollState.RUNNING; + } } - if (this.gamepad) { - if (gamepads[this.gamepad.index]) { - this.gamepad = gamepads[this.gamepad!.index]; - /* Button handling */ - this.gamepad.buttons.forEach((button, index) => { - // Ignore d-pad buttons (12-15) as we handle those as axis - if (index >= 12 && index <= 15) return; - // ignore trigger buttons (6-7) as we handle those as axis - if (index === 6 || index === 7) return; - // If state differs, send - if (button.pressed !== this.lastState.buttonState.get(index) || this.forceFullStateSend) { - const linuxCode = this.controllerButtonToVirtualKeyCode(index); - if (linuxCode === undefined) { - // Skip unmapped button index - this.lastState.buttonState.set(index, button.pressed); - return; - } - - const buttonMessage = createMessage( - create(ProtoControllerButtonSchema, { - sessionSlot: this.gamepad.index, - sessionId: this.wrtc.getSessionID(), - button: linuxCode, - pressed: button.pressed, - }), - "controllerInput", - ); - this.wrtc.sendBinary(toBinary(ProtoMessageSchema, buttonMessage)); - this.inputDetected = true; - // Store button state - this.lastState.buttonState.set(index, button.pressed); - } - }); - - /* Trigger handling */ - // map trigger value from 0.0 to 1.0 to -32768 to 32767 - const leftTrigger = Math.round( - this.remapFromTo( - this.gamepad.buttons[6]?.value ?? 0, - 0, - 1, - -32768, - 32767, - ), - ); - // If state differs, send - if (leftTrigger !== this.lastState.leftTrigger || this.forceFullStateSend) { - const triggerMessage = createMessage( - create(ProtoControllerTriggerSchema, { - sessionSlot: this.gamepad.index, - sessionId: this.wrtc.getSessionID(), - trigger: 0, // 0 = left, 1 = right - value: leftTrigger, - }), - "controllerInput", - ); - this.wrtc.sendBinary(toBinary(ProtoMessageSchema, triggerMessage)); - this.inputDetected = true; - this.lastState.leftTrigger = leftTrigger; - } - const rightTrigger = Math.round( - this.remapFromTo( - this.gamepad.buttons[7]?.value ?? 0, - 0, - 1, - -32768, - 32767, - ), - ); - // If state differs, send - if (rightTrigger !== this.lastState.rightTrigger || this.forceFullStateSend) { - const triggerMessage = createMessage( - create(ProtoControllerTriggerSchema, { - sessionSlot: this.gamepad.index, - sessionId: this.wrtc.getSessionID(), - trigger: 1, // 0 = left, 1 = right - value: rightTrigger, - }), - "controllerInput", - ); - this.wrtc.sendBinary(toBinary(ProtoMessageSchema, triggerMessage)); - this.inputDetected = true; - this.lastState.rightTrigger = rightTrigger; - } + const timeSinceInput = Date.now() - this.lastInputTime; + if (timeSinceInput > this.IDLE_THRESHOLD) { + // Changing from running to idle.. + if (this.pollingState === PollState.RUNNING) { + // Send full state on idle assumption + this.sendBatchedState(0xFF, 0); + this.pollingState = PollState.IDLE; + } + } - /* DPad handling */ - // We send dpad buttons as axis values -1 to 1 for left/up, right/down - const dpadLeft = this.gamepad.buttons[14]?.pressed ? 1 : 0; - const dpadRight = this.gamepad.buttons[15]?.pressed ? 1 : 0; - const dpadX = dpadLeft ? -1 : dpadRight ? 1 : 0; - if (dpadX !== this.lastState.dpadX || this.forceFullStateSend) { - const dpadMessage = createMessage( - create(ProtoControllerAxisSchema, { - sessionSlot: this.gamepad.index, - sessionId: this.wrtc.getSessionID(), - axis: 0, // 0 = dpadX, 1 = dpadY - value: dpadX, - }), - "controllerInput", - ); - this.wrtc.sendBinary(toBinary(ProtoMessageSchema, dpadMessage)); - this.inputDetected = true; - this.lastState.dpadX = dpadX; - } + this.state.buttonState.forEach((b, i) => + this.state.previousButtonState.set(i, b), + ); + } - const dpadUp = this.gamepad.buttons[12]?.pressed ? 1 : 0; - const dpadDown = this.gamepad.buttons[13]?.pressed ? 1 : 0; - const dpadY = dpadUp ? -1 : dpadDown ? 1 : 0; - if (dpadY !== this.lastState.dpadY || this.forceFullStateSend) { - const dpadMessage = createMessage( - create(ProtoControllerAxisSchema, { - sessionSlot: this.gamepad.index, - sessionId: this.wrtc.getSessionID(), - axis: 1, // 0 = dpadX, 1 = dpadY - value: dpadY, - }), - "controllerInput", - ); - this.wrtc.sendBinary(toBinary(ProtoMessageSchema, dpadMessage)); - this.inputDetected = true; - this.lastState.dpadY = dpadY; - } + private collectStateChanges(): number { + let changedFields = 0; + + // Collect analog values + const leftTrigger = Math.round( + this.remapFromTo( + this.gamepad.buttons[6]?.value ?? 0, + 0, + 1, + -32768, + 32767, + ), + ); + const rightTrigger = Math.round( + this.remapFromTo( + this.gamepad.buttons[7]?.value ?? 0, + 0, + 1, + -32768, + 32767, + ), + ); - /* Stick handling */ - // stick values need to be mapped from -1.0 to 1.0 to -32768 to 32767 - const leftX = this.remapFromTo( - this.gamepad.axes[0] ?? 0, - -1, - 1, - -32768, - 32767, - ); - const leftY = this.remapFromTo( - this.gamepad.axes[1] ?? 0, - -1, - 1, - -32768, - 32767, - ); - // Apply deadzone - const sendLeftX = - Math.abs(leftX) > this.stickDeadzone ? Math.round(leftX) : 0; - const sendLeftY = - Math.abs(leftY) > this.stickDeadzone ? Math.round(leftY) : 0; - // if outside deadzone, send normally if changed - // if moves inside deadzone, zero it if not inside deadzone last time - if ( - sendLeftX !== this.lastState.leftX || - sendLeftY !== this.lastState.leftY || this.forceFullStateSend - ) { - const stickMessage = createMessage( - create(ProtoControllerStickSchema, { - sessionSlot: this.gamepad.index, - sessionId: this.wrtc.getSessionID(), - stick: 0, // 0 = left, 1 = right - x: sendLeftX, - y: sendLeftY, - }), - "controllerInput", - ); - this.wrtc.sendBinary(toBinary(ProtoMessageSchema, stickMessage)); - this.inputDetected = true; - this.lastState.leftX = sendLeftX; - this.lastState.leftY = sendLeftY; - } + const leftX = this.remapFromTo( + this.gamepad.axes[0] ?? 0, + -1, + 1, + -32768, + 32767, + ); + const leftY = this.remapFromTo( + this.gamepad.axes[1] ?? 0, + -1, + 1, + -32768, + 32767, + ); + const sendLeftX = + Math.abs(leftX) > this.stickDeadzone ? Math.round(leftX) : 0; + const sendLeftY = + Math.abs(leftY) > this.stickDeadzone ? Math.round(leftY) : 0; + + const rightX = this.remapFromTo( + this.gamepad.axes[2] ?? 0, + -1, + 1, + -32768, + 32767, + ); + const rightY = this.remapFromTo( + this.gamepad.axes[3] ?? 0, + -1, + 1, + -32768, + 32767, + ); + const sendRightX = + Math.abs(rightX) > this.stickDeadzone ? Math.round(rightX) : 0; + const sendRightY = + Math.abs(rightY) > this.stickDeadzone ? Math.round(rightY) : 0; + + const dpadX = + (this.gamepad.buttons[14]?.pressed ? -1 : 0) + + (this.gamepad.buttons[15]?.pressed ? 1 : 0); + const dpadY = + (this.gamepad.buttons[12]?.pressed ? -1 : 0) + + (this.gamepad.buttons[13]?.pressed ? 1 : 0); + + // Check what changed + for (let i = 0; i < this.gamepad.buttons.length; i++) { + if (i >= 6 && i <= 7) continue; // Skip triggers + if (i >= 12 && i <= 15) continue; // Skip d-pad + if (this.state.buttonState.get(i) !== this.gamepad.buttons[i].pressed) { + changedFields |= this.CHANGED_BUTTONS_STATE; + } + this.state.buttonState.set(i, this.gamepad.buttons[i].pressed); + } + if (leftTrigger !== this.state.leftTrigger) { + changedFields |= this.CHANGED_LEFT_TRIGGER; + } + this.state.leftTrigger = leftTrigger; + if (rightTrigger !== this.state.rightTrigger) { + changedFields |= this.CHANGED_RIGHT_TRIGGER; + } + this.state.rightTrigger = rightTrigger; + if (sendLeftX !== this.state.leftX) { + changedFields |= this.CHANGED_LEFT_STICK_X; + } + this.state.leftX = sendLeftX; + if (sendLeftY !== this.state.leftY) { + changedFields |= this.CHANGED_LEFT_STICK_Y; + } + this.state.leftY = sendLeftY; + if (sendRightX !== this.state.rightX) { + changedFields |= this.CHANGED_RIGHT_STICK_X; + } + this.state.rightX = sendRightX; + if (sendRightY !== this.state.rightY) { + changedFields |= this.CHANGED_RIGHT_STICK_Y; + } + this.state.rightY = sendRightY; + if (dpadX !== this.state.dpadX) { + changedFields |= this.CHANGED_DPAD_X; + } + this.state.dpadX = dpadX; + if (dpadY !== this.state.dpadY) { + changedFields |= this.CHANGED_DPAD_Y; + } + this.state.dpadY = dpadY; - const rightX = this.remapFromTo( - this.gamepad.axes[2] ?? 0, - -1, - 1, - -32768, - 32767, - ); - const rightY = this.remapFromTo( - this.gamepad.axes[3] ?? 0, - -1, - 1, - -32768, - 32767, - ); - // Apply deadzone - const sendRightX = - Math.abs(rightX) > this.stickDeadzone ? Math.round(rightX) : 0; - const sendRightY = - Math.abs(rightY) > this.stickDeadzone ? Math.round(rightY) : 0; - if ( - sendRightX !== this.lastState.rightX || - sendRightY !== this.lastState.rightY || this.forceFullStateSend - ) { - const stickMessage = createMessage( - create(ProtoControllerStickSchema, { - sessionSlot: this.gamepad.index, - sessionId: this.wrtc.getSessionID(), - stick: 1, // 0 = left, 1 = right - x: sendRightX, - y: sendRightY, + return changedFields; + } + + private sendBatchedState(changedFields: number, updateType: number) { + // @ts-ignore + let message: ProtoControllerStateBatch = { + sessionSlot: this.gamepad.index, + sessionId: this.wrtc.getSessionID(), + updateType: updateType, + sequence: this.sequence++, + }; + + // For FULL_STATE, include everything + if (updateType === 0) { + message.changedFields = 0xFF; + + message.buttonChangedMask = Object.fromEntries( + Array.from(this.state.buttonState).map(([key, value]) => { + return [this.controllerButtonToVirtualKeyCode(key), value]; + }), + ); + message.leftStickX = this.state.leftX; + message.leftStickY = this.state.leftY; + message.rightStickX = this.state.rightX; + message.rightStickY = this.state.rightY; + message.leftTrigger = this.state.leftTrigger; + message.rightTrigger = this.state.rightTrigger; + message.dpadX = this.state.dpadX; + message.dpadY = this.state.dpadY; + } + // For DELTA, only include changed fields + else { + message.changedFields = changedFields; + + if (changedFields & this.CHANGED_BUTTONS_STATE) { + const currentStateMap = this.state.buttonState; + const previousStateMap = this.state.previousButtonState; + const allKeys = new Set([ + // @ts-ignore + ...currentStateMap.keys(), + // @ts-ignore + ...previousStateMap.keys(), + ]); + message.buttonChangedMask = Object.fromEntries( + Array.from(allKeys) + .filter((key) => { + const newState = currentStateMap.get(key); + const oldState = previousStateMap.get(key); + return newState !== oldState; + }) + .map((key) => { + const newValue = currentStateMap.get(key) ?? false; + + return [this.controllerButtonToVirtualKeyCode(key), newValue]; }), - "controllerInput", - ); - this.wrtc.sendBinary(toBinary(ProtoMessageSchema, stickMessage)); - this.inputDetected = true; - this.lastState.rightX = sendRightX; - this.lastState.rightY = sendRightY; - } + ); + } + if (changedFields & this.CHANGED_LEFT_STICK_X) { + message.leftStickX = this.state.leftX; + } + if (changedFields & this.CHANGED_LEFT_STICK_Y) { + message.leftStickY = this.state.leftY; + } + if (changedFields & this.CHANGED_RIGHT_STICK_X) { + message.rightStickX = this.state.rightX; + } + if (changedFields & this.CHANGED_RIGHT_STICK_Y) { + message.rightStickY = this.state.rightY; + } + if (changedFields & this.CHANGED_LEFT_TRIGGER) { + message.leftTrigger = this.state.leftTrigger; + } + if (changedFields & this.CHANGED_RIGHT_TRIGGER) { + message.rightTrigger = this.state.rightTrigger; + } + if (changedFields & this.CHANGED_DPAD_X) { + message.dpadX = this.state.dpadX; + } + if (changedFields & this.CHANGED_DPAD_Y) { + message.dpadY = this.state.dpadY; } } - this.forceFullStateSend = false; + // Send message + const batchMessage = createMessage( + create( + ProtoControllerStateBatchSchema, + message as ProtoControllerStateBatch, + ), + "controllerInput", + ); + this.wrtc.sendBinary(toBinary(ProtoMessageSchema, batchMessage)); } - private loopInterval: any = null; - public run() { if (this.connected) this.stop(); this.connected = true; - this.isIdle = true; - this.lastInputTime = Date.now(); - this.loopInterval = setInterval(() => { - if (this.connected) { - this.inputDetected = false; // Reset before poll - this.pollGamepad(); - - // Switch polling rate based on input - if (this.inputDetected) { - this.lastInputTime = Date.now(); - if (this.isIdle) { - this.isIdle = false; - clearInterval(this.loopInterval); - this.loopInterval = setInterval(() => { - if (this.connected) this.pollGamepad(); - }, this.updateInterval); - } - } else if (!this.isIdle && Date.now() - this.lastInputTime > 200) { - // Switch to idle polling after 200ms of no input - this.isIdle = true; - clearInterval(this.loopInterval); - this.loopInterval = setInterval(() => { - if (this.connected) this.pollGamepad(); - }, this.idleUpdateInterval); - } - } - }, this.isIdle ? this.idleUpdateInterval : this.updateInterval); + // Start with active polling + this.restartPolling(); } public stop() { - if (this.loopInterval) { - clearInterval(this.loopInterval); - this.loopInterval = null; + if (this.pollInterval) { + clearInterval(this.pollInterval); + this.pollInterval = null; } this.connected = false; } @@ -443,17 +478,18 @@ export class Controller { this.wrtc.sendBinary(toBinary(ProtoMessageSchema, detachMsg)); } - private controllerButtonToVirtualKeyCode(code: number) { + private controllerButtonToVirtualKeyCode(code: number): number | undefined { return controllerButtonToLinuxEventCode[code] || undefined; } private rumbleCallback(rumbleMsg: ProtoControllerRumble) { - // If not connected, ignore - if (!this.connected) return; + if (!this.connected || !this.gamepad) return; - // Check if aimed at this controller slot - if (rumbleMsg.sessionId !== this.wrtc.getSessionID() && - rumbleMsg.sessionSlot !== this.gamepad.index) + // Check if this rumble is for us + if ( + rumbleMsg.sessionId !== this.wrtc.getSessionID() && + rumbleMsg.sessionSlot !== this.gamepad.index + ) return; // Trigger actual rumble diff --git a/packages/input/src/proto/messages_pb.ts b/packages/input/src/proto/messages_pb.ts index 4e8c70de..51243dab 100644 --- a/packages/input/src/proto/messages_pb.ts +++ b/packages/input/src/proto/messages_pb.ts @@ -4,7 +4,7 @@ import type { GenFile, GenMessage } from "@bufbuild/protobuf/codegenv2"; import { fileDesc, messageDesc } from "@bufbuild/protobuf/codegenv2"; -import type { ProtoClientDisconnected, ProtoClientRequestRoomStream, ProtoControllerAttach, ProtoControllerAxis, ProtoControllerButton, ProtoControllerDetach, ProtoControllerRumble, ProtoControllerStick, ProtoControllerTrigger, ProtoICE, ProtoKeyDown, ProtoKeyUp, ProtoMouseKeyDown, ProtoMouseKeyUp, ProtoMouseMove, ProtoMouseMoveAbs, ProtoMouseWheel, ProtoRaw, ProtoSDP, ProtoServerPushStream } from "./types_pb"; +import type { ProtoClientDisconnected, ProtoClientRequestRoomStream, ProtoControllerAttach, ProtoControllerDetach, ProtoControllerRumble, ProtoControllerStateBatch, ProtoICE, ProtoKeyDown, ProtoKeyUp, ProtoMouseKeyDown, ProtoMouseKeyUp, ProtoMouseMove, ProtoMouseMoveAbs, ProtoMouseWheel, ProtoRaw, ProtoSDP, ProtoServerPushStream } from "./types_pb"; import { file_types } from "./types_pb"; import type { ProtoLatencyTracker } from "./latency_tracker_pb"; import { file_latency_tracker } from "./latency_tracker_pb"; @@ -14,7 +14,7 @@ import type { Message } from "@bufbuild/protobuf"; * Describes the file messages.proto. */ export const file_messages: GenFile = /*@__PURE__*/ - fileDesc("Cg5tZXNzYWdlcy5wcm90bxIFcHJvdG8iVQoQUHJvdG9NZXNzYWdlQmFzZRIUCgxwYXlsb2FkX3R5cGUYASABKAkSKwoHbGF0ZW5jeRgCIAEoCzIaLnByb3RvLlByb3RvTGF0ZW5jeVRyYWNrZXIiyQgKDFByb3RvTWVzc2FnZRItCgxtZXNzYWdlX2Jhc2UYASABKAsyFy5wcm90by5Qcm90b01lc3NhZ2VCYXNlEisKCm1vdXNlX21vdmUYAiABKAsyFS5wcm90by5Qcm90b01vdXNlTW92ZUgAEjIKDm1vdXNlX21vdmVfYWJzGAMgASgLMhgucHJvdG8uUHJvdG9Nb3VzZU1vdmVBYnNIABItCgttb3VzZV93aGVlbBgEIAEoCzIWLnByb3RvLlByb3RvTW91c2VXaGVlbEgAEjIKDm1vdXNlX2tleV9kb3duGAUgASgLMhgucHJvdG8uUHJvdG9Nb3VzZUtleURvd25IABIuCgxtb3VzZV9rZXlfdXAYBiABKAsyFi5wcm90by5Qcm90b01vdXNlS2V5VXBIABInCghrZXlfZG93bhgHIAEoCzITLnByb3RvLlByb3RvS2V5RG93bkgAEiMKBmtleV91cBgIIAEoCzIRLnByb3RvLlByb3RvS2V5VXBIABI5ChFjb250cm9sbGVyX2F0dGFjaBgJIAEoCzIcLnByb3RvLlByb3RvQ29udHJvbGxlckF0dGFjaEgAEjkKEWNvbnRyb2xsZXJfZGV0YWNoGAogASgLMhwucHJvdG8uUHJvdG9Db250cm9sbGVyRGV0YWNoSAASOQoRY29udHJvbGxlcl9idXR0b24YCyABKAsyHC5wcm90by5Qcm90b0NvbnRyb2xsZXJCdXR0b25IABI7ChJjb250cm9sbGVyX3RyaWdnZXIYDCABKAsyHS5wcm90by5Qcm90b0NvbnRyb2xsZXJUcmlnZ2VySAASNwoQY29udHJvbGxlcl9zdGljaxgNIAEoCzIbLnByb3RvLlByb3RvQ29udHJvbGxlclN0aWNrSAASNQoPY29udHJvbGxlcl9heGlzGA4gASgLMhoucHJvdG8uUHJvdG9Db250cm9sbGVyQXhpc0gAEjkKEWNvbnRyb2xsZXJfcnVtYmxlGA8gASgLMhwucHJvdG8uUHJvdG9Db250cm9sbGVyUnVtYmxlSAASHgoDaWNlGBQgASgLMg8ucHJvdG8uUHJvdG9JQ0VIABIeCgNzZHAYFSABKAsyDy5wcm90by5Qcm90b1NEUEgAEh4KA3JhdxgWIAEoCzIPLnByb3RvLlByb3RvUmF3SAASSQoaY2xpZW50X3JlcXVlc3Rfcm9vbV9zdHJlYW0YFyABKAsyIy5wcm90by5Qcm90b0NsaWVudFJlcXVlc3RSb29tU3RyZWFtSAASPQoTY2xpZW50X2Rpc2Nvbm5lY3RlZBgYIAEoCzIeLnByb3RvLlByb3RvQ2xpZW50RGlzY29ubmVjdGVkSAASOgoSc2VydmVyX3B1c2hfc3RyZWFtGBkgASgLMhwucHJvdG8uUHJvdG9TZXJ2ZXJQdXNoU3RyZWFtSABCCQoHcGF5bG9hZEIWWhRyZWxheS9pbnRlcm5hbC9wcm90b2IGcHJvdG8z", [file_types, file_latency_tracker]); + fileDesc("Cg5tZXNzYWdlcy5wcm90bxIFcHJvdG8iVQoQUHJvdG9NZXNzYWdlQmFzZRIUCgxwYXlsb2FkX3R5cGUYASABKAkSKwoHbGF0ZW5jeRgCIAEoCzIaLnByb3RvLlByb3RvTGF0ZW5jeVRyYWNrZXIipQcKDFByb3RvTWVzc2FnZRItCgxtZXNzYWdlX2Jhc2UYASABKAsyFy5wcm90by5Qcm90b01lc3NhZ2VCYXNlEisKCm1vdXNlX21vdmUYAiABKAsyFS5wcm90by5Qcm90b01vdXNlTW92ZUgAEjIKDm1vdXNlX21vdmVfYWJzGAMgASgLMhgucHJvdG8uUHJvdG9Nb3VzZU1vdmVBYnNIABItCgttb3VzZV93aGVlbBgEIAEoCzIWLnByb3RvLlByb3RvTW91c2VXaGVlbEgAEjIKDm1vdXNlX2tleV9kb3duGAUgASgLMhgucHJvdG8uUHJvdG9Nb3VzZUtleURvd25IABIuCgxtb3VzZV9rZXlfdXAYBiABKAsyFi5wcm90by5Qcm90b01vdXNlS2V5VXBIABInCghrZXlfZG93bhgHIAEoCzITLnByb3RvLlByb3RvS2V5RG93bkgAEiMKBmtleV91cBgIIAEoCzIRLnByb3RvLlByb3RvS2V5VXBIABI5ChFjb250cm9sbGVyX2F0dGFjaBgJIAEoCzIcLnByb3RvLlByb3RvQ29udHJvbGxlckF0dGFjaEgAEjkKEWNvbnRyb2xsZXJfZGV0YWNoGAogASgLMhwucHJvdG8uUHJvdG9Db250cm9sbGVyRGV0YWNoSAASOQoRY29udHJvbGxlcl9ydW1ibGUYCyABKAsyHC5wcm90by5Qcm90b0NvbnRyb2xsZXJSdW1ibGVIABJCChZjb250cm9sbGVyX3N0YXRlX2JhdGNoGAwgASgLMiAucHJvdG8uUHJvdG9Db250cm9sbGVyU3RhdGVCYXRjaEgAEh4KA2ljZRgUIAEoCzIPLnByb3RvLlByb3RvSUNFSAASHgoDc2RwGBUgASgLMg8ucHJvdG8uUHJvdG9TRFBIABIeCgNyYXcYFiABKAsyDy5wcm90by5Qcm90b1Jhd0gAEkkKGmNsaWVudF9yZXF1ZXN0X3Jvb21fc3RyZWFtGBcgASgLMiMucHJvdG8uUHJvdG9DbGllbnRSZXF1ZXN0Um9vbVN0cmVhbUgAEj0KE2NsaWVudF9kaXNjb25uZWN0ZWQYGCABKAsyHi5wcm90by5Qcm90b0NsaWVudERpc2Nvbm5lY3RlZEgAEjoKEnNlcnZlcl9wdXNoX3N0cmVhbRgZIAEoCzIcLnByb3RvLlByb3RvU2VydmVyUHVzaFN0cmVhbUgAQgkKB3BheWxvYWRCFloUcmVsYXkvaW50ZXJuYWwvcHJvdG9iBnByb3RvMw", [file_types, file_latency_tracker]); /** * @generated from message proto.ProtoMessageBase @@ -96,6 +96,8 @@ export type ProtoMessage = Message<"proto.ProtoMessage"> & { case: "keyUp"; } | { /** + * Controller input types + * * @generated from field: proto.ProtoControllerAttach controller_attach = 9; */ value: ProtoControllerAttach; @@ -108,34 +110,16 @@ export type ProtoMessage = Message<"proto.ProtoMessage"> & { case: "controllerDetach"; } | { /** - * @generated from field: proto.ProtoControllerButton controller_button = 11; - */ - value: ProtoControllerButton; - case: "controllerButton"; - } | { - /** - * @generated from field: proto.ProtoControllerTrigger controller_trigger = 12; - */ - value: ProtoControllerTrigger; - case: "controllerTrigger"; - } | { - /** - * @generated from field: proto.ProtoControllerStick controller_stick = 13; + * @generated from field: proto.ProtoControllerRumble controller_rumble = 11; */ - value: ProtoControllerStick; - case: "controllerStick"; - } | { - /** - * @generated from field: proto.ProtoControllerAxis controller_axis = 14; - */ - value: ProtoControllerAxis; - case: "controllerAxis"; + value: ProtoControllerRumble; + case: "controllerRumble"; } | { /** - * @generated from field: proto.ProtoControllerRumble controller_rumble = 15; + * @generated from field: proto.ProtoControllerStateBatch controller_state_batch = 12; */ - value: ProtoControllerRumble; - case: "controllerRumble"; + value: ProtoControllerStateBatch; + case: "controllerStateBatch"; } | { /** * Signaling types diff --git a/packages/input/src/proto/types_pb.ts b/packages/input/src/proto/types_pb.ts index 6da4bf1e..d4a8c6cc 100644 --- a/packages/input/src/proto/types_pb.ts +++ b/packages/input/src/proto/types_pb.ts @@ -2,15 +2,15 @@ // @generated from file types.proto (package proto, syntax proto3) /* eslint-disable */ -import type { GenFile, GenMessage } from "@bufbuild/protobuf/codegenv2"; -import { fileDesc, messageDesc } from "@bufbuild/protobuf/codegenv2"; +import type { GenEnum, GenFile, GenMessage } from "@bufbuild/protobuf/codegenv2"; +import { enumDesc, fileDesc, messageDesc } from "@bufbuild/protobuf/codegenv2"; import type { Message } from "@bufbuild/protobuf"; /** * Describes the file types.proto. */ export const file_types: GenFile = /*@__PURE__*/ - fileDesc("Cgt0eXBlcy5wcm90bxIFcHJvdG8iJgoOUHJvdG9Nb3VzZU1vdmUSCQoBeBgBIAEoBRIJCgF5GAIgASgFIikKEVByb3RvTW91c2VNb3ZlQWJzEgkKAXgYASABKAUSCQoBeRgCIAEoBSInCg9Qcm90b01vdXNlV2hlZWwSCQoBeBgBIAEoBRIJCgF5GAIgASgFIiAKEVByb3RvTW91c2VLZXlEb3duEgsKA2tleRgBIAEoBSIeCg9Qcm90b01vdXNlS2V5VXASCwoDa2V5GAEgASgFIhsKDFByb3RvS2V5RG93bhILCgNrZXkYASABKAUiGQoKUHJvdG9LZXlVcBILCgNrZXkYASABKAUiTQoVUHJvdG9Db250cm9sbGVyQXR0YWNoEgoKAmlkGAEgASgJEhQKDHNlc3Npb25fc2xvdBgCIAEoBRISCgpzZXNzaW9uX2lkGAMgASgJIkEKFVByb3RvQ29udHJvbGxlckRldGFjaBIUCgxzZXNzaW9uX3Nsb3QYASABKAUSEgoKc2Vzc2lvbl9pZBgCIAEoCSJiChVQcm90b0NvbnRyb2xsZXJCdXR0b24SFAoMc2Vzc2lvbl9zbG90GAEgASgFEhIKCnNlc3Npb25faWQYAiABKAkSDgoGYnV0dG9uGAMgASgFEg8KB3ByZXNzZWQYBCABKAgiYgoWUHJvdG9Db250cm9sbGVyVHJpZ2dlchIUCgxzZXNzaW9uX3Nsb3QYASABKAUSEgoKc2Vzc2lvbl9pZBgCIAEoCRIPCgd0cmlnZ2VyGAMgASgFEg0KBXZhbHVlGAQgASgFImUKFFByb3RvQ29udHJvbGxlclN0aWNrEhQKDHNlc3Npb25fc2xvdBgBIAEoBRISCgpzZXNzaW9uX2lkGAIgASgJEg0KBXN0aWNrGAMgASgFEgkKAXgYBCABKAUSCQoBeRgFIAEoBSJcChNQcm90b0NvbnRyb2xsZXJBeGlzEhQKDHNlc3Npb25fc2xvdBgBIAEoBRISCgpzZXNzaW9uX2lkGAIgASgJEgwKBGF4aXMYAyABKAUSDQoFdmFsdWUYBCABKAUiggEKFVByb3RvQ29udHJvbGxlclJ1bWJsZRIUCgxzZXNzaW9uX3Nsb3QYASABKAUSEgoKc2Vzc2lvbl9pZBgCIAEoCRIVCg1sb3dfZnJlcXVlbmN5GAMgASgFEhYKDmhpZ2hfZnJlcXVlbmN5GAQgASgFEhAKCGR1cmF0aW9uGAUgASgFIqoBChNSVENJY2VDYW5kaWRhdGVJbml0EhEKCWNhbmRpZGF0ZRgBIAEoCRIaCg1zZHBNTGluZUluZGV4GAIgASgNSACIAQESEwoGc2RwTWlkGAMgASgJSAGIAQESHQoQdXNlcm5hbWVGcmFnbWVudBgEIAEoCUgCiAEBQhAKDl9zZHBNTGluZUluZGV4QgkKB19zZHBNaWRCEwoRX3VzZXJuYW1lRnJhZ21lbnQiNgoZUlRDU2Vzc2lvbkRlc2NyaXB0aW9uSW5pdBILCgNzZHAYASABKAkSDAoEdHlwZRgCIAEoCSI5CghQcm90b0lDRRItCgljYW5kaWRhdGUYASABKAsyGi5wcm90by5SVENJY2VDYW5kaWRhdGVJbml0IjkKCFByb3RvU0RQEi0KA3NkcBgBIAEoCzIgLnByb3RvLlJUQ1Nlc3Npb25EZXNjcmlwdGlvbkluaXQiGAoIUHJvdG9SYXcSDAoEZGF0YRgBIAEoCSJFChxQcm90b0NsaWVudFJlcXVlc3RSb29tU3RyZWFtEhEKCXJvb21fbmFtZRgBIAEoCRISCgpzZXNzaW9uX2lkGAIgASgJIkcKF1Byb3RvQ2xpZW50RGlzY29ubmVjdGVkEhIKCnNlc3Npb25faWQYASABKAkSGAoQY29udHJvbGxlcl9zbG90cxgCIAMoBSIqChVQcm90b1NlcnZlclB1c2hTdHJlYW0SEQoJcm9vbV9uYW1lGAEgASgJQhZaFHJlbGF5L2ludGVybmFsL3Byb3RvYgZwcm90bzM"); + fileDesc("Cgt0eXBlcy5wcm90bxIFcHJvdG8iJgoOUHJvdG9Nb3VzZU1vdmUSCQoBeBgBIAEoBRIJCgF5GAIgASgFIikKEVByb3RvTW91c2VNb3ZlQWJzEgkKAXgYASABKAUSCQoBeRgCIAEoBSInCg9Qcm90b01vdXNlV2hlZWwSCQoBeBgBIAEoBRIJCgF5GAIgASgFIiAKEVByb3RvTW91c2VLZXlEb3duEgsKA2tleRgBIAEoBSIeCg9Qcm90b01vdXNlS2V5VXASCwoDa2V5GAEgASgFIhsKDFByb3RvS2V5RG93bhILCgNrZXkYASABKAUiGQoKUHJvdG9LZXlVcBILCgNrZXkYASABKAUiTQoVUHJvdG9Db250cm9sbGVyQXR0YWNoEgoKAmlkGAEgASgJEhQKDHNlc3Npb25fc2xvdBgCIAEoBRISCgpzZXNzaW9uX2lkGAMgASgJIkEKFVByb3RvQ29udHJvbGxlckRldGFjaBIUCgxzZXNzaW9uX3Nsb3QYASABKAUSEgoKc2Vzc2lvbl9pZBgCIAEoCSKCAQoVUHJvdG9Db250cm9sbGVyUnVtYmxlEhQKDHNlc3Npb25fc2xvdBgBIAEoBRISCgpzZXNzaW9uX2lkGAIgASgJEhUKDWxvd19mcmVxdWVuY3kYAyABKAUSFgoOaGlnaF9mcmVxdWVuY3kYBCABKAUSEAoIZHVyYXRpb24YBSABKAUi0AUKGVByb3RvQ29udHJvbGxlclN0YXRlQmF0Y2gSFAoMc2Vzc2lvbl9zbG90GAEgASgFEhIKCnNlc3Npb25faWQYAiABKAkSQAoLdXBkYXRlX3R5cGUYAyABKA4yKy5wcm90by5Qcm90b0NvbnRyb2xsZXJTdGF0ZUJhdGNoLlVwZGF0ZVR5cGUSEAoIc2VxdWVuY2UYBCABKA0SVAoTYnV0dG9uX2NoYW5nZWRfbWFzaxgFIAMoCzI3LnByb3RvLlByb3RvQ29udHJvbGxlclN0YXRlQmF0Y2guQnV0dG9uQ2hhbmdlZE1hc2tFbnRyeRIZCgxsZWZ0X3N0aWNrX3gYBiABKAVIAIgBARIZCgxsZWZ0X3N0aWNrX3kYByABKAVIAYgBARIaCg1yaWdodF9zdGlja194GAggASgFSAKIAQESGgoNcmlnaHRfc3RpY2tfeRgJIAEoBUgDiAEBEhkKDGxlZnRfdHJpZ2dlchgKIAEoBUgEiAEBEhoKDXJpZ2h0X3RyaWdnZXIYCyABKAVIBYgBARITCgZkcGFkX3gYDCABKAVIBogBARITCgZkcGFkX3kYDSABKAVIB4gBARIbCg5jaGFuZ2VkX2ZpZWxkcxgOIAEoDUgIiAEBGjgKFkJ1dHRvbkNoYW5nZWRNYXNrRW50cnkSCwoDa2V5GAEgASgFEg0KBXZhbHVlGAIgASgIOgI4ASInCgpVcGRhdGVUeXBlEg4KCkZVTExfU1RBVEUQABIJCgVERUxUQRABQg8KDV9sZWZ0X3N0aWNrX3hCDwoNX2xlZnRfc3RpY2tfeUIQCg5fcmlnaHRfc3RpY2tfeEIQCg5fcmlnaHRfc3RpY2tfeUIPCg1fbGVmdF90cmlnZ2VyQhAKDl9yaWdodF90cmlnZ2VyQgkKB19kcGFkX3hCCQoHX2RwYWRfeUIRCg9fY2hhbmdlZF9maWVsZHMiqgEKE1JUQ0ljZUNhbmRpZGF0ZUluaXQSEQoJY2FuZGlkYXRlGAEgASgJEhoKDXNkcE1MaW5lSW5kZXgYAiABKA1IAIgBARITCgZzZHBNaWQYAyABKAlIAYgBARIdChB1c2VybmFtZUZyYWdtZW50GAQgASgJSAKIAQFCEAoOX3NkcE1MaW5lSW5kZXhCCQoHX3NkcE1pZEITChFfdXNlcm5hbWVGcmFnbWVudCI2ChlSVENTZXNzaW9uRGVzY3JpcHRpb25Jbml0EgsKA3NkcBgBIAEoCRIMCgR0eXBlGAIgASgJIjkKCFByb3RvSUNFEi0KCWNhbmRpZGF0ZRgBIAEoCzIaLnByb3RvLlJUQ0ljZUNhbmRpZGF0ZUluaXQiOQoIUHJvdG9TRFASLQoDc2RwGAEgASgLMiAucHJvdG8uUlRDU2Vzc2lvbkRlc2NyaXB0aW9uSW5pdCIYCghQcm90b1JhdxIMCgRkYXRhGAEgASgJIkUKHFByb3RvQ2xpZW50UmVxdWVzdFJvb21TdHJlYW0SEQoJcm9vbV9uYW1lGAEgASgJEhIKCnNlc3Npb25faWQYAiABKAkiRwoXUHJvdG9DbGllbnREaXNjb25uZWN0ZWQSEgoKc2Vzc2lvbl9pZBgBIAEoCRIYChBjb250cm9sbGVyX3Nsb3RzGAIgAygFIioKFVByb3RvU2VydmVyUHVzaFN0cmVhbRIRCglyb29tX25hbWUYASABKAlCFloUcmVsYXkvaW50ZXJuYWwvcHJvdG9iBnByb3RvMw"); /** * MouseMove message @@ -224,11 +224,11 @@ export const ProtoControllerDetachSchema: GenMessage = /* messageDesc(file_types, 8); /** - * ControllerButton message + * ControllerRumble message * - * @generated from message proto.ProtoControllerButton + * @generated from message proto.ProtoControllerRumble */ -export type ProtoControllerButton = Message<"proto.ProtoControllerButton"> & { +export type ProtoControllerRumble = Message<"proto.ProtoControllerRumble"> & { /** * Session specific slot number (0-3) * @@ -244,33 +244,40 @@ export type ProtoControllerButton = Message<"proto.ProtoControllerButton"> & { sessionId: string; /** - * Button code (linux input event code) + * Low frequency rumble (0-65535) * - * @generated from field: int32 button = 3; + * @generated from field: int32 low_frequency = 3; */ - button: number; + lowFrequency: number; + + /** + * High frequency rumble (0-65535) + * + * @generated from field: int32 high_frequency = 4; + */ + highFrequency: number; /** - * true if pressed, false if released + * Duration in milliseconds * - * @generated from field: bool pressed = 4; + * @generated from field: int32 duration = 5; */ - pressed: boolean; + duration: number; }; /** - * Describes the message proto.ProtoControllerButton. - * Use `create(ProtoControllerButtonSchema)` to create a new message. + * Describes the message proto.ProtoControllerRumble. + * Use `create(ProtoControllerRumbleSchema)` to create a new message. */ -export const ProtoControllerButtonSchema: GenMessage = /*@__PURE__*/ +export const ProtoControllerRumbleSchema: GenMessage = /*@__PURE__*/ messageDesc(file_types, 9); /** - * ControllerTriggers message + * ControllerStateBatch - single message containing full or partial controller state * - * @generated from message proto.ProtoControllerTrigger + * @generated from message proto.ProtoControllerStateBatch */ -export type ProtoControllerTrigger = Message<"proto.ProtoControllerTrigger"> & { +export type ProtoControllerStateBatch = Message<"proto.ProtoControllerStateBatch"> & { /** * Session specific slot number (0-3) * @@ -286,166 +293,122 @@ export type ProtoControllerTrigger = Message<"proto.ProtoControllerTrigger"> & { sessionId: string; /** - * Trigger number (0 for left, 1 for right) - * - * @generated from field: int32 trigger = 3; + * @generated from field: proto.ProtoControllerStateBatch.UpdateType update_type = 3; */ - trigger: number; + updateType: ProtoControllerStateBatch_UpdateType; /** - * trigger value (-32768 to 32767) + * Sequence number for packet loss detection * - * @generated from field: int32 value = 4; + * @generated from field: uint32 sequence = 4; */ - value: number; -}; - -/** - * Describes the message proto.ProtoControllerTrigger. - * Use `create(ProtoControllerTriggerSchema)` to create a new message. - */ -export const ProtoControllerTriggerSchema: GenMessage = /*@__PURE__*/ - messageDesc(file_types, 10); + sequence: number; -/** - * ControllerSticks message - * - * @generated from message proto.ProtoControllerStick - */ -export type ProtoControllerStick = Message<"proto.ProtoControllerStick"> & { /** - * Session specific slot number (0-3) + * Button state map (Linux event codes) * - * @generated from field: int32 session_slot = 1; + * @generated from field: map button_changed_mask = 5; */ - sessionSlot: number; + buttonChangedMask: { [key: number]: boolean }; /** - * Session ID of the client + * Analog inputs * - * @generated from field: string session_id = 2; + * -32768 to 32767 + * + * @generated from field: optional int32 left_stick_x = 6; */ - sessionId: string; + leftStickX?: number; /** - * Stick number (0 for left, 1 for right) + * -32768 to 32767 * - * @generated from field: int32 stick = 3; + * @generated from field: optional int32 left_stick_y = 7; */ - stick: number; + leftStickY?: number; /** - * X axis value (-32768 to 32767) + * -32768 to 32767 * - * @generated from field: int32 x = 4; + * @generated from field: optional int32 right_stick_x = 8; */ - x: number; + rightStickX?: number; /** - * Y axis value (-32768 to 32767) + * -32768 to 32767 * - * @generated from field: int32 y = 5; + * @generated from field: optional int32 right_stick_y = 9; */ - y: number; -}; + rightStickY?: number; -/** - * Describes the message proto.ProtoControllerStick. - * Use `create(ProtoControllerStickSchema)` to create a new message. - */ -export const ProtoControllerStickSchema: GenMessage = /*@__PURE__*/ - messageDesc(file_types, 11); + /** + * -32768 to 32767 + * + * @generated from field: optional int32 left_trigger = 10; + */ + leftTrigger?: number; -/** - * ControllerAxis message - * - * @generated from message proto.ProtoControllerAxis - */ -export type ProtoControllerAxis = Message<"proto.ProtoControllerAxis"> & { /** - * Session specific slot number (0-3) + * -32768 to 32767 * - * @generated from field: int32 session_slot = 1; + * @generated from field: optional int32 right_trigger = 11; */ - sessionSlot: number; + rightTrigger?: number; /** - * Session ID of the client + * -1, 0, or 1 * - * @generated from field: string session_id = 2; + * @generated from field: optional int32 dpad_x = 12; */ - sessionId: string; + dpadX?: number; /** - * Axis number (0 for d-pad horizontal, 1 for d-pad vertical) + * -1, 0, or 1 * - * @generated from field: int32 axis = 3; + * @generated from field: optional int32 dpad_y = 13; */ - axis: number; + dpadY?: number; /** - * axis value (-1 to 1) + * Bitmask indicating which fields have changed + * Bit 0: button_changed_mask, Bit 1: left_stick_x, Bit 2: left_stick_y, etc. * - * @generated from field: int32 value = 4; + * @generated from field: optional uint32 changed_fields = 14; */ - value: number; + changedFields?: number; }; /** - * Describes the message proto.ProtoControllerAxis. - * Use `create(ProtoControllerAxisSchema)` to create a new message. + * Describes the message proto.ProtoControllerStateBatch. + * Use `create(ProtoControllerStateBatchSchema)` to create a new message. */ -export const ProtoControllerAxisSchema: GenMessage = /*@__PURE__*/ - messageDesc(file_types, 12); +export const ProtoControllerStateBatchSchema: GenMessage = /*@__PURE__*/ + messageDesc(file_types, 10); /** - * ControllerRumble message - * - * @generated from message proto.ProtoControllerRumble + * @generated from enum proto.ProtoControllerStateBatch.UpdateType */ -export type ProtoControllerRumble = Message<"proto.ProtoControllerRumble"> & { - /** - * Session specific slot number (0-3) - * - * @generated from field: int32 session_slot = 1; - */ - sessionSlot: number; - - /** - * Session ID of the client - * - * @generated from field: string session_id = 2; - */ - sessionId: string; - - /** - * Low frequency rumble (0-65535) - * - * @generated from field: int32 low_frequency = 3; - */ - lowFrequency: number; - +export enum ProtoControllerStateBatch_UpdateType { /** - * High frequency rumble (0-65535) + * Complete controller state * - * @generated from field: int32 high_frequency = 4; + * @generated from enum value: FULL_STATE = 0; */ - highFrequency: number; + FULL_STATE = 0, /** - * Duration in milliseconds + * Only changed fields * - * @generated from field: int32 duration = 5; + * @generated from enum value: DELTA = 1; */ - duration: number; -}; + DELTA = 1, +} /** - * Describes the message proto.ProtoControllerRumble. - * Use `create(ProtoControllerRumbleSchema)` to create a new message. + * Describes the enum proto.ProtoControllerStateBatch.UpdateType. */ -export const ProtoControllerRumbleSchema: GenMessage = /*@__PURE__*/ - messageDesc(file_types, 13); +export const ProtoControllerStateBatch_UpdateTypeSchema: GenEnum = /*@__PURE__*/ + enumDesc(file_types, 10, 0); /** * @generated from message proto.RTCIceCandidateInit @@ -477,7 +440,7 @@ export type RTCIceCandidateInit = Message<"proto.RTCIceCandidateInit"> & { * Use `create(RTCIceCandidateInitSchema)` to create a new message. */ export const RTCIceCandidateInitSchema: GenMessage = /*@__PURE__*/ - messageDesc(file_types, 14); + messageDesc(file_types, 11); /** * @generated from message proto.RTCSessionDescriptionInit @@ -499,7 +462,7 @@ export type RTCSessionDescriptionInit = Message<"proto.RTCSessionDescriptionInit * Use `create(RTCSessionDescriptionInitSchema)` to create a new message. */ export const RTCSessionDescriptionInitSchema: GenMessage = /*@__PURE__*/ - messageDesc(file_types, 15); + messageDesc(file_types, 12); /** * ProtoICE message @@ -518,7 +481,7 @@ export type ProtoICE = Message<"proto.ProtoICE"> & { * Use `create(ProtoICESchema)` to create a new message. */ export const ProtoICESchema: GenMessage = /*@__PURE__*/ - messageDesc(file_types, 16); + messageDesc(file_types, 13); /** * ProtoSDP message @@ -537,7 +500,7 @@ export type ProtoSDP = Message<"proto.ProtoSDP"> & { * Use `create(ProtoSDPSchema)` to create a new message. */ export const ProtoSDPSchema: GenMessage = /*@__PURE__*/ - messageDesc(file_types, 17); + messageDesc(file_types, 14); /** * ProtoRaw message @@ -556,7 +519,7 @@ export type ProtoRaw = Message<"proto.ProtoRaw"> & { * Use `create(ProtoRawSchema)` to create a new message. */ export const ProtoRawSchema: GenMessage = /*@__PURE__*/ - messageDesc(file_types, 18); + messageDesc(file_types, 15); /** * ProtoClientRequestRoomStream message @@ -580,7 +543,7 @@ export type ProtoClientRequestRoomStream = Message<"proto.ProtoClientRequestRoom * Use `create(ProtoClientRequestRoomStreamSchema)` to create a new message. */ export const ProtoClientRequestRoomStreamSchema: GenMessage = /*@__PURE__*/ - messageDesc(file_types, 19); + messageDesc(file_types, 16); /** * ProtoClientDisconnected message @@ -604,7 +567,7 @@ export type ProtoClientDisconnected = Message<"proto.ProtoClientDisconnected"> & * Use `create(ProtoClientDisconnectedSchema)` to create a new message. */ export const ProtoClientDisconnectedSchema: GenMessage = /*@__PURE__*/ - messageDesc(file_types, 20); + messageDesc(file_types, 17); /** * ProtoServerPushStream message @@ -623,5 +586,5 @@ export type ProtoServerPushStream = Message<"proto.ProtoServerPushStream"> & { * Use `create(ProtoServerPushStreamSchema)` to create a new message. */ export const ProtoServerPushStreamSchema: GenMessage = /*@__PURE__*/ - messageDesc(file_types, 21); + messageDesc(file_types, 18); diff --git a/packages/play-standalone/package.json b/packages/play-standalone/package.json index 9db4e167..7b43d6e6 100644 --- a/packages/play-standalone/package.json +++ b/packages/play-standalone/package.json @@ -9,8 +9,8 @@ "astro": "astro" }, "dependencies": { - "@astrojs/node": "^9.4.2", + "@astrojs/node": "9.5.0", "@nestri/input": "*", - "astro": "5.14.5" + "astro": "5.15.1" } } \ No newline at end of file diff --git a/packages/relay/go.mod b/packages/relay/go.mod index f0f53dd3..28e53f7f 100644 --- a/packages/relay/go.mod +++ b/packages/relay/go.mod @@ -10,7 +10,7 @@ require ( github.com/oklog/ulid/v2 v2.1.1 github.com/pion/ice/v4 v4.0.10 github.com/pion/interceptor v0.1.41 - github.com/pion/rtp v1.8.24 + github.com/pion/rtp v1.8.25 github.com/pion/webrtc/v4 v4.1.6 github.com/prometheus/client_golang v1.23.2 google.golang.org/protobuf v1.36.10 @@ -30,7 +30,7 @@ require ( github.com/gorilla/websocket v1.5.3 // indirect github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect github.com/huin/goupnp v1.3.0 // indirect - github.com/ipfs/go-cid v0.5.0 // indirect + github.com/ipfs/go-cid v0.6.0 // indirect github.com/jackpal/go-nat-pmp v1.0.2 // indirect github.com/jbenet/go-temp-err-catcher v0.1.0 // indirect github.com/klauspost/compress v1.18.1 // indirect @@ -40,7 +40,7 @@ require ( github.com/libp2p/go-flow-metrics v0.3.0 // indirect github.com/libp2p/go-libp2p-asn-util v0.4.1 // indirect github.com/libp2p/go-msgio v0.3.0 // indirect - github.com/libp2p/go-netroute v0.3.0 // indirect + github.com/libp2p/go-netroute v0.4.0 // indirect github.com/libp2p/go-yamux/v5 v5.1.0 // indirect github.com/libp2p/zeroconf/v2 v2.2.0 // indirect github.com/marten-seemann/tcp v0.0.0-20210406111302-dfbc87cc63fd // indirect @@ -71,13 +71,13 @@ require ( github.com/pion/sdp/v3 v3.0.16 // indirect github.com/pion/srtp/v3 v3.0.8 // indirect github.com/pion/stun v0.6.1 // indirect - github.com/pion/stun/v3 v3.0.0 // indirect + github.com/pion/stun/v3 v3.0.1 // indirect github.com/pion/transport/v2 v2.2.10 // indirect github.com/pion/transport/v3 v3.0.8 // indirect - github.com/pion/turn/v4 v4.1.1 // indirect + github.com/pion/turn/v4 v4.1.2 // indirect github.com/prometheus/client_model v0.6.2 // indirect - github.com/prometheus/common v0.67.1 // indirect - github.com/prometheus/procfs v0.17.0 // indirect + github.com/prometheus/common v0.67.2 // indirect + github.com/prometheus/procfs v0.19.2 // indirect github.com/quic-go/qpack v0.5.1 // indirect github.com/quic-go/quic-go v0.55.0 // indirect github.com/quic-go/webtransport-go v0.9.0 // indirect @@ -91,12 +91,12 @@ require ( go.uber.org/zap v1.27.0 // indirect go.yaml.in/yaml/v2 v2.4.3 // indirect golang.org/x/crypto v0.43.0 // indirect - golang.org/x/exp v0.0.0-20251017212417-90e834f514db // indirect + golang.org/x/exp v0.0.0-20251023183803-a4bb9ffd2546 // indirect golang.org/x/mod v0.29.0 // indirect golang.org/x/net v0.46.0 // indirect golang.org/x/sync v0.17.0 // indirect golang.org/x/sys v0.37.0 // indirect - golang.org/x/telemetry v0.0.0-20251014153721-24f779f6aaef // indirect + golang.org/x/telemetry v0.0.0-20251028164327-d7a2859f34e8 // indirect golang.org/x/text v0.30.0 // indirect golang.org/x/time v0.14.0 // indirect golang.org/x/tools v0.38.0 // indirect diff --git a/packages/relay/go.sum b/packages/relay/go.sum index 7f5ff0b3..0500e36c 100644 --- a/packages/relay/go.sum +++ b/packages/relay/go.sum @@ -71,8 +71,8 @@ github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= github.com/huin/goupnp v1.3.0 h1:UvLUlWDNpoUdYzb2TCn+MuTWtcjXKSza2n6CBdQ0xXc= github.com/huin/goupnp v1.3.0/go.mod h1:gnGPsThkYa7bFi/KWmEysQRf48l2dvR5bxr2OFckNX8= -github.com/ipfs/go-cid v0.5.0 h1:goEKKhaGm0ul11IHA7I6p1GmKz8kEYniqFopaB5Otwg= -github.com/ipfs/go-cid v0.5.0/go.mod h1:0L7vmeNXpQpUS9vt+yEARkJ8rOg43DF3iPgn4GIN0mk= +github.com/ipfs/go-cid v0.6.0 h1:DlOReBV1xhHBhhfy/gBNNTSyfOM6rLiIx9J7A4DGf30= +github.com/ipfs/go-cid v0.6.0/go.mod h1:NC4kS1LZjzfhK40UGmpXv5/qD2kcMzACYJNntCUiDhQ= github.com/jackpal/go-nat-pmp v1.0.2 h1:KzKSgb7qkJvOUTqYl9/Hg/me3pWgBmERKrTGD7BdWus= github.com/jackpal/go-nat-pmp v1.0.2/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc= github.com/jbenet/go-temp-err-catcher v0.1.0 h1:zpb3ZH6wIE8Shj2sKS+khgRvf7T7RABoLk/+KKHggpk= @@ -113,8 +113,8 @@ github.com/libp2p/go-libp2p-testing v0.12.0 h1:EPvBb4kKMWO29qP4mZGyhVzUyR25dvfUI github.com/libp2p/go-libp2p-testing v0.12.0/go.mod h1:KcGDRXyN7sQCllucn1cOOS+Dmm7ujhfEyXQL5lvkcPg= github.com/libp2p/go-msgio v0.3.0 h1:mf3Z8B1xcFN314sWX+2vOTShIE0Mmn2TXn3YCUQGNj0= github.com/libp2p/go-msgio v0.3.0/go.mod h1:nyRM819GmVaF9LX3l03RMh10QdOroF++NBbxAb0mmDM= -github.com/libp2p/go-netroute v0.3.0 h1:nqPCXHmeNmgTJnktosJ/sIef9hvwYCrsLxXmfNks/oc= -github.com/libp2p/go-netroute v0.3.0/go.mod h1:Nkd5ShYgSMS5MUKy/MU2T57xFoOKvvLR92Lic48LEyA= +github.com/libp2p/go-netroute v0.4.0 h1:sZZx9hyANYUx9PZyqcgE/E1GUG3iEtTZHUEvdtXT7/Q= +github.com/libp2p/go-netroute v0.4.0/go.mod h1:Nkd5ShYgSMS5MUKy/MU2T57xFoOKvvLR92Lic48LEyA= github.com/libp2p/go-reuseport v0.4.0 h1:nR5KU7hD0WxXCJbmw7r2rhRYruNRl2koHw8fQscQm2s= github.com/libp2p/go-reuseport v0.4.0/go.mod h1:ZtI03j/wO5hZVDFo2jKywN6bYKWLOy8Se6DrI2E1cLU= github.com/libp2p/go-yamux/v5 v5.1.0 h1:8Qlxj4E9JGJAQVW6+uj2o7mqkqsIVlSUGmTWhlXzoHE= @@ -199,8 +199,8 @@ github.com/pion/randutil v0.1.0 h1:CFG1UdESneORglEsnimhUjf33Rwjubwj6xfiOXBa3mA= github.com/pion/randutil v0.1.0/go.mod h1:XcJrSMMbbMRhASFVOlj/5hQial/Y8oH/HVo7TBZq+j8= github.com/pion/rtcp v1.2.16 h1:fk1B1dNW4hsI78XUCljZJlC4kZOPk67mNRuQ0fcEkSo= github.com/pion/rtcp v1.2.16/go.mod h1:/as7VKfYbs5NIb4h6muQ35kQF/J0ZVNz2Z3xKoCBYOo= -github.com/pion/rtp v1.8.24 h1:+ICyZXUQDv95EsHN70RrA4XKJf5MGWyC6QQc1u6/ynI= -github.com/pion/rtp v1.8.24/go.mod h1:rF5nS1GqbR7H/TCpKwylzeq6yDM+MM6k+On5EgeThEM= +github.com/pion/rtp v1.8.25 h1:b8+y44GNbwOJTYWuVan7SglX/hMlicVCAtL50ztyZHw= +github.com/pion/rtp v1.8.25/go.mod h1:rF5nS1GqbR7H/TCpKwylzeq6yDM+MM6k+On5EgeThEM= github.com/pion/sctp v1.8.40 h1:bqbgWYOrUhsYItEnRObUYZuzvOMsVplS3oNgzedBlG8= github.com/pion/sctp v1.8.40/go.mod h1:SPBBUENXE6ThkEksN5ZavfAhFYll+h+66ZiG6IZQuzo= github.com/pion/sdp/v3 v3.0.16 h1:0dKzYO6gTAvuLaAKQkC02eCPjMIi4NuAr/ibAwrGDCo= @@ -209,16 +209,16 @@ github.com/pion/srtp/v3 v3.0.8 h1:RjRrjcIeQsilPzxvdaElN0CpuQZdMvcl9VZ5UY9suUM= github.com/pion/srtp/v3 v3.0.8/go.mod h1:2Sq6YnDH7/UDCvkSoHSDNDeyBcFgWL0sAVycVbAsXFg= github.com/pion/stun v0.6.1 h1:8lp6YejULeHBF8NmV8e2787BogQhduZugh5PdhDyyN4= github.com/pion/stun v0.6.1/go.mod h1:/hO7APkX4hZKu/D0f2lHzNyvdkTGtIy3NDmLR7kSz/8= -github.com/pion/stun/v3 v3.0.0 h1:4h1gwhWLWuZWOJIJR9s2ferRO+W3zA/b6ijOI6mKzUw= -github.com/pion/stun/v3 v3.0.0/go.mod h1:HvCN8txt8mwi4FBvS3EmDghW6aQJ24T+y+1TKjB5jyU= +github.com/pion/stun/v3 v3.0.1 h1:jx1uUq6BdPihF0yF33Jj2mh+C9p0atY94IkdnW174kA= +github.com/pion/stun/v3 v3.0.1/go.mod h1:RHnvlKFg+qHgoKIqtQWMOJF52wsImCAf/Jh5GjX+4Tw= github.com/pion/transport/v2 v2.2.1/go.mod h1:cXXWavvCnFF6McHTft3DWS9iic2Mftcz1Aq29pGcU5g= github.com/pion/transport/v2 v2.2.4/go.mod h1:q2U/tf9FEfnSBGSW6w5Qp5PFWRLRj3NjLhCCgpRK4p0= github.com/pion/transport/v2 v2.2.10 h1:ucLBLE8nuxiHfvkFKnkDQRYWYfp8ejf4YBOPfaQpw6Q= github.com/pion/transport/v2 v2.2.10/go.mod h1:sq1kSLWs+cHW9E+2fJP95QudkzbK7wscs8yYgQToO5E= github.com/pion/transport/v3 v3.0.8 h1:oI3myyYnTKUSTthu/NZZ8eu2I5sHbxbUNNFW62olaYc= github.com/pion/transport/v3 v3.0.8/go.mod h1:+c2eewC5WJQHiAA46fkMMzoYZSuGzA/7E2FPrOYHctQ= -github.com/pion/turn/v4 v4.1.1 h1:9UnY2HB99tpDyz3cVVZguSxcqkJ1DsTSZ+8TGruh4fc= -github.com/pion/turn/v4 v4.1.1/go.mod h1:2123tHk1O++vmjI5VSD0awT50NywDAq5A2NNNU4Jjs8= +github.com/pion/turn/v4 v4.1.2 h1:Em2svpl6aBFa88dLhxypMUzaLjC79kWZWx8FIov01cc= +github.com/pion/turn/v4 v4.1.2/go.mod h1:ISYWfZYy0Z3tXzRpyYZHTL+U23yFQIspfxogdQ8pn9Y= github.com/pion/webrtc/v4 v4.1.6 h1:srHH2HwvCGwPba25EYJgUzgLqCQoXl1VCUnrGQMSzUw= github.com/pion/webrtc/v4 v4.1.6/go.mod h1:wKecGRlkl3ox/As/MYghJL+b/cVXMEhoPMJWPuGQFhU= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -231,11 +231,11 @@ github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1: github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk= github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE= github.com/prometheus/common v0.0.0-20180801064454-c7de2306084e/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= -github.com/prometheus/common v0.67.1 h1:OTSON1P4DNxzTg4hmKCc37o4ZAZDv0cfXLkOt0oEowI= -github.com/prometheus/common v0.67.1/go.mod h1:RpmT9v35q2Y+lsieQsdOh5sXZ6ajUGC8NjZAmr8vb0Q= +github.com/prometheus/common v0.67.2 h1:PcBAckGFTIHt2+L3I33uNRTlKTplNzFctXcWhPyAEN8= +github.com/prometheus/common v0.67.2/go.mod h1:63W3KZb1JOKgcjlIr64WW/LvFGAqKPj0atm+knVGEko= github.com/prometheus/procfs v0.0.0-20180725123919-05ee40e3a273/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= -github.com/prometheus/procfs v0.17.0 h1:FuLQ+05u4ZI+SS/w9+BWEM2TXiHKsUQ9TADiRH7DuK0= -github.com/prometheus/procfs v0.17.0/go.mod h1:oPQLaDAMRbA+u8H5Pbfq+dl3VDAvHxMUOVhe0wYB2zw= +github.com/prometheus/procfs v0.19.2 h1:zUMhqEW66Ex7OXIiDkll3tl9a1ZdilUOd/F6ZXw4Vws= +github.com/prometheus/procfs v0.19.2/go.mod h1:M0aotyiemPhBCM0z5w87kL22CxfcH05ZpYlu+b4J7mw= github.com/quic-go/qpack v0.5.1 h1:giqksBPnT/HDtZ6VhtFKgoLOWmlyo9Ei6u9PqzIMbhI= github.com/quic-go/qpack v0.5.1/go.mod h1:+PC4XFrEskIVkcLzpEkbLqq1uCoxPhQuvK5rH1ZgaEg= github.com/quic-go/quic-go v0.55.0 h1:zccPQIqYCXDt5NmcEabyYvOnomjs8Tlwl7tISjJh9Mk= @@ -323,8 +323,8 @@ golang.org/x/crypto v0.18.0/go.mod h1:R0j02AL6hcrfOiy9T4ZYp/rcWeMxM3L6QYxlOuEG1m golang.org/x/crypto v0.43.0 h1:dduJYIi3A3KOfdGOHX8AVZ/jGiyPa3IbBozJ5kNuE04= golang.org/x/crypto v0.43.0/go.mod h1:BFbav4mRNlXJL4wNeejLpWxB7wMbc79PdRGhWKncxR0= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20251017212417-90e834f514db h1:by6IehL4BH5k3e3SJmcoNbOobMey2SLpAF79iPOEBvw= -golang.org/x/exp v0.0.0-20251017212417-90e834f514db/go.mod h1:j/pmGrbnkbPtQfxEe5D0VQhZC6qKbfKifgD0oM7sR70= +golang.org/x/exp v0.0.0-20251023183803-a4bb9ffd2546 h1:mgKeJMpvi0yx/sU5GsxQ7p6s2wtOnGAHZWCHUM4KGzY= +golang.org/x/exp v0.0.0-20251023183803-a4bb9ffd2546/go.mod h1:j/pmGrbnkbPtQfxEe5D0VQhZC6qKbfKifgD0oM7sR70= golang.org/x/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= @@ -396,8 +396,8 @@ golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.37.0 h1:fdNQudmxPjkdUTPnLn5mdQv7Zwvbvpaxqs831goi9kQ= golang.org/x/sys v0.37.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= -golang.org/x/telemetry v0.0.0-20251014153721-24f779f6aaef h1:5xFtU4tmJMJSxSeDlr1dgBff2tDXrq0laLdS1EA3LYw= -golang.org/x/telemetry v0.0.0-20251014153721-24f779f6aaef/go.mod h1:Pi4ztBfryZoJEkyFTI5/Ocsu2jXyDr6iSdgJiYE/uwE= +golang.org/x/telemetry v0.0.0-20251028164327-d7a2859f34e8 h1:DwMAzqwLj2rVin75cRFh1kfhwQY3hyHrU1oCEDZXPmQ= +golang.org/x/telemetry v0.0.0-20251028164327-d7a2859f34e8/go.mod h1:Pi4ztBfryZoJEkyFTI5/Ocsu2jXyDr6iSdgJiYE/uwE= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= diff --git a/packages/relay/internal/core/core.go b/packages/relay/internal/core/core.go index 6288d5f6..09b26bb2 100644 --- a/packages/relay/internal/core/core.go +++ b/packages/relay/internal/core/core.go @@ -10,7 +10,6 @@ import ( "os" "relay/internal/common" "relay/internal/shared" - "time" "github.com/libp2p/go-libp2p" pubsub "github.com/libp2p/go-libp2p-pubsub" @@ -38,16 +37,6 @@ var globalRelay *Relay // -- Structs -- -// ClientSession tracks browser client connections -type ClientSession struct { - PeerID peer.ID - SessionID string - RoomName string - ConnectedAt time.Time - LastActivity time.Time - ControllerSlots []int32 // Track which controller slots this client owns -} - // Relay structure enhanced with metrics and state type Relay struct { *PeerInfo @@ -59,7 +48,6 @@ type Relay struct { // Local LocalRooms *common.SafeMap[ulid.ULID, *shared.Room] // room ID -> local Room struct (hosted by this relay) LocalMeshConnections *common.SafeMap[peer.ID, *webrtc.PeerConnection] // peer ID -> PeerConnection (connected to this relay) - ClientSessions *common.SafeMap[peer.ID, *ClientSession] // peer ID -> ClientSession // Protocols ProtocolRegistry @@ -156,7 +144,6 @@ func NewRelay(ctx context.Context, port int, identityKey crypto.PrivKey) (*Relay PingService: pingSvc, LocalRooms: common.NewSafeMap[ulid.ULID, *shared.Room](), LocalMeshConnections: common.NewSafeMap[peer.ID, *webrtc.PeerConnection](), - ClientSessions: common.NewSafeMap[peer.ID, *ClientSession](), } // Add network notifier after relay is initialized diff --git a/packages/relay/internal/core/protocol_stream.go b/packages/relay/internal/core/protocol_stream.go index 19f3934f..e7566ed9 100644 --- a/packages/relay/internal/core/protocol_stream.go +++ b/packages/relay/internal/core/protocol_stream.go @@ -11,7 +11,6 @@ import ( "relay/internal/common" "relay/internal/connections" "relay/internal/shared" - "time" gen "relay/internal/proto" @@ -111,16 +110,7 @@ func (sp *StreamProtocol) handleStreamRequest(stream network.Stream) { sessionID = ulid.String() } - session := &ClientSession{ - PeerID: stream.Conn().RemotePeer(), - SessionID: sessionID, - RoomName: reqMsg.RoomName, - ConnectedAt: time.Now(), - LastActivity: time.Now(), - } - sp.relay.ClientSessions.Set(stream.Conn().RemotePeer(), session) - - slog.Info("Client session established", "peer", session.PeerID, "session", sessionID, "room", reqMsg.RoomName) + slog.Info("Client session requested room stream", "session", sessionID, "room", reqMsg.RoomName) // Send session ID back to client sesMsg, err := common.CreateMessage( @@ -177,7 +167,7 @@ func (sp *StreamProtocol) handleStreamRequest(stream network.Stream) { // Create participant for this viewer participant, err := shared.NewParticipant( - "", + sessionID, stream.Conn().RemotePeer(), ) if err != nil { @@ -185,11 +175,6 @@ func (sp *StreamProtocol) handleStreamRequest(stream network.Stream) { continue } - // If this is a client session, link it - if session, ok := sp.relay.ClientSessions.Get(stream.Conn().RemotePeer()); ok { - participant.SessionID = session.SessionID - } - // Assign peer connection participant.PeerConnection = pc @@ -265,57 +250,9 @@ func (sp *StreamProtocol) handleStreamRequest(stream network.Stream) { // Track controller input separately ndc.RegisterMessageCallback("controllerInput", func(data []byte) { // Parse the message to track controller slots for client sessions - var msgWrapper gen.ProtoMessage - if err = proto.Unmarshal(data, &msgWrapper); err != nil { + var controllerMsgWrapper gen.ProtoMessage + if err = proto.Unmarshal(data, &controllerMsgWrapper); err != nil { slog.Error("Failed to unmarshal controller input", "err", err) - } else if msgWrapper.Payload != nil { - // Get the peer ID for this connection - peerID := stream.Conn().RemotePeer() - - // Check if it's a controller attach with assigned slot - if attach := msgWrapper.GetControllerAttach(); attach != nil && attach.SessionSlot >= 0 { - if session, ok := sp.relay.ClientSessions.Get(peerID); ok { - // Check if slot already tracked - hasSlot := false - for _, slot := range session.ControllerSlots { - if slot == attach.SessionSlot { - hasSlot = true - break - } - } - if !hasSlot { - session.ControllerSlots = append(session.ControllerSlots, attach.SessionSlot) - session.LastActivity = time.Now() - slog.Info("Controller slot assigned to client session", - "session", session.SessionID, - "slot", attach.SessionSlot, - "total_slots", len(session.ControllerSlots)) - } - } - } - - // Check if it's a controller detach - if detach := msgWrapper.GetControllerDetach(); detach != nil && detach.SessionSlot >= 0 { - if session, ok := sp.relay.ClientSessions.Get(peerID); ok { - newSlots := make([]int32, 0, len(session.ControllerSlots)) - for _, slot := range session.ControllerSlots { - if slot != detach.SessionSlot { - newSlots = append(newSlots, slot) - } - } - session.ControllerSlots = newSlots - session.LastActivity = time.Now() - slog.Info("Controller slot removed from client session", - "session", session.SessionID, - "slot", detach.SessionSlot, - "remaining_slots", len(session.ControllerSlots)) - } - } - - // Update last activity on any controller input - if session, ok := sp.relay.ClientSessions.Get(peerID); ok { - session.LastActivity = time.Now() - } } // Forward to upstream room @@ -609,7 +546,12 @@ func (sp *StreamProtocol) handleStreamPush(stream network.Stream) { roomMap.Range(func(peerID peer.ID, conn *StreamConnection) bool { if conn.ndc != nil { if err = conn.ndc.SendBinary(data); err != nil { - slog.Error("Failed to forward controller input from pushed stream to viewer", "room", room.Name, "peer", peerID, "err", err) + if errors.Is(err, io.ErrClosedPipe) { + slog.Warn("Failed to forward controller input to viewer, treating as disconnected", "err", err) + sp.relay.onPeerDisconnected(peerID) + } else { + slog.Error("Failed to forward controller input from pushed stream to viewer", "room", room.Name, "peer", peerID, "err", err) + } } } return true diff --git a/packages/relay/internal/core/state.go b/packages/relay/internal/core/state.go index 9323eadc..46bfa69b 100644 --- a/packages/relay/internal/core/state.go +++ b/packages/relay/internal/core/state.go @@ -5,14 +5,9 @@ import ( "encoding/json" "errors" "log/slog" - "relay/internal/common" "relay/internal/shared" "time" - gen "relay/internal/proto" - - "google.golang.org/protobuf/proto" - pubsub "github.com/libp2p/go-libp2p-pubsub" "github.com/libp2p/go-libp2p/core/network" "github.com/libp2p/go-libp2p/core/peer" @@ -134,46 +129,6 @@ func (r *Relay) onPeerConnected(peerID peer.ID) { // onPeerDisconnected marks a peer as disconnected in our status view and removes latency info func (r *Relay) onPeerDisconnected(peerID peer.ID) { - // Check if this was a client session disconnect - if session, ok := r.ClientSessions.Get(peerID); ok { - slog.Info("Client session disconnected", - "peer", peerID, - "session", session.SessionID, - "room", session.RoomName, - "controller_slots", session.ControllerSlots) - - // Send cleanup message to nestri-server if client had active controllers - if len(session.ControllerSlots) > 0 { - room := r.GetRoomByName(session.RoomName) - if room != nil && room.DataChannel != nil { - // Create disconnect notification - disconnectMsg, err := common.CreateMessage(&gen.ProtoClientDisconnected{ - SessionId: session.SessionID, - ControllerSlots: session.ControllerSlots, - }, "client-disconnected", nil) - if err != nil { - slog.Error("Failed to create client disconnect message", "err", err) - } - - disMarshal, err := proto.Marshal(disconnectMsg) - if err != nil { - slog.Error("Failed to marshal client disconnect message", "err", err) - } else { - if err = room.DataChannel.SendBinary(disMarshal); err != nil { - slog.Error("Failed to send client disconnect notification", "err", err) - } else { - slog.Info("Sent controller cleanup notification to nestri-server", - "session", session.SessionID, - "slots", session.ControllerSlots) - } - } - } - } - - r.ClientSessions.Delete(peerID) - return - } - // Relay peer disconnect handling slog.Info("Mesh peer disconnected, deleting from local peer map", "peer", peerID) if r.Peers.Has(peerID) { diff --git a/packages/relay/internal/proto/messages.pb.go b/packages/relay/internal/proto/messages.pb.go index fb9a2f6e..19ec59e5 100644 --- a/packages/relay/internal/proto/messages.pb.go +++ b/packages/relay/internal/proto/messages.pb.go @@ -87,11 +87,8 @@ type ProtoMessage struct { // *ProtoMessage_KeyUp // *ProtoMessage_ControllerAttach // *ProtoMessage_ControllerDetach - // *ProtoMessage_ControllerButton - // *ProtoMessage_ControllerTrigger - // *ProtoMessage_ControllerStick - // *ProtoMessage_ControllerAxis // *ProtoMessage_ControllerRumble + // *ProtoMessage_ControllerStateBatch // *ProtoMessage_Ice // *ProtoMessage_Sdp // *ProtoMessage_Raw @@ -228,46 +225,19 @@ func (x *ProtoMessage) GetControllerDetach() *ProtoControllerDetach { return nil } -func (x *ProtoMessage) GetControllerButton() *ProtoControllerButton { - if x != nil { - if x, ok := x.Payload.(*ProtoMessage_ControllerButton); ok { - return x.ControllerButton - } - } - return nil -} - -func (x *ProtoMessage) GetControllerTrigger() *ProtoControllerTrigger { - if x != nil { - if x, ok := x.Payload.(*ProtoMessage_ControllerTrigger); ok { - return x.ControllerTrigger - } - } - return nil -} - -func (x *ProtoMessage) GetControllerStick() *ProtoControllerStick { - if x != nil { - if x, ok := x.Payload.(*ProtoMessage_ControllerStick); ok { - return x.ControllerStick - } - } - return nil -} - -func (x *ProtoMessage) GetControllerAxis() *ProtoControllerAxis { +func (x *ProtoMessage) GetControllerRumble() *ProtoControllerRumble { if x != nil { - if x, ok := x.Payload.(*ProtoMessage_ControllerAxis); ok { - return x.ControllerAxis + if x, ok := x.Payload.(*ProtoMessage_ControllerRumble); ok { + return x.ControllerRumble } } return nil } -func (x *ProtoMessage) GetControllerRumble() *ProtoControllerRumble { +func (x *ProtoMessage) GetControllerStateBatch() *ProtoControllerStateBatch { if x != nil { - if x, ok := x.Payload.(*ProtoMessage_ControllerRumble); ok { - return x.ControllerRumble + if x, ok := x.Payload.(*ProtoMessage_ControllerStateBatch); ok { + return x.ControllerStateBatch } } return nil @@ -361,6 +331,7 @@ type ProtoMessage_KeyUp struct { } type ProtoMessage_ControllerAttach struct { + // Controller input types ControllerAttach *ProtoControllerAttach `protobuf:"bytes,9,opt,name=controller_attach,json=controllerAttach,proto3,oneof"` } @@ -368,24 +339,12 @@ type ProtoMessage_ControllerDetach struct { ControllerDetach *ProtoControllerDetach `protobuf:"bytes,10,opt,name=controller_detach,json=controllerDetach,proto3,oneof"` } -type ProtoMessage_ControllerButton struct { - ControllerButton *ProtoControllerButton `protobuf:"bytes,11,opt,name=controller_button,json=controllerButton,proto3,oneof"` -} - -type ProtoMessage_ControllerTrigger struct { - ControllerTrigger *ProtoControllerTrigger `protobuf:"bytes,12,opt,name=controller_trigger,json=controllerTrigger,proto3,oneof"` -} - -type ProtoMessage_ControllerStick struct { - ControllerStick *ProtoControllerStick `protobuf:"bytes,13,opt,name=controller_stick,json=controllerStick,proto3,oneof"` -} - -type ProtoMessage_ControllerAxis struct { - ControllerAxis *ProtoControllerAxis `protobuf:"bytes,14,opt,name=controller_axis,json=controllerAxis,proto3,oneof"` +type ProtoMessage_ControllerRumble struct { + ControllerRumble *ProtoControllerRumble `protobuf:"bytes,11,opt,name=controller_rumble,json=controllerRumble,proto3,oneof"` } -type ProtoMessage_ControllerRumble struct { - ControllerRumble *ProtoControllerRumble `protobuf:"bytes,15,opt,name=controller_rumble,json=controllerRumble,proto3,oneof"` +type ProtoMessage_ControllerStateBatch struct { + ControllerStateBatch *ProtoControllerStateBatch `protobuf:"bytes,12,opt,name=controller_state_batch,json=controllerStateBatch,proto3,oneof"` } type ProtoMessage_Ice struct { @@ -431,16 +390,10 @@ func (*ProtoMessage_ControllerAttach) isProtoMessage_Payload() {} func (*ProtoMessage_ControllerDetach) isProtoMessage_Payload() {} -func (*ProtoMessage_ControllerButton) isProtoMessage_Payload() {} - -func (*ProtoMessage_ControllerTrigger) isProtoMessage_Payload() {} - -func (*ProtoMessage_ControllerStick) isProtoMessage_Payload() {} - -func (*ProtoMessage_ControllerAxis) isProtoMessage_Payload() {} - func (*ProtoMessage_ControllerRumble) isProtoMessage_Payload() {} +func (*ProtoMessage_ControllerStateBatch) isProtoMessage_Payload() {} + func (*ProtoMessage_Ice) isProtoMessage_Payload() {} func (*ProtoMessage_Sdp) isProtoMessage_Payload() {} @@ -460,8 +413,7 @@ const file_messages_proto_rawDesc = "" + "\x0emessages.proto\x12\x05proto\x1a\vtypes.proto\x1a\x15latency_tracker.proto\"k\n" + "\x10ProtoMessageBase\x12!\n" + "\fpayload_type\x18\x01 \x01(\tR\vpayloadType\x124\n" + - "\alatency\x18\x02 \x01(\v2\x1a.proto.ProtoLatencyTrackerR\alatency\"\xef\n" + - "\n" + + "\alatency\x18\x02 \x01(\v2\x1a.proto.ProtoLatencyTrackerR\alatency\"\x9b\t\n" + "\fProtoMessage\x12:\n" + "\fmessage_base\x18\x01 \x01(\v2\x17.proto.ProtoMessageBaseR\vmessageBase\x126\n" + "\n" + @@ -477,11 +429,8 @@ const file_messages_proto_rawDesc = "" + "\x11controller_attach\x18\t \x01(\v2\x1c.proto.ProtoControllerAttachH\x00R\x10controllerAttach\x12K\n" + "\x11controller_detach\x18\n" + " \x01(\v2\x1c.proto.ProtoControllerDetachH\x00R\x10controllerDetach\x12K\n" + - "\x11controller_button\x18\v \x01(\v2\x1c.proto.ProtoControllerButtonH\x00R\x10controllerButton\x12N\n" + - "\x12controller_trigger\x18\f \x01(\v2\x1d.proto.ProtoControllerTriggerH\x00R\x11controllerTrigger\x12H\n" + - "\x10controller_stick\x18\r \x01(\v2\x1b.proto.ProtoControllerStickH\x00R\x0fcontrollerStick\x12E\n" + - "\x0fcontroller_axis\x18\x0e \x01(\v2\x1a.proto.ProtoControllerAxisH\x00R\x0econtrollerAxis\x12K\n" + - "\x11controller_rumble\x18\x0f \x01(\v2\x1c.proto.ProtoControllerRumbleH\x00R\x10controllerRumble\x12#\n" + + "\x11controller_rumble\x18\v \x01(\v2\x1c.proto.ProtoControllerRumbleH\x00R\x10controllerRumble\x12X\n" + + "\x16controller_state_batch\x18\f \x01(\v2 .proto.ProtoControllerStateBatchH\x00R\x14controllerStateBatch\x12#\n" + "\x03ice\x18\x14 \x01(\v2\x0f.proto.ProtoICEH\x00R\x03ice\x12#\n" + "\x03sdp\x18\x15 \x01(\v2\x0f.proto.ProtoSDPH\x00R\x03sdp\x12#\n" + "\x03raw\x18\x16 \x01(\v2\x0f.proto.ProtoRawH\x00R\x03raw\x12b\n" + @@ -516,17 +465,14 @@ var file_messages_proto_goTypes = []any{ (*ProtoKeyUp)(nil), // 9: proto.ProtoKeyUp (*ProtoControllerAttach)(nil), // 10: proto.ProtoControllerAttach (*ProtoControllerDetach)(nil), // 11: proto.ProtoControllerDetach - (*ProtoControllerButton)(nil), // 12: proto.ProtoControllerButton - (*ProtoControllerTrigger)(nil), // 13: proto.ProtoControllerTrigger - (*ProtoControllerStick)(nil), // 14: proto.ProtoControllerStick - (*ProtoControllerAxis)(nil), // 15: proto.ProtoControllerAxis - (*ProtoControllerRumble)(nil), // 16: proto.ProtoControllerRumble - (*ProtoICE)(nil), // 17: proto.ProtoICE - (*ProtoSDP)(nil), // 18: proto.ProtoSDP - (*ProtoRaw)(nil), // 19: proto.ProtoRaw - (*ProtoClientRequestRoomStream)(nil), // 20: proto.ProtoClientRequestRoomStream - (*ProtoClientDisconnected)(nil), // 21: proto.ProtoClientDisconnected - (*ProtoServerPushStream)(nil), // 22: proto.ProtoServerPushStream + (*ProtoControllerRumble)(nil), // 12: proto.ProtoControllerRumble + (*ProtoControllerStateBatch)(nil), // 13: proto.ProtoControllerStateBatch + (*ProtoICE)(nil), // 14: proto.ProtoICE + (*ProtoSDP)(nil), // 15: proto.ProtoSDP + (*ProtoRaw)(nil), // 16: proto.ProtoRaw + (*ProtoClientRequestRoomStream)(nil), // 17: proto.ProtoClientRequestRoomStream + (*ProtoClientDisconnected)(nil), // 18: proto.ProtoClientDisconnected + (*ProtoServerPushStream)(nil), // 19: proto.ProtoServerPushStream } var file_messages_proto_depIdxs = []int32{ 2, // 0: proto.ProtoMessageBase.latency:type_name -> proto.ProtoLatencyTracker @@ -540,22 +486,19 @@ var file_messages_proto_depIdxs = []int32{ 9, // 8: proto.ProtoMessage.key_up:type_name -> proto.ProtoKeyUp 10, // 9: proto.ProtoMessage.controller_attach:type_name -> proto.ProtoControllerAttach 11, // 10: proto.ProtoMessage.controller_detach:type_name -> proto.ProtoControllerDetach - 12, // 11: proto.ProtoMessage.controller_button:type_name -> proto.ProtoControllerButton - 13, // 12: proto.ProtoMessage.controller_trigger:type_name -> proto.ProtoControllerTrigger - 14, // 13: proto.ProtoMessage.controller_stick:type_name -> proto.ProtoControllerStick - 15, // 14: proto.ProtoMessage.controller_axis:type_name -> proto.ProtoControllerAxis - 16, // 15: proto.ProtoMessage.controller_rumble:type_name -> proto.ProtoControllerRumble - 17, // 16: proto.ProtoMessage.ice:type_name -> proto.ProtoICE - 18, // 17: proto.ProtoMessage.sdp:type_name -> proto.ProtoSDP - 19, // 18: proto.ProtoMessage.raw:type_name -> proto.ProtoRaw - 20, // 19: proto.ProtoMessage.client_request_room_stream:type_name -> proto.ProtoClientRequestRoomStream - 21, // 20: proto.ProtoMessage.client_disconnected:type_name -> proto.ProtoClientDisconnected - 22, // 21: proto.ProtoMessage.server_push_stream:type_name -> proto.ProtoServerPushStream - 22, // [22:22] is the sub-list for method output_type - 22, // [22:22] is the sub-list for method input_type - 22, // [22:22] is the sub-list for extension type_name - 22, // [22:22] is the sub-list for extension extendee - 0, // [0:22] is the sub-list for field type_name + 12, // 11: proto.ProtoMessage.controller_rumble:type_name -> proto.ProtoControllerRumble + 13, // 12: proto.ProtoMessage.controller_state_batch:type_name -> proto.ProtoControllerStateBatch + 14, // 13: proto.ProtoMessage.ice:type_name -> proto.ProtoICE + 15, // 14: proto.ProtoMessage.sdp:type_name -> proto.ProtoSDP + 16, // 15: proto.ProtoMessage.raw:type_name -> proto.ProtoRaw + 17, // 16: proto.ProtoMessage.client_request_room_stream:type_name -> proto.ProtoClientRequestRoomStream + 18, // 17: proto.ProtoMessage.client_disconnected:type_name -> proto.ProtoClientDisconnected + 19, // 18: proto.ProtoMessage.server_push_stream:type_name -> proto.ProtoServerPushStream + 19, // [19:19] is the sub-list for method output_type + 19, // [19:19] is the sub-list for method input_type + 19, // [19:19] is the sub-list for extension type_name + 19, // [19:19] is the sub-list for extension extendee + 0, // [0:19] is the sub-list for field type_name } func init() { file_messages_proto_init() } @@ -575,11 +518,8 @@ func file_messages_proto_init() { (*ProtoMessage_KeyUp)(nil), (*ProtoMessage_ControllerAttach)(nil), (*ProtoMessage_ControllerDetach)(nil), - (*ProtoMessage_ControllerButton)(nil), - (*ProtoMessage_ControllerTrigger)(nil), - (*ProtoMessage_ControllerStick)(nil), - (*ProtoMessage_ControllerAxis)(nil), (*ProtoMessage_ControllerRumble)(nil), + (*ProtoMessage_ControllerStateBatch)(nil), (*ProtoMessage_Ice)(nil), (*ProtoMessage_Sdp)(nil), (*ProtoMessage_Raw)(nil), diff --git a/packages/relay/internal/proto/types.pb.go b/packages/relay/internal/proto/types.pb.go index e71fa071..917530d8 100644 --- a/packages/relay/internal/proto/types.pb.go +++ b/packages/relay/internal/proto/types.pb.go @@ -21,6 +21,52 @@ const ( _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) ) +type ProtoControllerStateBatch_UpdateType int32 + +const ( + ProtoControllerStateBatch_FULL_STATE ProtoControllerStateBatch_UpdateType = 0 // Complete controller state + ProtoControllerStateBatch_DELTA ProtoControllerStateBatch_UpdateType = 1 // Only changed fields +) + +// Enum value maps for ProtoControllerStateBatch_UpdateType. +var ( + ProtoControllerStateBatch_UpdateType_name = map[int32]string{ + 0: "FULL_STATE", + 1: "DELTA", + } + ProtoControllerStateBatch_UpdateType_value = map[string]int32{ + "FULL_STATE": 0, + "DELTA": 1, + } +) + +func (x ProtoControllerStateBatch_UpdateType) Enum() *ProtoControllerStateBatch_UpdateType { + p := new(ProtoControllerStateBatch_UpdateType) + *p = x + return p +} + +func (x ProtoControllerStateBatch_UpdateType) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (ProtoControllerStateBatch_UpdateType) Descriptor() protoreflect.EnumDescriptor { + return file_types_proto_enumTypes[0].Descriptor() +} + +func (ProtoControllerStateBatch_UpdateType) Type() protoreflect.EnumType { + return &file_types_proto_enumTypes[0] +} + +func (x ProtoControllerStateBatch_UpdateType) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Use ProtoControllerStateBatch_UpdateType.Descriptor instead. +func (ProtoControllerStateBatch_UpdateType) EnumDescriptor() ([]byte, []int) { + return file_types_proto_rawDescGZIP(), []int{10, 0} +} + // MouseMove message type ProtoMouseMove struct { state protoimpl.MessageState `protogen:"open.v1"` @@ -474,31 +520,32 @@ func (x *ProtoControllerDetach) GetSessionId() string { return "" } -// ControllerButton message -type ProtoControllerButton struct { +// ControllerRumble message +type ProtoControllerRumble struct { state protoimpl.MessageState `protogen:"open.v1"` - SessionSlot int32 `protobuf:"varint,1,opt,name=session_slot,json=sessionSlot,proto3" json:"session_slot,omitempty"` // Session specific slot number (0-3) - SessionId string `protobuf:"bytes,2,opt,name=session_id,json=sessionId,proto3" json:"session_id,omitempty"` // Session ID of the client - Button int32 `protobuf:"varint,3,opt,name=button,proto3" json:"button,omitempty"` // Button code (linux input event code) - Pressed bool `protobuf:"varint,4,opt,name=pressed,proto3" json:"pressed,omitempty"` // true if pressed, false if released + SessionSlot int32 `protobuf:"varint,1,opt,name=session_slot,json=sessionSlot,proto3" json:"session_slot,omitempty"` // Session specific slot number (0-3) + SessionId string `protobuf:"bytes,2,opt,name=session_id,json=sessionId,proto3" json:"session_id,omitempty"` // Session ID of the client + LowFrequency int32 `protobuf:"varint,3,opt,name=low_frequency,json=lowFrequency,proto3" json:"low_frequency,omitempty"` // Low frequency rumble (0-65535) + HighFrequency int32 `protobuf:"varint,4,opt,name=high_frequency,json=highFrequency,proto3" json:"high_frequency,omitempty"` // High frequency rumble (0-65535) + Duration int32 `protobuf:"varint,5,opt,name=duration,proto3" json:"duration,omitempty"` // Duration in milliseconds unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } -func (x *ProtoControllerButton) Reset() { - *x = ProtoControllerButton{} +func (x *ProtoControllerRumble) Reset() { + *x = ProtoControllerRumble{} mi := &file_types_proto_msgTypes[9] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } -func (x *ProtoControllerButton) String() string { +func (x *ProtoControllerRumble) String() string { return protoimpl.X.MessageStringOf(x) } -func (*ProtoControllerButton) ProtoMessage() {} +func (*ProtoControllerRumble) ProtoMessage() {} -func (x *ProtoControllerButton) ProtoReflect() protoreflect.Message { +func (x *ProtoControllerRumble) ProtoReflect() protoreflect.Message { mi := &file_types_proto_msgTypes[9] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) @@ -510,135 +557,87 @@ func (x *ProtoControllerButton) ProtoReflect() protoreflect.Message { return mi.MessageOf(x) } -// Deprecated: Use ProtoControllerButton.ProtoReflect.Descriptor instead. -func (*ProtoControllerButton) Descriptor() ([]byte, []int) { +// Deprecated: Use ProtoControllerRumble.ProtoReflect.Descriptor instead. +func (*ProtoControllerRumble) Descriptor() ([]byte, []int) { return file_types_proto_rawDescGZIP(), []int{9} } -func (x *ProtoControllerButton) GetSessionSlot() int32 { +func (x *ProtoControllerRumble) GetSessionSlot() int32 { if x != nil { return x.SessionSlot } return 0 } -func (x *ProtoControllerButton) GetSessionId() string { +func (x *ProtoControllerRumble) GetSessionId() string { if x != nil { return x.SessionId } return "" } -func (x *ProtoControllerButton) GetButton() int32 { - if x != nil { - return x.Button - } - return 0 -} - -func (x *ProtoControllerButton) GetPressed() bool { - if x != nil { - return x.Pressed - } - return false -} - -// ControllerTriggers message -type ProtoControllerTrigger struct { - state protoimpl.MessageState `protogen:"open.v1"` - SessionSlot int32 `protobuf:"varint,1,opt,name=session_slot,json=sessionSlot,proto3" json:"session_slot,omitempty"` // Session specific slot number (0-3) - SessionId string `protobuf:"bytes,2,opt,name=session_id,json=sessionId,proto3" json:"session_id,omitempty"` // Session ID of the client - Trigger int32 `protobuf:"varint,3,opt,name=trigger,proto3" json:"trigger,omitempty"` // Trigger number (0 for left, 1 for right) - Value int32 `protobuf:"varint,4,opt,name=value,proto3" json:"value,omitempty"` // trigger value (-32768 to 32767) - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache -} - -func (x *ProtoControllerTrigger) Reset() { - *x = ProtoControllerTrigger{} - mi := &file_types_proto_msgTypes[10] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) -} - -func (x *ProtoControllerTrigger) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*ProtoControllerTrigger) ProtoMessage() {} - -func (x *ProtoControllerTrigger) ProtoReflect() protoreflect.Message { - mi := &file_types_proto_msgTypes[10] - if x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use ProtoControllerTrigger.ProtoReflect.Descriptor instead. -func (*ProtoControllerTrigger) Descriptor() ([]byte, []int) { - return file_types_proto_rawDescGZIP(), []int{10} -} - -func (x *ProtoControllerTrigger) GetSessionSlot() int32 { +func (x *ProtoControllerRumble) GetLowFrequency() int32 { if x != nil { - return x.SessionSlot + return x.LowFrequency } return 0 } -func (x *ProtoControllerTrigger) GetSessionId() string { - if x != nil { - return x.SessionId - } - return "" -} - -func (x *ProtoControllerTrigger) GetTrigger() int32 { +func (x *ProtoControllerRumble) GetHighFrequency() int32 { if x != nil { - return x.Trigger + return x.HighFrequency } return 0 } -func (x *ProtoControllerTrigger) GetValue() int32 { +func (x *ProtoControllerRumble) GetDuration() int32 { if x != nil { - return x.Value + return x.Duration } return 0 } -// ControllerSticks message -type ProtoControllerStick struct { - state protoimpl.MessageState `protogen:"open.v1"` - SessionSlot int32 `protobuf:"varint,1,opt,name=session_slot,json=sessionSlot,proto3" json:"session_slot,omitempty"` // Session specific slot number (0-3) - SessionId string `protobuf:"bytes,2,opt,name=session_id,json=sessionId,proto3" json:"session_id,omitempty"` // Session ID of the client - Stick int32 `protobuf:"varint,3,opt,name=stick,proto3" json:"stick,omitempty"` // Stick number (0 for left, 1 for right) - X int32 `protobuf:"varint,4,opt,name=x,proto3" json:"x,omitempty"` // X axis value (-32768 to 32767) - Y int32 `protobuf:"varint,5,opt,name=y,proto3" json:"y,omitempty"` // Y axis value (-32768 to 32767) +// ControllerStateBatch - single message containing full or partial controller state +type ProtoControllerStateBatch struct { + state protoimpl.MessageState `protogen:"open.v1"` + SessionSlot int32 `protobuf:"varint,1,opt,name=session_slot,json=sessionSlot,proto3" json:"session_slot,omitempty"` // Session specific slot number (0-3) + SessionId string `protobuf:"bytes,2,opt,name=session_id,json=sessionId,proto3" json:"session_id,omitempty"` // Session ID of the client + UpdateType ProtoControllerStateBatch_UpdateType `protobuf:"varint,3,opt,name=update_type,json=updateType,proto3,enum=proto.ProtoControllerStateBatch_UpdateType" json:"update_type,omitempty"` + // Sequence number for packet loss detection + Sequence uint32 `protobuf:"varint,4,opt,name=sequence,proto3" json:"sequence,omitempty"` + // Button state map (Linux event codes) + ButtonChangedMask map[int32]bool `protobuf:"bytes,5,rep,name=button_changed_mask,json=buttonChangedMask,proto3" json:"button_changed_mask,omitempty" protobuf_key:"varint,1,opt,name=key" protobuf_val:"varint,2,opt,name=value"` + // Analog inputs + LeftStickX *int32 `protobuf:"varint,6,opt,name=left_stick_x,json=leftStickX,proto3,oneof" json:"left_stick_x,omitempty"` // -32768 to 32767 + LeftStickY *int32 `protobuf:"varint,7,opt,name=left_stick_y,json=leftStickY,proto3,oneof" json:"left_stick_y,omitempty"` // -32768 to 32767 + RightStickX *int32 `protobuf:"varint,8,opt,name=right_stick_x,json=rightStickX,proto3,oneof" json:"right_stick_x,omitempty"` // -32768 to 32767 + RightStickY *int32 `protobuf:"varint,9,opt,name=right_stick_y,json=rightStickY,proto3,oneof" json:"right_stick_y,omitempty"` // -32768 to 32767 + LeftTrigger *int32 `protobuf:"varint,10,opt,name=left_trigger,json=leftTrigger,proto3,oneof" json:"left_trigger,omitempty"` // -32768 to 32767 + RightTrigger *int32 `protobuf:"varint,11,opt,name=right_trigger,json=rightTrigger,proto3,oneof" json:"right_trigger,omitempty"` // -32768 to 32767 + DpadX *int32 `protobuf:"varint,12,opt,name=dpad_x,json=dpadX,proto3,oneof" json:"dpad_x,omitempty"` // -1, 0, or 1 + DpadY *int32 `protobuf:"varint,13,opt,name=dpad_y,json=dpadY,proto3,oneof" json:"dpad_y,omitempty"` // -1, 0, or 1 + // Bitmask indicating which fields have changed + // Bit 0: button_changed_mask, Bit 1: left_stick_x, Bit 2: left_stick_y, etc. + ChangedFields *uint32 `protobuf:"varint,14,opt,name=changed_fields,json=changedFields,proto3,oneof" json:"changed_fields,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } -func (x *ProtoControllerStick) Reset() { - *x = ProtoControllerStick{} - mi := &file_types_proto_msgTypes[11] +func (x *ProtoControllerStateBatch) Reset() { + *x = ProtoControllerStateBatch{} + mi := &file_types_proto_msgTypes[10] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } -func (x *ProtoControllerStick) String() string { +func (x *ProtoControllerStateBatch) String() string { return protoimpl.X.MessageStringOf(x) } -func (*ProtoControllerStick) ProtoMessage() {} +func (*ProtoControllerStateBatch) ProtoMessage() {} -func (x *ProtoControllerStick) ProtoReflect() protoreflect.Message { - mi := &file_types_proto_msgTypes[11] +func (x *ProtoControllerStateBatch) ProtoReflect() protoreflect.Message { + mi := &file_types_proto_msgTypes[10] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -649,188 +648,105 @@ func (x *ProtoControllerStick) ProtoReflect() protoreflect.Message { return mi.MessageOf(x) } -// Deprecated: Use ProtoControllerStick.ProtoReflect.Descriptor instead. -func (*ProtoControllerStick) Descriptor() ([]byte, []int) { - return file_types_proto_rawDescGZIP(), []int{11} +// Deprecated: Use ProtoControllerStateBatch.ProtoReflect.Descriptor instead. +func (*ProtoControllerStateBatch) Descriptor() ([]byte, []int) { + return file_types_proto_rawDescGZIP(), []int{10} } -func (x *ProtoControllerStick) GetSessionSlot() int32 { +func (x *ProtoControllerStateBatch) GetSessionSlot() int32 { if x != nil { return x.SessionSlot } return 0 } -func (x *ProtoControllerStick) GetSessionId() string { +func (x *ProtoControllerStateBatch) GetSessionId() string { if x != nil { return x.SessionId } return "" } -func (x *ProtoControllerStick) GetStick() int32 { +func (x *ProtoControllerStateBatch) GetUpdateType() ProtoControllerStateBatch_UpdateType { if x != nil { - return x.Stick + return x.UpdateType } - return 0 + return ProtoControllerStateBatch_FULL_STATE } -func (x *ProtoControllerStick) GetX() int32 { +func (x *ProtoControllerStateBatch) GetSequence() uint32 { if x != nil { - return x.X + return x.Sequence } return 0 } -func (x *ProtoControllerStick) GetY() int32 { +func (x *ProtoControllerStateBatch) GetButtonChangedMask() map[int32]bool { if x != nil { - return x.Y + return x.ButtonChangedMask } - return 0 -} - -// ControllerAxis message -type ProtoControllerAxis struct { - state protoimpl.MessageState `protogen:"open.v1"` - SessionSlot int32 `protobuf:"varint,1,opt,name=session_slot,json=sessionSlot,proto3" json:"session_slot,omitempty"` // Session specific slot number (0-3) - SessionId string `protobuf:"bytes,2,opt,name=session_id,json=sessionId,proto3" json:"session_id,omitempty"` // Session ID of the client - Axis int32 `protobuf:"varint,3,opt,name=axis,proto3" json:"axis,omitempty"` // Axis number (0 for d-pad horizontal, 1 for d-pad vertical) - Value int32 `protobuf:"varint,4,opt,name=value,proto3" json:"value,omitempty"` // axis value (-1 to 1) - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache -} - -func (x *ProtoControllerAxis) Reset() { - *x = ProtoControllerAxis{} - mi := &file_types_proto_msgTypes[12] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) -} - -func (x *ProtoControllerAxis) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*ProtoControllerAxis) ProtoMessage() {} - -func (x *ProtoControllerAxis) ProtoReflect() protoreflect.Message { - mi := &file_types_proto_msgTypes[12] - if x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use ProtoControllerAxis.ProtoReflect.Descriptor instead. -func (*ProtoControllerAxis) Descriptor() ([]byte, []int) { - return file_types_proto_rawDescGZIP(), []int{12} + return nil } -func (x *ProtoControllerAxis) GetSessionSlot() int32 { - if x != nil { - return x.SessionSlot +func (x *ProtoControllerStateBatch) GetLeftStickX() int32 { + if x != nil && x.LeftStickX != nil { + return *x.LeftStickX } return 0 } -func (x *ProtoControllerAxis) GetSessionId() string { - if x != nil { - return x.SessionId - } - return "" -} - -func (x *ProtoControllerAxis) GetAxis() int32 { - if x != nil { - return x.Axis +func (x *ProtoControllerStateBatch) GetLeftStickY() int32 { + if x != nil && x.LeftStickY != nil { + return *x.LeftStickY } return 0 } -func (x *ProtoControllerAxis) GetValue() int32 { - if x != nil { - return x.Value +func (x *ProtoControllerStateBatch) GetRightStickX() int32 { + if x != nil && x.RightStickX != nil { + return *x.RightStickX } return 0 } -// ControllerRumble message -type ProtoControllerRumble struct { - state protoimpl.MessageState `protogen:"open.v1"` - SessionSlot int32 `protobuf:"varint,1,opt,name=session_slot,json=sessionSlot,proto3" json:"session_slot,omitempty"` // Session specific slot number (0-3) - SessionId string `protobuf:"bytes,2,opt,name=session_id,json=sessionId,proto3" json:"session_id,omitempty"` // Session ID of the client - LowFrequency int32 `protobuf:"varint,3,opt,name=low_frequency,json=lowFrequency,proto3" json:"low_frequency,omitempty"` // Low frequency rumble (0-65535) - HighFrequency int32 `protobuf:"varint,4,opt,name=high_frequency,json=highFrequency,proto3" json:"high_frequency,omitempty"` // High frequency rumble (0-65535) - Duration int32 `protobuf:"varint,5,opt,name=duration,proto3" json:"duration,omitempty"` // Duration in milliseconds - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache -} - -func (x *ProtoControllerRumble) Reset() { - *x = ProtoControllerRumble{} - mi := &file_types_proto_msgTypes[13] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) -} - -func (x *ProtoControllerRumble) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*ProtoControllerRumble) ProtoMessage() {} - -func (x *ProtoControllerRumble) ProtoReflect() protoreflect.Message { - mi := &file_types_proto_msgTypes[13] - if x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms +func (x *ProtoControllerStateBatch) GetRightStickY() int32 { + if x != nil && x.RightStickY != nil { + return *x.RightStickY } - return mi.MessageOf(x) -} - -// Deprecated: Use ProtoControllerRumble.ProtoReflect.Descriptor instead. -func (*ProtoControllerRumble) Descriptor() ([]byte, []int) { - return file_types_proto_rawDescGZIP(), []int{13} + return 0 } -func (x *ProtoControllerRumble) GetSessionSlot() int32 { - if x != nil { - return x.SessionSlot +func (x *ProtoControllerStateBatch) GetLeftTrigger() int32 { + if x != nil && x.LeftTrigger != nil { + return *x.LeftTrigger } return 0 } -func (x *ProtoControllerRumble) GetSessionId() string { - if x != nil { - return x.SessionId +func (x *ProtoControllerStateBatch) GetRightTrigger() int32 { + if x != nil && x.RightTrigger != nil { + return *x.RightTrigger } - return "" + return 0 } -func (x *ProtoControllerRumble) GetLowFrequency() int32 { - if x != nil { - return x.LowFrequency +func (x *ProtoControllerStateBatch) GetDpadX() int32 { + if x != nil && x.DpadX != nil { + return *x.DpadX } return 0 } -func (x *ProtoControllerRumble) GetHighFrequency() int32 { - if x != nil { - return x.HighFrequency +func (x *ProtoControllerStateBatch) GetDpadY() int32 { + if x != nil && x.DpadY != nil { + return *x.DpadY } return 0 } -func (x *ProtoControllerRumble) GetDuration() int32 { - if x != nil { - return x.Duration +func (x *ProtoControllerStateBatch) GetChangedFields() uint32 { + if x != nil && x.ChangedFields != nil { + return *x.ChangedFields } return 0 } @@ -847,7 +763,7 @@ type RTCIceCandidateInit struct { func (x *RTCIceCandidateInit) Reset() { *x = RTCIceCandidateInit{} - mi := &file_types_proto_msgTypes[14] + mi := &file_types_proto_msgTypes[11] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -859,7 +775,7 @@ func (x *RTCIceCandidateInit) String() string { func (*RTCIceCandidateInit) ProtoMessage() {} func (x *RTCIceCandidateInit) ProtoReflect() protoreflect.Message { - mi := &file_types_proto_msgTypes[14] + mi := &file_types_proto_msgTypes[11] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -872,7 +788,7 @@ func (x *RTCIceCandidateInit) ProtoReflect() protoreflect.Message { // Deprecated: Use RTCIceCandidateInit.ProtoReflect.Descriptor instead. func (*RTCIceCandidateInit) Descriptor() ([]byte, []int) { - return file_types_proto_rawDescGZIP(), []int{14} + return file_types_proto_rawDescGZIP(), []int{11} } func (x *RTCIceCandidateInit) GetCandidate() string { @@ -913,7 +829,7 @@ type RTCSessionDescriptionInit struct { func (x *RTCSessionDescriptionInit) Reset() { *x = RTCSessionDescriptionInit{} - mi := &file_types_proto_msgTypes[15] + mi := &file_types_proto_msgTypes[12] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -925,7 +841,7 @@ func (x *RTCSessionDescriptionInit) String() string { func (*RTCSessionDescriptionInit) ProtoMessage() {} func (x *RTCSessionDescriptionInit) ProtoReflect() protoreflect.Message { - mi := &file_types_proto_msgTypes[15] + mi := &file_types_proto_msgTypes[12] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -938,7 +854,7 @@ func (x *RTCSessionDescriptionInit) ProtoReflect() protoreflect.Message { // Deprecated: Use RTCSessionDescriptionInit.ProtoReflect.Descriptor instead. func (*RTCSessionDescriptionInit) Descriptor() ([]byte, []int) { - return file_types_proto_rawDescGZIP(), []int{15} + return file_types_proto_rawDescGZIP(), []int{12} } func (x *RTCSessionDescriptionInit) GetSdp() string { @@ -965,7 +881,7 @@ type ProtoICE struct { func (x *ProtoICE) Reset() { *x = ProtoICE{} - mi := &file_types_proto_msgTypes[16] + mi := &file_types_proto_msgTypes[13] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -977,7 +893,7 @@ func (x *ProtoICE) String() string { func (*ProtoICE) ProtoMessage() {} func (x *ProtoICE) ProtoReflect() protoreflect.Message { - mi := &file_types_proto_msgTypes[16] + mi := &file_types_proto_msgTypes[13] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -990,7 +906,7 @@ func (x *ProtoICE) ProtoReflect() protoreflect.Message { // Deprecated: Use ProtoICE.ProtoReflect.Descriptor instead. func (*ProtoICE) Descriptor() ([]byte, []int) { - return file_types_proto_rawDescGZIP(), []int{16} + return file_types_proto_rawDescGZIP(), []int{13} } func (x *ProtoICE) GetCandidate() *RTCIceCandidateInit { @@ -1010,7 +926,7 @@ type ProtoSDP struct { func (x *ProtoSDP) Reset() { *x = ProtoSDP{} - mi := &file_types_proto_msgTypes[17] + mi := &file_types_proto_msgTypes[14] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1022,7 +938,7 @@ func (x *ProtoSDP) String() string { func (*ProtoSDP) ProtoMessage() {} func (x *ProtoSDP) ProtoReflect() protoreflect.Message { - mi := &file_types_proto_msgTypes[17] + mi := &file_types_proto_msgTypes[14] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1035,7 +951,7 @@ func (x *ProtoSDP) ProtoReflect() protoreflect.Message { // Deprecated: Use ProtoSDP.ProtoReflect.Descriptor instead. func (*ProtoSDP) Descriptor() ([]byte, []int) { - return file_types_proto_rawDescGZIP(), []int{17} + return file_types_proto_rawDescGZIP(), []int{14} } func (x *ProtoSDP) GetSdp() *RTCSessionDescriptionInit { @@ -1055,7 +971,7 @@ type ProtoRaw struct { func (x *ProtoRaw) Reset() { *x = ProtoRaw{} - mi := &file_types_proto_msgTypes[18] + mi := &file_types_proto_msgTypes[15] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1067,7 +983,7 @@ func (x *ProtoRaw) String() string { func (*ProtoRaw) ProtoMessage() {} func (x *ProtoRaw) ProtoReflect() protoreflect.Message { - mi := &file_types_proto_msgTypes[18] + mi := &file_types_proto_msgTypes[15] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1080,7 +996,7 @@ func (x *ProtoRaw) ProtoReflect() protoreflect.Message { // Deprecated: Use ProtoRaw.ProtoReflect.Descriptor instead. func (*ProtoRaw) Descriptor() ([]byte, []int) { - return file_types_proto_rawDescGZIP(), []int{18} + return file_types_proto_rawDescGZIP(), []int{15} } func (x *ProtoRaw) GetData() string { @@ -1101,7 +1017,7 @@ type ProtoClientRequestRoomStream struct { func (x *ProtoClientRequestRoomStream) Reset() { *x = ProtoClientRequestRoomStream{} - mi := &file_types_proto_msgTypes[19] + mi := &file_types_proto_msgTypes[16] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1113,7 +1029,7 @@ func (x *ProtoClientRequestRoomStream) String() string { func (*ProtoClientRequestRoomStream) ProtoMessage() {} func (x *ProtoClientRequestRoomStream) ProtoReflect() protoreflect.Message { - mi := &file_types_proto_msgTypes[19] + mi := &file_types_proto_msgTypes[16] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1126,7 +1042,7 @@ func (x *ProtoClientRequestRoomStream) ProtoReflect() protoreflect.Message { // Deprecated: Use ProtoClientRequestRoomStream.ProtoReflect.Descriptor instead. func (*ProtoClientRequestRoomStream) Descriptor() ([]byte, []int) { - return file_types_proto_rawDescGZIP(), []int{19} + return file_types_proto_rawDescGZIP(), []int{16} } func (x *ProtoClientRequestRoomStream) GetRoomName() string { @@ -1154,7 +1070,7 @@ type ProtoClientDisconnected struct { func (x *ProtoClientDisconnected) Reset() { *x = ProtoClientDisconnected{} - mi := &file_types_proto_msgTypes[20] + mi := &file_types_proto_msgTypes[17] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1166,7 +1082,7 @@ func (x *ProtoClientDisconnected) String() string { func (*ProtoClientDisconnected) ProtoMessage() {} func (x *ProtoClientDisconnected) ProtoReflect() protoreflect.Message { - mi := &file_types_proto_msgTypes[20] + mi := &file_types_proto_msgTypes[17] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1179,7 +1095,7 @@ func (x *ProtoClientDisconnected) ProtoReflect() protoreflect.Message { // Deprecated: Use ProtoClientDisconnected.ProtoReflect.Descriptor instead. func (*ProtoClientDisconnected) Descriptor() ([]byte, []int) { - return file_types_proto_rawDescGZIP(), []int{20} + return file_types_proto_rawDescGZIP(), []int{17} } func (x *ProtoClientDisconnected) GetSessionId() string { @@ -1206,7 +1122,7 @@ type ProtoServerPushStream struct { func (x *ProtoServerPushStream) Reset() { *x = ProtoServerPushStream{} - mi := &file_types_proto_msgTypes[21] + mi := &file_types_proto_msgTypes[18] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1218,7 +1134,7 @@ func (x *ProtoServerPushStream) String() string { func (*ProtoServerPushStream) ProtoMessage() {} func (x *ProtoServerPushStream) ProtoReflect() protoreflect.Message { - mi := &file_types_proto_msgTypes[21] + mi := &file_types_proto_msgTypes[18] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1231,7 +1147,7 @@ func (x *ProtoServerPushStream) ProtoReflect() protoreflect.Message { // Deprecated: Use ProtoServerPushStream.ProtoReflect.Descriptor instead. func (*ProtoServerPushStream) Descriptor() ([]byte, []int) { - return file_types_proto_rawDescGZIP(), []int{21} + return file_types_proto_rawDescGZIP(), []int{18} } func (x *ProtoServerPushStream) GetRoomName() string { @@ -1272,39 +1188,51 @@ const file_types_proto_rawDesc = "" + "\x15ProtoControllerDetach\x12!\n" + "\fsession_slot\x18\x01 \x01(\x05R\vsessionSlot\x12\x1d\n" + "\n" + - "session_id\x18\x02 \x01(\tR\tsessionId\"\x8b\x01\n" + - "\x15ProtoControllerButton\x12!\n" + - "\fsession_slot\x18\x01 \x01(\x05R\vsessionSlot\x12\x1d\n" + - "\n" + - "session_id\x18\x02 \x01(\tR\tsessionId\x12\x16\n" + - "\x06button\x18\x03 \x01(\x05R\x06button\x12\x18\n" + - "\apressed\x18\x04 \x01(\bR\apressed\"\x8a\x01\n" + - "\x16ProtoControllerTrigger\x12!\n" + - "\fsession_slot\x18\x01 \x01(\x05R\vsessionSlot\x12\x1d\n" + - "\n" + - "session_id\x18\x02 \x01(\tR\tsessionId\x12\x18\n" + - "\atrigger\x18\x03 \x01(\x05R\atrigger\x12\x14\n" + - "\x05value\x18\x04 \x01(\x05R\x05value\"\x8a\x01\n" + - "\x14ProtoControllerStick\x12!\n" + - "\fsession_slot\x18\x01 \x01(\x05R\vsessionSlot\x12\x1d\n" + - "\n" + - "session_id\x18\x02 \x01(\tR\tsessionId\x12\x14\n" + - "\x05stick\x18\x03 \x01(\x05R\x05stick\x12\f\n" + - "\x01x\x18\x04 \x01(\x05R\x01x\x12\f\n" + - "\x01y\x18\x05 \x01(\x05R\x01y\"\x81\x01\n" + - "\x13ProtoControllerAxis\x12!\n" + - "\fsession_slot\x18\x01 \x01(\x05R\vsessionSlot\x12\x1d\n" + - "\n" + - "session_id\x18\x02 \x01(\tR\tsessionId\x12\x12\n" + - "\x04axis\x18\x03 \x01(\x05R\x04axis\x12\x14\n" + - "\x05value\x18\x04 \x01(\x05R\x05value\"\xc1\x01\n" + + "session_id\x18\x02 \x01(\tR\tsessionId\"\xc1\x01\n" + "\x15ProtoControllerRumble\x12!\n" + "\fsession_slot\x18\x01 \x01(\x05R\vsessionSlot\x12\x1d\n" + "\n" + "session_id\x18\x02 \x01(\tR\tsessionId\x12#\n" + "\rlow_frequency\x18\x03 \x01(\x05R\flowFrequency\x12%\n" + "\x0ehigh_frequency\x18\x04 \x01(\x05R\rhighFrequency\x12\x1a\n" + - "\bduration\x18\x05 \x01(\x05R\bduration\"\xde\x01\n" + + "\bduration\x18\x05 \x01(\x05R\bduration\"\x87\a\n" + + "\x19ProtoControllerStateBatch\x12!\n" + + "\fsession_slot\x18\x01 \x01(\x05R\vsessionSlot\x12\x1d\n" + + "\n" + + "session_id\x18\x02 \x01(\tR\tsessionId\x12L\n" + + "\vupdate_type\x18\x03 \x01(\x0e2+.proto.ProtoControllerStateBatch.UpdateTypeR\n" + + "updateType\x12\x1a\n" + + "\bsequence\x18\x04 \x01(\rR\bsequence\x12g\n" + + "\x13button_changed_mask\x18\x05 \x03(\v27.proto.ProtoControllerStateBatch.ButtonChangedMaskEntryR\x11buttonChangedMask\x12%\n" + + "\fleft_stick_x\x18\x06 \x01(\x05H\x00R\n" + + "leftStickX\x88\x01\x01\x12%\n" + + "\fleft_stick_y\x18\a \x01(\x05H\x01R\n" + + "leftStickY\x88\x01\x01\x12'\n" + + "\rright_stick_x\x18\b \x01(\x05H\x02R\vrightStickX\x88\x01\x01\x12'\n" + + "\rright_stick_y\x18\t \x01(\x05H\x03R\vrightStickY\x88\x01\x01\x12&\n" + + "\fleft_trigger\x18\n" + + " \x01(\x05H\x04R\vleftTrigger\x88\x01\x01\x12(\n" + + "\rright_trigger\x18\v \x01(\x05H\x05R\frightTrigger\x88\x01\x01\x12\x1a\n" + + "\x06dpad_x\x18\f \x01(\x05H\x06R\x05dpadX\x88\x01\x01\x12\x1a\n" + + "\x06dpad_y\x18\r \x01(\x05H\aR\x05dpadY\x88\x01\x01\x12*\n" + + "\x0echanged_fields\x18\x0e \x01(\rH\bR\rchangedFields\x88\x01\x01\x1aD\n" + + "\x16ButtonChangedMaskEntry\x12\x10\n" + + "\x03key\x18\x01 \x01(\x05R\x03key\x12\x14\n" + + "\x05value\x18\x02 \x01(\bR\x05value:\x028\x01\"'\n" + + "\n" + + "UpdateType\x12\x0e\n" + + "\n" + + "FULL_STATE\x10\x00\x12\t\n" + + "\x05DELTA\x10\x01B\x0f\n" + + "\r_left_stick_xB\x0f\n" + + "\r_left_stick_yB\x10\n" + + "\x0e_right_stick_xB\x10\n" + + "\x0e_right_stick_yB\x0f\n" + + "\r_left_triggerB\x10\n" + + "\x0e_right_triggerB\t\n" + + "\a_dpad_xB\t\n" + + "\a_dpad_yB\x11\n" + + "\x0f_changed_fields\"\xde\x01\n" + "\x13RTCIceCandidateInit\x12\x1c\n" + "\tcandidate\x18\x01 \x01(\tR\tcandidate\x12)\n" + "\rsdpMLineIndex\x18\x02 \x01(\rH\x00R\rsdpMLineIndex\x88\x01\x01\x12\x1b\n" + @@ -1345,39 +1273,41 @@ func file_types_proto_rawDescGZIP() []byte { return file_types_proto_rawDescData } -var file_types_proto_msgTypes = make([]protoimpl.MessageInfo, 22) +var file_types_proto_enumTypes = make([]protoimpl.EnumInfo, 1) +var file_types_proto_msgTypes = make([]protoimpl.MessageInfo, 20) var file_types_proto_goTypes = []any{ - (*ProtoMouseMove)(nil), // 0: proto.ProtoMouseMove - (*ProtoMouseMoveAbs)(nil), // 1: proto.ProtoMouseMoveAbs - (*ProtoMouseWheel)(nil), // 2: proto.ProtoMouseWheel - (*ProtoMouseKeyDown)(nil), // 3: proto.ProtoMouseKeyDown - (*ProtoMouseKeyUp)(nil), // 4: proto.ProtoMouseKeyUp - (*ProtoKeyDown)(nil), // 5: proto.ProtoKeyDown - (*ProtoKeyUp)(nil), // 6: proto.ProtoKeyUp - (*ProtoControllerAttach)(nil), // 7: proto.ProtoControllerAttach - (*ProtoControllerDetach)(nil), // 8: proto.ProtoControllerDetach - (*ProtoControllerButton)(nil), // 9: proto.ProtoControllerButton - (*ProtoControllerTrigger)(nil), // 10: proto.ProtoControllerTrigger - (*ProtoControllerStick)(nil), // 11: proto.ProtoControllerStick - (*ProtoControllerAxis)(nil), // 12: proto.ProtoControllerAxis - (*ProtoControllerRumble)(nil), // 13: proto.ProtoControllerRumble - (*RTCIceCandidateInit)(nil), // 14: proto.RTCIceCandidateInit - (*RTCSessionDescriptionInit)(nil), // 15: proto.RTCSessionDescriptionInit - (*ProtoICE)(nil), // 16: proto.ProtoICE - (*ProtoSDP)(nil), // 17: proto.ProtoSDP - (*ProtoRaw)(nil), // 18: proto.ProtoRaw - (*ProtoClientRequestRoomStream)(nil), // 19: proto.ProtoClientRequestRoomStream - (*ProtoClientDisconnected)(nil), // 20: proto.ProtoClientDisconnected - (*ProtoServerPushStream)(nil), // 21: proto.ProtoServerPushStream + (ProtoControllerStateBatch_UpdateType)(0), // 0: proto.ProtoControllerStateBatch.UpdateType + (*ProtoMouseMove)(nil), // 1: proto.ProtoMouseMove + (*ProtoMouseMoveAbs)(nil), // 2: proto.ProtoMouseMoveAbs + (*ProtoMouseWheel)(nil), // 3: proto.ProtoMouseWheel + (*ProtoMouseKeyDown)(nil), // 4: proto.ProtoMouseKeyDown + (*ProtoMouseKeyUp)(nil), // 5: proto.ProtoMouseKeyUp + (*ProtoKeyDown)(nil), // 6: proto.ProtoKeyDown + (*ProtoKeyUp)(nil), // 7: proto.ProtoKeyUp + (*ProtoControllerAttach)(nil), // 8: proto.ProtoControllerAttach + (*ProtoControllerDetach)(nil), // 9: proto.ProtoControllerDetach + (*ProtoControllerRumble)(nil), // 10: proto.ProtoControllerRumble + (*ProtoControllerStateBatch)(nil), // 11: proto.ProtoControllerStateBatch + (*RTCIceCandidateInit)(nil), // 12: proto.RTCIceCandidateInit + (*RTCSessionDescriptionInit)(nil), // 13: proto.RTCSessionDescriptionInit + (*ProtoICE)(nil), // 14: proto.ProtoICE + (*ProtoSDP)(nil), // 15: proto.ProtoSDP + (*ProtoRaw)(nil), // 16: proto.ProtoRaw + (*ProtoClientRequestRoomStream)(nil), // 17: proto.ProtoClientRequestRoomStream + (*ProtoClientDisconnected)(nil), // 18: proto.ProtoClientDisconnected + (*ProtoServerPushStream)(nil), // 19: proto.ProtoServerPushStream + nil, // 20: proto.ProtoControllerStateBatch.ButtonChangedMaskEntry } var file_types_proto_depIdxs = []int32{ - 14, // 0: proto.ProtoICE.candidate:type_name -> proto.RTCIceCandidateInit - 15, // 1: proto.ProtoSDP.sdp:type_name -> proto.RTCSessionDescriptionInit - 2, // [2:2] is the sub-list for method output_type - 2, // [2:2] is the sub-list for method input_type - 2, // [2:2] is the sub-list for extension type_name - 2, // [2:2] is the sub-list for extension extendee - 0, // [0:2] is the sub-list for field type_name + 0, // 0: proto.ProtoControllerStateBatch.update_type:type_name -> proto.ProtoControllerStateBatch.UpdateType + 20, // 1: proto.ProtoControllerStateBatch.button_changed_mask:type_name -> proto.ProtoControllerStateBatch.ButtonChangedMaskEntry + 12, // 2: proto.ProtoICE.candidate:type_name -> proto.RTCIceCandidateInit + 13, // 3: proto.ProtoSDP.sdp:type_name -> proto.RTCSessionDescriptionInit + 4, // [4:4] is the sub-list for method output_type + 4, // [4:4] is the sub-list for method input_type + 4, // [4:4] is the sub-list for extension type_name + 4, // [4:4] is the sub-list for extension extendee + 0, // [0:4] is the sub-list for field type_name } func init() { file_types_proto_init() } @@ -1385,19 +1315,21 @@ func file_types_proto_init() { if File_types_proto != nil { return } - file_types_proto_msgTypes[14].OneofWrappers = []any{} + file_types_proto_msgTypes[10].OneofWrappers = []any{} + file_types_proto_msgTypes[11].OneofWrappers = []any{} type x struct{} out := protoimpl.TypeBuilder{ File: protoimpl.DescBuilder{ GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: unsafe.Slice(unsafe.StringData(file_types_proto_rawDesc), len(file_types_proto_rawDesc)), - NumEnums: 0, - NumMessages: 22, + NumEnums: 1, + NumMessages: 20, NumExtensions: 0, NumServices: 0, }, GoTypes: file_types_proto_goTypes, DependencyIndexes: file_types_proto_depIdxs, + EnumInfos: file_types_proto_enumTypes, MessageInfos: file_types_proto_msgTypes, }.Build() File_types_proto = out.File diff --git a/packages/server/Cargo.lock b/packages/server/Cargo.lock index ef4f4763..996a53fe 100644 --- a/packages/server/Cargo.lock +++ b/packages/server/Cargo.lock @@ -5079,9 +5079,9 @@ checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" [[package]] name = "vimputti" -version = "0.1.4" +version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ffb370ee43e3ee4ca5329886e64dc5b27c83dc8cced5a63c2418777dac9a41a8" +checksum = "6440b3684270f355fb89193bfb0de957686119626b8b207f21d91024a892d05c" dependencies = [ "anyhow", "libc", diff --git a/packages/server/Cargo.toml b/packages/server/Cargo.toml index 9da600f4..7516d1fe 100644 --- a/packages/server/Cargo.toml +++ b/packages/server/Cargo.toml @@ -22,7 +22,7 @@ rand = "0.9" rustls = { version = "0.23", features = ["ring"] } tracing = "0.1" tracing-subscriber = { version = "0.3", features = ["env-filter"] } -vimputti = "0.1.4" +vimputti = "0.1.7" chrono = "0.4" prost = "0.14" prost-types = "0.14" diff --git a/packages/server/src/input/controller.rs b/packages/server/src/input/controller.rs index d34e8600..284c2e07 100644 --- a/packages/server/src/input/controller.rs +++ b/packages/server/src/input/controller.rs @@ -29,10 +29,8 @@ impl ControllerInput { client: &vimputti::client::VimputtiClient, ) -> Result { let config = controller_string_to_type(&controller_type)?; - Ok(Self { - config: config.clone(), - device: client.create_device(config).await?, - }) + let device = client.create_device(config.clone()).await?; + Ok(Self { config, device }) } pub fn device_mut(&mut self) -> &mut vimputti::client::VirtualController { @@ -121,9 +119,60 @@ async fn command_loop( slot.session_id == session_id && slot.session_slot == session_slot as u32 }) .map(|(slot_num, _)| *slot_num); - let slot = existing_slot.or_else(|| get_free_slot(&controllers)); - if let Some(slot) = slot { + if let Some(existing_slot) = existing_slot { + if let Some(controller_slot) = controllers.get_mut(&existing_slot) { + let rumble_tx = rumble_tx.clone(); + let attach_tx = attach_tx.clone(); + + controller_slot + .controller + .device_mut() + .on_rumble(move |strong, weak, duration_ms| { + let _ = rumble_tx.try_send(( + existing_slot, + strong, + weak, + duration_ms, + data.session_id.clone(), + )); + }) + .await + .map_err(|e| { + tracing::warn!( + "Failed to register rumble callback for slot {}: {}", + existing_slot, + e + ); + }) + .ok(); + + // Return to attach_tx what slot was assigned + let attach_info = ProtoControllerAttach { + id: data.id.clone(), + session_slot: existing_slot as i32, + session_id: session_id.clone(), + }; + + match attach_tx.send(attach_info).await { + Ok(_) => { + tracing::info!( + "Controller {} re-attached to slot {} (session: {})", + data.id, + existing_slot, + session_id + ); + } + Err(e) => { + tracing::error!( + "Failed to send re-attach info for slot {}: {}", + existing_slot, + e + ); + } + } + } + } else if let Some(slot) = get_free_slot(&controllers) { if let Ok(mut controller) = ControllerInput::new(data.id.clone(), &vimputti_client).await { @@ -133,7 +182,13 @@ async fn command_loop( controller .device_mut() .on_rumble(move |strong, weak, duration_ms| { - let _ = rumble_tx.try_send((slot, strong, weak, duration_ms, data.session_id.clone())); + let _ = rumble_tx.try_send(( + slot, + strong, + weak, + duration_ms, + data.session_id.clone(), + )); }) .await .map_err(|e| { @@ -190,65 +245,10 @@ async fn command_loop( if controllers.remove(&(data.session_slot as u32)).is_some() { tracing::info!("Controller detached from slot {}", data.session_slot); } else { - tracing::warn!("No controller found in slot {} to detach", data.session_slot); - } - } - Payload::ControllerButton(data) => { - if let Some(controller) = controllers.get(&(data.session_slot as u32)) { - if let Some(button) = vimputti::Button::from_ev_code(data.button as u16) { - let device = controller.controller.device(); - device.button(button, data.pressed); - device.sync(); - } - } else { - tracing::warn!("Controller slot {} not found for button event", data.session_slot); - } - } - Payload::ControllerStick(data) => { - if let Some(controller) = controllers.get(&(data.session_slot as u32)) { - let device = controller.controller.device(); - if data.stick == 0 { - // Left stick - device.axis(vimputti::Axis::LeftStickX, data.x); - device.sync(); - device.axis(vimputti::Axis::LeftStickY, data.y); - } else if data.stick == 1 { - // Right stick - device.axis(vimputti::Axis::RightStickX, data.x); - device.sync(); - device.axis(vimputti::Axis::RightStickY, data.y); - } - device.sync(); - } else { - tracing::warn!("Controller slot {} not found for stick event", data.session_slot); - } - } - Payload::ControllerTrigger(data) => { - if let Some(controller) = controllers.get(&(data.session_slot as u32)) { - let device = controller.controller.device(); - if data.trigger == 0 { - // Left trigger - device.axis(vimputti::Axis::LowerLeftTrigger, data.value); - } else if data.trigger == 1 { - // Right trigger - device.axis(vimputti::Axis::LowerRightTrigger, data.value); - } - device.sync(); - } else { - tracing::warn!("Controller slot {} not found for trigger event", data.session_slot); - } - } - Payload::ControllerAxis(data) => { - if let Some(controller) = controllers.get(&(data.session_slot as u32)) { - let device = controller.controller.device(); - if data.axis == 0 { - // dpad x - device.axis(vimputti::Axis::DPadX, data.value); - } else if data.axis == 1 { - // dpad y - device.axis(vimputti::Axis::DPadY, data.value); - } - device.sync(); + tracing::warn!( + "No controller found in slot {} to detach", + data.session_slot + ); } } Payload::ClientDisconnected(data) => { @@ -274,6 +274,125 @@ async fn command_loop( } } } + Payload::ControllerStateBatch(data) => { + if let Some(controller) = controllers.get(&(data.session_slot as u32)) { + let device = controller.controller.device(); + + // Handle inputs based on update type + if data.update_type == 0 { + // FULL_STATE: Update all values + let _ = device.sync().await; + for (btn_code, pressed) in data.button_changed_mask { + if let Some(button) = vimputti::Button::from_ev_code(btn_code as u16) { + let _ = device.button(button, pressed).await; + let _ = device.sync().await; + } + } + if let Some(x) = data.left_stick_x { + let _ = device.axis(vimputti::Axis::LeftStickX, x).await; + let _ = device.sync().await; + } + if let Some(y) = data.left_stick_y { + let _ = device.axis(vimputti::Axis::LeftStickY, y).await; + let _ = device.sync().await; + } + if let Some(x) = data.right_stick_x { + let _ = device.axis(vimputti::Axis::RightStickX, x).await; + let _ = device.sync().await; + } + if let Some(y) = data.right_stick_y { + let _ = device.axis(vimputti::Axis::RightStickY, y).await; + let _ = device.sync().await; + } + if let Some(value) = data.left_trigger { + let _ = device.axis(vimputti::Axis::LowerLeftTrigger, value).await; + let _ = device.sync().await; + } + if let Some(value) = data.right_trigger { + let _ = device.axis(vimputti::Axis::LowerRightTrigger, value).await; + let _ = device.sync().await; + } + if let Some(x) = data.dpad_x { + let _ = device.axis(vimputti::Axis::DPadX, x).await; + let _ = device.sync().await; + } + if let Some(y) = data.dpad_y { + let _ = device.axis(vimputti::Axis::DPadY, y).await; + let _ = device.sync().await; + } + } else { + // DELTA: Only update changed values + if let Some(changed_fields) = data.changed_fields { + let _ = device.sync().await; + if (changed_fields & (1 << 0)) != 0 { + for (btn_code, pressed) in data.button_changed_mask { + if let Some(button) = + vimputti::Button::from_ev_code(btn_code as u16) + { + let _ = device.button(button, pressed).await; + let _ = device.sync().await; + } + } + } + if (changed_fields & (1 << 1)) != 0 { + if let Some(x) = data.left_stick_x { + let _ = device.axis(vimputti::Axis::LeftStickX, x).await; + let _ = device.sync().await; + } + } + if (changed_fields & (1 << 2)) != 0 { + if let Some(y) = data.left_stick_y { + let _ = device.axis(vimputti::Axis::LeftStickY, y).await; + let _ = device.sync().await; + } + } + if (changed_fields & (1 << 3)) != 0 { + if let Some(x) = data.right_stick_x { + let _ = device.axis(vimputti::Axis::RightStickX, x).await; + let _ = device.sync().await; + } + } + if (changed_fields & (1 << 4)) != 0 { + if let Some(y) = data.right_stick_y { + let _ = device.axis(vimputti::Axis::RightStickY, y).await; + let _ = device.sync().await; + } + } + if (changed_fields & (1 << 5)) != 0 { + if let Some(value) = data.left_trigger { + let _ = + device.axis(vimputti::Axis::LowerLeftTrigger, value).await; + let _ = device.sync().await; + } + } + if (changed_fields & (1 << 6)) != 0 { + if let Some(value) = data.right_trigger { + let _ = + device.axis(vimputti::Axis::LowerRightTrigger, value).await; + let _ = device.sync().await; + } + } + if (changed_fields & (1 << 7)) != 0 { + if let Some(x) = data.dpad_x { + let _ = device.axis(vimputti::Axis::DPadX, x).await; + let _ = device.sync().await; + } + } + if (changed_fields & (1 << 8)) != 0 { + if let Some(y) = data.dpad_y { + let _ = device.axis(vimputti::Axis::DPadY, y).await; + let _ = device.sync().await; + } + } + } + } + } else { + tracing::warn!( + "Controller slot {} not found for state batch event", + data.session_slot + ); + } + } _ => { //no-op } diff --git a/packages/server/src/proto/proto.rs b/packages/server/src/proto/proto.rs index 635b8f3e..a9d68c76 100644 --- a/packages/server/src/proto/proto.rs +++ b/packages/server/src/proto/proto.rs @@ -102,96 +102,105 @@ pub struct ProtoControllerDetach { #[prost(string, tag="2")] pub session_id: ::prost::alloc::string::String, } -/// ControllerButton message -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct ProtoControllerButton { - /// Session specific slot number (0-3) - #[prost(int32, tag="1")] - pub session_slot: i32, - /// Session ID of the client - #[prost(string, tag="2")] - pub session_id: ::prost::alloc::string::String, - /// Button code (linux input event code) - #[prost(int32, tag="3")] - pub button: i32, - /// true if pressed, false if released - #[prost(bool, tag="4")] - pub pressed: bool, -} -/// ControllerTriggers message -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct ProtoControllerTrigger { - /// Session specific slot number (0-3) - #[prost(int32, tag="1")] - pub session_slot: i32, - /// Session ID of the client - #[prost(string, tag="2")] - pub session_id: ::prost::alloc::string::String, - /// Trigger number (0 for left, 1 for right) - #[prost(int32, tag="3")] - pub trigger: i32, - /// trigger value (-32768 to 32767) - #[prost(int32, tag="4")] - pub value: i32, -} -/// ControllerSticks message +/// ControllerRumble message #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] -pub struct ProtoControllerStick { +pub struct ProtoControllerRumble { /// Session specific slot number (0-3) #[prost(int32, tag="1")] pub session_slot: i32, /// Session ID of the client #[prost(string, tag="2")] pub session_id: ::prost::alloc::string::String, - /// Stick number (0 for left, 1 for right) + /// Low frequency rumble (0-65535) #[prost(int32, tag="3")] - pub stick: i32, - /// X axis value (-32768 to 32767) + pub low_frequency: i32, + /// High frequency rumble (0-65535) #[prost(int32, tag="4")] - pub x: i32, - /// Y axis value (-32768 to 32767) + pub high_frequency: i32, + /// Duration in milliseconds #[prost(int32, tag="5")] - pub y: i32, + pub duration: i32, } -/// ControllerAxis message +/// ControllerStateBatch - single message containing full or partial controller state #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] -pub struct ProtoControllerAxis { +pub struct ProtoControllerStateBatch { /// Session specific slot number (0-3) #[prost(int32, tag="1")] pub session_slot: i32, /// Session ID of the client #[prost(string, tag="2")] pub session_id: ::prost::alloc::string::String, - /// Axis number (0 for d-pad horizontal, 1 for d-pad vertical) - #[prost(int32, tag="3")] - pub axis: i32, - /// axis value (-1 to 1) - #[prost(int32, tag="4")] - pub value: i32, + #[prost(enumeration="proto_controller_state_batch::UpdateType", tag="3")] + pub update_type: i32, + /// Sequence number for packet loss detection + #[prost(uint32, tag="4")] + pub sequence: u32, + /// Button state map (Linux event codes) + #[prost(map="int32, bool", tag="5")] + pub button_changed_mask: ::std::collections::HashMap, + /// Analog inputs + /// + /// -32768 to 32767 + #[prost(int32, optional, tag="6")] + pub left_stick_x: ::core::option::Option, + /// -32768 to 32767 + #[prost(int32, optional, tag="7")] + pub left_stick_y: ::core::option::Option, + /// -32768 to 32767 + #[prost(int32, optional, tag="8")] + pub right_stick_x: ::core::option::Option, + /// -32768 to 32767 + #[prost(int32, optional, tag="9")] + pub right_stick_y: ::core::option::Option, + /// -32768 to 32767 + #[prost(int32, optional, tag="10")] + pub left_trigger: ::core::option::Option, + /// -32768 to 32767 + #[prost(int32, optional, tag="11")] + pub right_trigger: ::core::option::Option, + /// -1, 0, or 1 + #[prost(int32, optional, tag="12")] + pub dpad_x: ::core::option::Option, + /// -1, 0, or 1 + #[prost(int32, optional, tag="13")] + pub dpad_y: ::core::option::Option, + /// Bitmask indicating which fields have changed + /// Bit 0: button_changed_mask, Bit 1: left_stick_x, Bit 2: left_stick_y, etc. + #[prost(uint32, optional, tag="14")] + pub changed_fields: ::core::option::Option, } -/// ControllerRumble message -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct ProtoControllerRumble { - /// Session specific slot number (0-3) - #[prost(int32, tag="1")] - pub session_slot: i32, - /// Session ID of the client - #[prost(string, tag="2")] - pub session_id: ::prost::alloc::string::String, - /// Low frequency rumble (0-65535) - #[prost(int32, tag="3")] - pub low_frequency: i32, - /// High frequency rumble (0-65535) - #[prost(int32, tag="4")] - pub high_frequency: i32, - /// Duration in milliseconds - #[prost(int32, tag="5")] - pub duration: i32, +/// Nested message and enum types in `ProtoControllerStateBatch`. +pub mod proto_controller_state_batch { + #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] + #[repr(i32)] + pub enum UpdateType { + /// Complete controller state + FullState = 0, + /// Only changed fields + Delta = 1, + } + impl UpdateType { + /// String value of the enum field names used in the ProtoBuf definition. + /// + /// The values are not transformed in any way and thus are considered stable + /// (if the ProtoBuf definition does not change) and safe for programmatic use. + pub fn as_str_name(&self) -> &'static str { + match self { + UpdateType::FullState => "FULL_STATE", + UpdateType::Delta => "DELTA", + } + } + /// Creates an enum from field names used in the ProtoBuf definition. + pub fn from_str_name(value: &str) -> ::core::option::Option { + match value { + "FULL_STATE" => Some(Self::FullState), + "DELTA" => Some(Self::Delta), + _ => None, + } + } + } } // WebRTC + signaling @@ -274,7 +283,7 @@ pub struct ProtoMessageBase { pub struct ProtoMessage { #[prost(message, optional, tag="1")] pub message_base: ::core::option::Option, - #[prost(oneof="proto_message::Payload", tags="2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 20, 21, 22, 23, 24, 25")] + #[prost(oneof="proto_message::Payload", tags="2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 20, 21, 22, 23, 24, 25")] pub payload: ::core::option::Option, } /// Nested message and enum types in `ProtoMessage`. @@ -297,20 +306,15 @@ pub mod proto_message { KeyDown(super::ProtoKeyDown), #[prost(message, tag="8")] KeyUp(super::ProtoKeyUp), + /// Controller input types #[prost(message, tag="9")] ControllerAttach(super::ProtoControllerAttach), #[prost(message, tag="10")] ControllerDetach(super::ProtoControllerDetach), #[prost(message, tag="11")] - ControllerButton(super::ProtoControllerButton), - #[prost(message, tag="12")] - ControllerTrigger(super::ProtoControllerTrigger), - #[prost(message, tag="13")] - ControllerStick(super::ProtoControllerStick), - #[prost(message, tag="14")] - ControllerAxis(super::ProtoControllerAxis), - #[prost(message, tag="15")] ControllerRumble(super::ProtoControllerRumble), + #[prost(message, tag="12")] + ControllerStateBatch(super::ProtoControllerStateBatch), /// Signaling types #[prost(message, tag="20")] Ice(super::ProtoIce), diff --git a/protobufs/messages.proto b/protobufs/messages.proto index bc14fd1b..82d147ce 100644 --- a/protobufs/messages.proto +++ b/protobufs/messages.proto @@ -23,13 +23,12 @@ message ProtoMessage { ProtoMouseKeyUp mouse_key_up = 6; ProtoKeyDown key_down = 7; ProtoKeyUp key_up = 8; + + // Controller input types ProtoControllerAttach controller_attach = 9; ProtoControllerDetach controller_detach = 10; - ProtoControllerButton controller_button = 11; - ProtoControllerTrigger controller_trigger = 12; - ProtoControllerStick controller_stick = 13; - ProtoControllerAxis controller_axis = 14; - ProtoControllerRumble controller_rumble = 15; + ProtoControllerRumble controller_rumble = 11; + ProtoControllerStateBatch controller_state_batch = 12; // Signaling types ProtoICE ice = 20; diff --git a/protobufs/types.proto b/protobufs/types.proto index 1f951f16..deeed486 100644 --- a/protobufs/types.proto +++ b/protobufs/types.proto @@ -61,39 +61,6 @@ message ProtoControllerDetach { string session_id = 2; // Session ID of the client } -// ControllerButton message -message ProtoControllerButton { - int32 session_slot = 1; // Session specific slot number (0-3) - string session_id = 2; // Session ID of the client - int32 button = 3; // Button code (linux input event code) - bool pressed = 4; // true if pressed, false if released -} - -// ControllerTriggers message -message ProtoControllerTrigger { - int32 session_slot = 1; // Session specific slot number (0-3) - string session_id = 2; // Session ID of the client - int32 trigger = 3; // Trigger number (0 for left, 1 for right) - int32 value = 4; // trigger value (-32768 to 32767) -} - -// ControllerSticks message -message ProtoControllerStick { - int32 session_slot = 1; // Session specific slot number (0-3) - string session_id = 2; // Session ID of the client - int32 stick = 3; // Stick number (0 for left, 1 for right) - int32 x = 4; // X axis value (-32768 to 32767) - int32 y = 5; // Y axis value (-32768 to 32767) -} - -// ControllerAxis message -message ProtoControllerAxis { - int32 session_slot = 1; // Session specific slot number (0-3) - string session_id = 2; // Session ID of the client - int32 axis = 3; // Axis number (0 for d-pad horizontal, 1 for d-pad vertical) - int32 value = 4; // axis value (-1 to 1) -} - // ControllerRumble message message ProtoControllerRumble { int32 session_slot = 1; // Session specific slot number (0-3) @@ -103,6 +70,38 @@ message ProtoControllerRumble { int32 duration = 5; // Duration in milliseconds } +// ControllerStateBatch - single message containing full or partial controller state +message ProtoControllerStateBatch { + int32 session_slot = 1; // Session specific slot number (0-3) + string session_id = 2; // Session ID of the client + + enum UpdateType { + FULL_STATE = 0; // Complete controller state + DELTA = 1; // Only changed fields + } + UpdateType update_type = 3; + + // Sequence number for packet loss detection + uint32 sequence = 4; + + // Button state map (Linux event codes) + map button_changed_mask = 5; + + // Analog inputs + optional int32 left_stick_x = 6; // -32768 to 32767 + optional int32 left_stick_y = 7; // -32768 to 32767 + optional int32 right_stick_x = 8; // -32768 to 32767 + optional int32 right_stick_y = 9; // -32768 to 32767 + optional int32 left_trigger = 10; // -32768 to 32767 + optional int32 right_trigger = 11; // -32768 to 32767 + optional int32 dpad_x = 12; // -1, 0, or 1 + optional int32 dpad_y = 13; // -1, 0, or 1 + + // Bitmask indicating which fields have changed + // Bit 0: button_changed_mask, Bit 1: left_stick_x, Bit 2: left_stick_y, etc. + optional uint32 changed_fields = 14; +} + /* WebRTC + signaling */ message RTCIceCandidateInit { From 8d5895fc5e23acdbd73283e4da1cd3361b0dce1e Mon Sep 17 00:00:00 2001 From: DatCaptainHorse Date: Sat, 1 Nov 2025 05:02:23 +0200 Subject: [PATCH 4/6] Some rabbit nitpick fixes --- packages/input/src/controller.ts | 44 ++++++++------- packages/input/src/webrtc-stream.ts | 2 +- packages/relay/internal/common/ice_helper.go | 53 +++++++++++++++++++ .../relay/internal/core/protocol_stream.go | 50 ++++------------- packages/relay/internal/shared/participant.go | 7 ++- 5 files changed, 95 insertions(+), 61 deletions(-) create mode 100644 packages/relay/internal/common/ice_helper.go diff --git a/packages/input/src/controller.ts b/packages/input/src/controller.ts index ba71c69d..e1760945 100644 --- a/packages/input/src/controller.ts +++ b/packages/input/src/controller.ts @@ -56,7 +56,7 @@ export class Controller { // Polling configuration private readonly FULL_RATE_MS = 10; // 100 UPS private readonly IDLE_THRESHOLD = 100; // ms before considering idle/hands off controller - private readonly FULL_INTERVAL= 250; // ms before sending full state occassionally, to verify inputs are synced + private readonly FULL_INTERVAL = 250; // ms before sending full state occassionally, to verify inputs are synced // Polling state private pollingState: PollState = PollState.IDLE; @@ -230,7 +230,7 @@ export class Controller { // Changing from running to idle.. if (this.pollingState === PollState.RUNNING) { // Send full state on idle assumption - this.sendBatchedState(0xFF, 0); + this.sendBatchedState(0xff, 0); this.pollingState = PollState.IDLE; } } @@ -364,12 +364,15 @@ export class Controller { // For FULL_STATE, include everything if (updateType === 0) { - message.changedFields = 0xFF; + message.changedFields = 0xff; message.buttonChangedMask = Object.fromEntries( - Array.from(this.state.buttonState).map(([key, value]) => { - return [this.controllerButtonToVirtualKeyCode(key), value]; - }), + Array.from(this.state.buttonState) + .map( + ([key, value]) => + [this.controllerButtonToVirtualKeyCode(key), value] as const, + ) + .filter(([code]) => code !== undefined), ); message.leftStickX = this.state.leftX; message.leftStickY = this.state.leftY; @@ -402,9 +405,12 @@ export class Controller { }) .map((key) => { const newValue = currentStateMap.get(key) ?? false; - - return [this.controllerButtonToVirtualKeyCode(key), newValue]; - }), + return [ + this.controllerButtonToVirtualKeyCode(key), + newValue, + ] as const; + }) + .filter(([code]) => code !== undefined), ); } if (changedFields & this.CHANGED_LEFT_STICK_X) { @@ -468,14 +474,16 @@ export class Controller { this.wrtc.removeDataChannelCallback(this._dcHandler); this._dcHandler = null; } - // Gamepad disconnected - const detachMsg = createMessage( - create(ProtoControllerDetachSchema, { - sessionSlot: this.gamepad.index, - }), - "controllerInput", - ); - this.wrtc.sendBinary(toBinary(ProtoMessageSchema, detachMsg)); + if (this.gamepad) { + // Gamepad disconnected + const detachMsg = createMessage( + create(ProtoControllerDetachSchema, { + sessionSlot: this.gamepad.index, + }), + "controllerInput", + ); + this.wrtc.sendBinary(toBinary(ProtoMessageSchema, detachMsg)); + } } private controllerButtonToVirtualKeyCode(code: number): number | undefined { @@ -487,7 +495,7 @@ export class Controller { // Check if this rumble is for us if ( - rumbleMsg.sessionId !== this.wrtc.getSessionID() && + rumbleMsg.sessionId !== this.wrtc.getSessionID() || rumbleMsg.sessionSlot !== this.gamepad.index ) return; diff --git a/packages/input/src/webrtc-stream.ts b/packages/input/src/webrtc-stream.ts index 8b43627b..0b90faf2 100644 --- a/packages/input/src/webrtc-stream.ts +++ b/packages/input/src/webrtc-stream.ts @@ -172,7 +172,7 @@ export class WebRTCStream { const requestMsg = createMessage( create(ProtoClientRequestRoomStreamSchema, { roomName: roomName, - sessionId: clientId, + sessionId: clientId ?? "", }), "request-stream-room", ); diff --git a/packages/relay/internal/common/ice_helper.go b/packages/relay/internal/common/ice_helper.go new file mode 100644 index 00000000..d519fd1c --- /dev/null +++ b/packages/relay/internal/common/ice_helper.go @@ -0,0 +1,53 @@ +package common + +import ( + "log/slog" + + "github.com/pion/webrtc/v4" +) + +// ICEHelper holds webrtc.ICECandidateInit(s) until remote candidate is set for given webrtc.PeerConnection +// Held candidates should be flushed at the end of negotiation to ensure all are available for connection +type ICEHelper struct { + candidates []webrtc.ICECandidateInit + pc *webrtc.PeerConnection +} + +func NewICEHelper(pc *webrtc.PeerConnection) *ICEHelper { + return &ICEHelper{ + pc: pc, + candidates: make([]webrtc.ICECandidateInit, 0), + } +} + +func (ice *ICEHelper) SetPeerConnection(pc *webrtc.PeerConnection) { + ice.pc = pc +} + +func (ice *ICEHelper) AddCandidate(c webrtc.ICECandidateInit) { + if ice.pc != nil { + if ice.pc.RemoteDescription() != nil { + // Add immediately if remote is set + if err := ice.pc.AddICECandidate(c); err != nil { + slog.Error("Failed to add ICE candidate", "err", err) + } + // Also flush held candidates automatically + ice.FlushHeldCandidates() + } else { + // Hold in slice until remote is set + ice.candidates = append(ice.candidates, c) + } + } +} + +func (ice *ICEHelper) FlushHeldCandidates() { + if ice.pc != nil && len(ice.candidates) > 0 { + for _, heldCandidate := range ice.candidates { + if err := ice.pc.AddICECandidate(heldCandidate); err != nil { + slog.Error("Failed to add held ICE candidate", "err", err) + } + } + // Clear the held candidates + ice.candidates = make([]webrtc.ICECandidateInit, 0) + } +} diff --git a/packages/relay/internal/core/protocol_stream.go b/packages/relay/internal/core/protocol_stream.go index e7566ed9..925eb4bf 100644 --- a/packages/relay/internal/core/protocol_stream.go +++ b/packages/relay/internal/core/protocol_stream.go @@ -71,7 +71,7 @@ func (sp *StreamProtocol) handleStreamRequest(stream network.Stream) { safeBRW := common.NewSafeBufioRW(brw) var currentRoomName string // Track the current room for this stream - iceHolder := make([]webrtc.ICECandidateInit, 0) + iceHelper := common.NewICEHelper(nil) for { var msgWrapper gen.ProtoMessage err := safeBRW.ReceiveProto(&msgWrapper) @@ -177,6 +177,7 @@ func (sp *StreamProtocol) handleStreamRequest(stream network.Stream) { // Assign peer connection participant.PeerConnection = pc + iceHelper.SetPeerConnection(pc) // Add audio/video tracks { @@ -344,29 +345,7 @@ func (sp *StreamProtocol) handleStreamRequest(stream network.Stream) { SDPMLineIndex: &smollified, UsernameFragment: iceMsg.Candidate.UsernameFragment, } - // Use currentRoomName to get the connection from nested map - if len(currentRoomName) > 0 { - if roomMap, ok := sp.servedConns.Get(currentRoomName); ok { - if conn, ok := roomMap.Get(stream.Conn().RemotePeer()); ok && conn.pc.RemoteDescription() != nil { - if err = conn.pc.AddICECandidate(cand); err != nil { - slog.Error("Failed to add ICE candidate", "err", err) - } - for _, heldIce := range iceHolder { - if err := conn.pc.AddICECandidate(heldIce); err != nil { - slog.Error("Failed to add held ICE candidate", "err", err) - } - } - // Clear the held candidates - iceHolder = make([]webrtc.ICECandidateInit, 0) - } else { - // Hold the candidate until remote description is set - iceHolder = append(iceHolder, cand) - } - } - } else { - // Hold the candidate until remote description is set - iceHolder = append(iceHolder, cand) - } + iceHelper.AddCandidate(cand) } else { slog.Error("Could not GetIce from ice-candidate") } @@ -386,6 +365,8 @@ func (sp *StreamProtocol) handleStreamRequest(stream network.Stream) { continue } slog.Debug("Set remote description for answer") + // Flush held candidates now if missed before (race-condition) + iceHelper.FlushHeldCandidates() } else { slog.Warn("Received answer without active PeerConnection") } @@ -406,7 +387,7 @@ func (sp *StreamProtocol) handleStreamPush(stream network.Stream) { safeBRW := common.NewSafeBufioRW(brw) var room *shared.Room - iceHolder := make([]webrtc.ICECandidateInit, 0) + iceHelper := common.NewICEHelper(nil) for { var msgWrapper gen.ProtoMessage err := safeBRW.ReceiveProto(&msgWrapper) @@ -483,21 +464,7 @@ func (sp *StreamProtocol) handleStreamPush(stream network.Stream) { SDPMLineIndex: &smollified, UsernameFragment: iceMsg.Candidate.UsernameFragment, } - if conn, ok := sp.incomingConns.Get(room.Name); ok && conn.pc.RemoteDescription() != nil { - if err = conn.pc.AddICECandidate(cand); err != nil { - slog.Error("Failed to add ICE candidate for pushed stream", "err", err) - } - for _, heldIce := range iceHolder { - if err = conn.pc.AddICECandidate(heldIce); err != nil { - slog.Error("Failed to add held ICE candidate for pushed stream", "err", err) - } - } - // Clear the held candidates - iceHolder = make([]webrtc.ICECandidateInit, 0) - } else { - // Hold the candidate until remote description is set - iceHolder = append(iceHolder, cand) - } + iceHelper.AddCandidate(cand) } else { slog.Error("Failed to GetIce in pushed stream ice-candidate") } @@ -529,6 +496,7 @@ func (sp *StreamProtocol) handleStreamPush(stream network.Stream) { // Assign room peer connection room.PeerConnection = pc + iceHelper.SetPeerConnection(pc) pc.OnDataChannel(func(dc *webrtc.DataChannel) { // TODO: Is this the best way to handle DataChannel? Should we just use the map directly? @@ -689,6 +657,8 @@ func (sp *StreamProtocol) handleStreamPush(stream network.Stream) { continue } slog.Debug("Set remote description for pushed stream", "room", room.Name) + // Flush candidates now if they weren't before (race-condition) + iceHelper.FlushHeldCandidates() // Create an answer answer, err := pc.CreateAnswer(nil) diff --git a/packages/relay/internal/shared/participant.go b/packages/relay/internal/shared/participant.go index 74b87d60..9fef6ad1 100644 --- a/packages/relay/internal/shared/participant.go +++ b/packages/relay/internal/shared/participant.go @@ -63,13 +63,13 @@ func (p *Participant) SetTrack(trackType webrtc.RTPCodecType, track *webrtc.Trac p.AudioTrack = track _, err := p.PeerConnection.AddTrack(track) if err != nil { - slog.Error("Failed to add Participant audio track", err) + slog.Error("Failed to add Participant audio track", "participant", p.ID, "err", err) } case webrtc.RTPCodecTypeVideo: p.VideoTrack = track _, err := p.PeerConnection.AddTrack(track) if err != nil { - slog.Error("Failed to add Participant video track", err) + slog.Error("Failed to add Participant video track", "participant", p.ID, "err", err) } default: slog.Warn("Unknown track type", "participant", p.ID, "trackType", trackType) @@ -78,6 +78,9 @@ func (p *Participant) SetTrack(trackType webrtc.RTPCodecType, track *webrtc.Trac // Close cleans up participant resources func (p *Participant) Close() { + p.closeOnce.Do(func() { + close(p.packetQueue) + }) if p.DataChannel != nil { err := p.DataChannel.Close() if err != nil { From 9bee9d49351de1bdec0d86d439e7c53448958714 Mon Sep 17 00:00:00 2001 From: DatCaptainHorse Date: Fri, 7 Nov 2025 10:01:22 +0200 Subject: [PATCH 5/6] Some rabbit nitpick fixes 2 --- packages/input/src/webrtc-stream.ts | 10 ++++++++-- packages/relay/internal/core/protocol_stream.go | 14 ++++++++++---- packages/relay/internal/shared/participant.go | 8 ++++---- 3 files changed, 22 insertions(+), 10 deletions(-) diff --git a/packages/input/src/webrtc-stream.ts b/packages/input/src/webrtc-stream.ts index 0b90faf2..8d202034 100644 --- a/packages/input/src/webrtc-stream.ts +++ b/packages/input/src/webrtc-stream.ts @@ -123,8 +123,6 @@ export class WebRTCStream { } else { iceHolder.push(cand); } - } else { - iceHolder.push(cand); } }); @@ -143,6 +141,14 @@ export class WebRTCStream { sdp: data.sdp.sdp, type: data.sdp.type as RTCSdpType, }); + // Add held candidates + iceHolder.forEach((candidate) => { + this._pc!.addIceCandidate(candidate).catch((err) => { + console.error("Error adding held ICE candidate:", err); + }); + }); + iceHolder = []; + // Create our answer const answer = await this._pc!.createAnswer(); // Force stereo in Chromium browsers diff --git a/packages/relay/internal/core/protocol_stream.go b/packages/relay/internal/core/protocol_stream.go index 925eb4bf..4832f726 100644 --- a/packages/relay/internal/core/protocol_stream.go +++ b/packages/relay/internal/core/protocol_stream.go @@ -271,12 +271,16 @@ func (sp *StreamProtocol) handleStreamRequest(stream network.Stream) { } candInit := candidate.ToJSON() - biggified := uint32(*candInit.SDPMLineIndex) + var sdpMLineIndex *uint32 + if candInit.SDPMLineIndex != nil { + idx := uint32(*candInit.SDPMLineIndex) + sdpMLineIndex = &idx + } iceMsg, err := common.CreateMessage( &gen.ProtoICE{ Candidate: &gen.RTCIceCandidateInit{ Candidate: candInit.Candidate, - SdpMLineIndex: &biggified, + SdpMLineIndex: sdpMLineIndex, SdpMid: candInit.SDPMid, }, }, @@ -338,13 +342,15 @@ func (sp *StreamProtocol) handleStreamRequest(stream network.Stream) { case "ice-candidate": iceMsg := msgWrapper.GetIce() if iceMsg != nil { - smollified := uint16(*iceMsg.Candidate.SdpMLineIndex) cand := webrtc.ICECandidateInit{ Candidate: iceMsg.Candidate.Candidate, SDPMid: iceMsg.Candidate.SdpMid, - SDPMLineIndex: &smollified, UsernameFragment: iceMsg.Candidate.UsernameFragment, } + if iceMsg.Candidate.SdpMLineIndex != nil { + smollified := uint16(*iceMsg.Candidate.SdpMLineIndex) + cand.SDPMLineIndex = &smollified + } iceHelper.AddCandidate(cand) } else { slog.Error("Could not GetIce from ice-candidate") diff --git a/packages/relay/internal/shared/participant.go b/packages/relay/internal/shared/participant.go index 9fef6ad1..f9db6637 100644 --- a/packages/relay/internal/shared/participant.go +++ b/packages/relay/internal/shared/participant.go @@ -63,13 +63,13 @@ func (p *Participant) SetTrack(trackType webrtc.RTPCodecType, track *webrtc.Trac p.AudioTrack = track _, err := p.PeerConnection.AddTrack(track) if err != nil { - slog.Error("Failed to add Participant audio track", "participant", p.ID, "err", err) + slog.Error("Failed to add audio track", "participant", p.ID, "err", err) } case webrtc.RTPCodecTypeVideo: p.VideoTrack = track _, err := p.PeerConnection.AddTrack(track) if err != nil { - slog.Error("Failed to add Participant video track", "participant", p.ID, "err", err) + slog.Error("Failed to add video track", "participant", p.ID, "err", err) } default: slog.Warn("Unknown track type", "participant", p.ID, "trackType", trackType) @@ -84,14 +84,14 @@ func (p *Participant) Close() { if p.DataChannel != nil { err := p.DataChannel.Close() if err != nil { - slog.Error("Failed to close Participant DataChannel", err) + slog.Error("Failed to close DataChannel", "participant", p.ID, "err", err) } p.DataChannel = nil } if p.PeerConnection != nil { err := p.PeerConnection.Close() if err != nil { - slog.Error("Failed to close Participant PeerConnection", err) + slog.Error("Failed to close PeerConnection", "participant", p.ID, "err", err) } p.PeerConnection = nil } From 9576327863552f5bd3669f92e461d3aa7ab459b1 Mon Sep 17 00:00:00 2001 From: DatCaptainHorse Date: Fri, 7 Nov 2025 15:50:42 +0200 Subject: [PATCH 6/6] Rabbit nitpicks 3 and new MangoHud config --- containerfiles/runner.Containerfile | 5 ++ packages/configs/MangoHud/MangoHud.conf | 48 +++++++++++++++++++ .../relay/internal/core/protocol_stream.go | 4 +- packages/scripts/envs.sh | 3 -- 4 files changed, 55 insertions(+), 5 deletions(-) create mode 100644 packages/configs/MangoHud/MangoHud.conf diff --git a/containerfiles/runner.Containerfile b/containerfiles/runner.Containerfile index 7e0de173..7cdc6c4f 100644 --- a/containerfiles/runner.Containerfile +++ b/containerfiles/runner.Containerfile @@ -72,6 +72,11 @@ RUN mkdir -p "${NESTRI_HOME}/.local/share/Steam/config" COPY packages/configs/steam/config.vdf "${NESTRI_HOME}/.local/share/Steam/config/" +## MangoHud Config ## +RUN mkdir -p "${NESTRI_HOME}/.config/MangoHud" + +COPY packages/configs/MangoHud/MangoHud.conf "${NESTRI_HOME}/.config/MangoHud/" + ### Artifacts from Builder ### COPY --from=builder /artifacts/bin/nestri-server /usr/bin/ COPY --from=builder /artifacts/bin/bwrap /usr/bin/ diff --git a/packages/configs/MangoHud/MangoHud.conf b/packages/configs/MangoHud/MangoHud.conf new file mode 100644 index 00000000..41a1cf48 --- /dev/null +++ b/packages/configs/MangoHud/MangoHud.conf @@ -0,0 +1,48 @@ +legacy_layout=false + +# common +horizontal +horizontal_stretch +hud_no_margin +no_small_font +background_alpha=0.66 +round_corners=0 +background_color=000000 +font_size=24 +position=top-left +engine_short_names + +# colors +text_color=DFDFDF +gpu_color=FF4E00 +cpu_color=00AA00 +engine_color=00AA00 +vram_color=00AA00 +ram_color=00AA00 +frametime_color=FF4E00 + +# load colors +cpu_load_color=DFDFDF,DF964D,DF3D3D +gpu_load_color=DFDFDF,DF964D,DF3D3D + +# GPU and VRAM +gpu_text=NESTRI +gpu_stats +gpu_load_change +gpu_load_value=70,90 + +vram + +# CPU and RAM +cpu_text=CPU +cpu_stats +cpu_load_change +cpu_load_value=70,90 + +ram + +# FPS and timing +fps +fps_metrics=0.01 + +frame_timing diff --git a/packages/relay/internal/core/protocol_stream.go b/packages/relay/internal/core/protocol_stream.go index 4832f726..9c847ba0 100644 --- a/packages/relay/internal/core/protocol_stream.go +++ b/packages/relay/internal/core/protocol_stream.go @@ -402,7 +402,7 @@ func (sp *StreamProtocol) handleStreamPush(stream network.Stream) { slog.Debug("Stream push connection closed by peer", "peer", stream.Conn().RemotePeer(), "error", err) if room != nil { room.Close() - sp.incomingConns.Set(room.Name, nil) + sp.incomingConns.Delete(room.Name) } return } @@ -411,7 +411,7 @@ func (sp *StreamProtocol) handleStreamPush(stream network.Stream) { _ = stream.Reset() if room != nil { room.Close() - sp.incomingConns.Set(room.Name, nil) + sp.incomingConns.Delete(room.Name) } return } diff --git a/packages/scripts/envs.sh b/packages/scripts/envs.sh index 2542da6c..43650f17 100644 --- a/packages/scripts/envs.sh +++ b/packages/scripts/envs.sh @@ -10,9 +10,6 @@ export DISPLAY=:0 # Causes some setups to break export PROTON_NO_FSYNC=1 -# Sleeker Mangohud preset :) -export MANGOHUD_CONFIG=preset=2 - # Make gstreamer GL elements work without display output (NVIDIA issue..) export GST_GL_API=gles2 export GST_GL_WINDOW=surfaceless