Current URL Packet and On-screen Keyboard

src/broadcaster.ts

@@ -1,5 +1,5 @@
 import { WebSocket } from "ws";
-import { buildFrameStatsPacket, buildFramePackets } from "./protocol.js";
+import { buildFrameStatsPacket, buildFramePackets, buildCurrentURLPacket } from "./protocol.js";
 import type { FrameOut } from "./frameProcessor.js";
 
 type OutFrame = { frameId: number; packets: Buffer[] };
@@ -62,6 +62,20 @@ export class DeviceBroadcaster {
     this._drainAsync(id).catch(() => {});
   }
 
+  // Send packet with current URL info to connected client:
+  public sendCurrentURL(id: string, url: string): void {
+    const peers = this._clients.get(id);
+    if (!peers || peers.size === 0) return;
+
+    // We use the URL packet packer from protocol.js here
+    const packet = buildCurrentURLPacket(url);
+    const st = this._ensureState(id);
+
+    // We use frameId: 0 since this is a control packet, not an image frame
+    st.queue.push({ frameId: 0, packets: [packet] });
+    this._drainAsync(id).catch(() => {});
+  }
+
   private _ensureState(id: string): BroadcasterState {
     let st = this._state.get(id);
     if (!st) {

src/deviceManager.ts

@@ -73,6 +73,10 @@ export async function ensureDeviceAsync(id: string, cfg: DeviceConfig): Promise<
     });
   }
 
+  // On-screen keyboard
+  const kioskKeyboardScript = "(function(){console.log('[VKB] Script injected. Initializing...');const VKB_WIDTH='100%';const VKB_HEIGHT='196px';if(window.__kioskKeyboardInitialized){console.log('[VKB] Already initialized. Aborting duplicate injection.');return;}window.__kioskKeyboardInitialized=true;let keyboardContainer=null;let currentLayout='default';let activeInput=null;let isShifted=false;const layouts={default:[['q','w','e','r','t','y','u','i','o','p'],['a','s','d','f','g','h','j','k','l'],['⇧','z','x','c','v','b','n','m','⌫'],['▼','?123',',','◀','Space','▶','.','⏎']],shift:[['Q','W','E','R','T','Y','U','I','O','P'],['A','S','D','F','G','H','J','K','L'],['⇧','Z','X','C','V','B','N','M','⌫'],['▼','?123',',','◀','Space','▶','.','⏎']],symbols:[['1','2','3','4','5','6','7','8','9','0'],['@','#','$','%','&','*','-','+','(',')'],['ABC','!','\"',\"'\",':',';','/','?','⌫'],['▼','=\\\\<',',','◀','Space','▶','.','⏎']],extended:[['~','|','^','_','=','{','}','[',']','✓'],['<','>','£','€','¢','°','±','÷','×','\\\\'],['?123','↹','©','®','™','¿','¡','§','⌫'],['▼','ABC',',','◀','Space','▶','.','⏎']]};function ensureDOM(){if(!document.body||!document.head){console.warn('[VKB] document.body or head not ready.');return false;}if(!document.getElementById('kiosk-vkb-style')){console.log('[VKB] Injecting CSS overrides.');const style=document.createElement('style');style.id='kiosk-vkb-style';style.textContent=`#kiosk-vkb-container{position:fixed !important;top:auto !important;bottom:-200vh !important;left:0 !important;right:0 !important;margin:0 auto !important;width:${VKB_WIDTH} !important;height:${VKB_HEIGHT} !important;container-type:size;background:#1e1e1e;border-top:2px solid #333;z-index:2147483647;display:flex;flex-direction:column;padding:4px;box-sizing:border-box;user-select:none;-webkit-user-select:none;font-family:'DejaVu Sans','Liberation Sans',Ubuntu,Roboto,sans-serif;touch-action:manipulation;border:none;}#kiosk-vkb-container:popover-open{display:flex;}#kiosk-vkb-container.vkb-visible{bottom:0 !important;}.vkb-row{display:flex;justify-content:center;margin-bottom:4px;width:100%;gap:4px;flex:1;}.vkb-row:last-child{margin-bottom:0;}.vkb-key{flex:1;background:#383838;color:#f8f8f2;border:1px solid #2a2a2a;border-radius:2px;font-size:11.5cqh;font-weight:normal;cursor:pointer;display:flex;align-items:center;justify-content:center;padding:0;}.vkb-key:active{background:#555555;}.vkb-key-layout{background:#324a5f;color:#e2e8f0;font-size:9cqh;}.vkb-key-layout:active{background:#233544;}.vkb-key-special{background:#485c4a;color:#e2e8f0;font-size:11cqh;}.vkb-key-special:active{background:#364538;}.vkb-key-large-icon{font-size:15cqh;}.vkb-key-backspace{font-size:18cqh;}.vkb-key-hide{background:#8b3a3a;color:#e2e8f0;font-size:12.5cqh;}.vkb-key-hide:active{background:#6b2a2a;}.vkb-key-enter{background:#E95420;color:#ffffff;border-color:#c94618;font-size:12.5cqh;}.vkb-key-enter:active{background:#c94618;}.vkb-key-space{flex:3;}.vkb-key-arrow{flex:0.8;}`;document.head.appendChild(style);}if(!keyboardContainer){console.log('[VKB] Creating keyboard DOM elements.');keyboardContainer=document.createElement('div');keyboardContainer.id='kiosk-vkb-container';if(keyboardContainer.popover!==undefined)keyboardContainer.popover='manual';renderKeyboard();}if(!document.body.contains(keyboardContainer)){console.log('[VKB] Appending keyboard to body.');document.body.appendChild(keyboardContainer);}return true;}function renderKeyboard(){if(!keyboardContainer)return;keyboardContainer.innerHTML='';const layout=layouts[currentLayout];layout.forEach(row=>{const rowDiv=document.createElement('div');rowDiv.className='vkb-row';row.forEach(key=>{const keyBtn=document.createElement('button');keyBtn.className='vkb-key';keyBtn.textContent=key==='Space'?'':key;keyBtn.dataset.key=key;if(['?123','ABC','=\\\\<'].includes(key)){keyBtn.classList.add('vkb-key-layout');}if(['⇧','⌫','◀','▶','↹'].includes(key)){keyBtn.classList.add('vkb-key-special');}if(['⇧','↹'].includes(key)){keyBtn.classList.add('vkb-key-large-icon');}if(key==='⌫'){keyBtn.classList.add('vkb-key-backspace');}if(key==='▼'){keyBtn.classList.add('vkb-key-hide');}if(key==='Space'){keyBtn.classList.add('vkb-key-space');}if(key==='◀'||key==='▶'){keyBtn.classList.add('vkb-key-arrow');}if(key==='⏎'){keyBtn.classList.add('vkb-key-enter');}if(key==='⇧'&&isShifted){keyBtn.style.background='#e2e8f0';keyBtn.style.color='#121212';}rowDiv.appendChild(keyBtn);});keyboardContainer.appendChild(rowDiv);});}function processKey(key){if(!activeInput){console.warn('[VKB] Key pressed but activeInput is null.');return;}console.log('[VKB] Processing key:',key);if(typeof activeInput.focus==='function'){activeInput.focus();}const insertText=(text)=>{if(activeInput.isContentEditable){document.execCommand('insertText',false,text);}else{let val=activeInput.value||'';let start=activeInput.selectionStart||0;let end=activeInput.selectionEnd||0;activeInput.value=val.substring(0,start)+text+val.substring(end);activeInput.selectionStart=activeInput.selectionEnd=start+text.length;}};switch(key){case'▼':hideKeyboard();break;case'⇧':isShifted=!isShifted;currentLayout=isShifted?'shift':'default';renderKeyboard();break;case'?123':currentLayout='symbols';isShifted=false;renderKeyboard();break;case'ABC':currentLayout='default';isShifted=false;renderKeyboard();break;case'=\\\\<':currentLayout='extended';isShifted=false;renderKeyboard();break;case'↹':insertText('\\t');break;case'⌫':if(activeInput.isContentEditable){document.execCommand('delete',false,null);}else{let val=activeInput.value||'';let start=activeInput.selectionStart||0;let end=activeInput.selectionEnd||0;if(start===end&&start>0){activeInput.value=val.substring(0,start-1)+val.substring(end);activeInput.selectionStart=activeInput.selectionEnd=start-1;}else if(start!==end){activeInput.value=val.substring(0,start)+val.substring(end);activeInput.selectionStart=activeInput.selectionEnd=start;}}break;case'Space':insertText(' ');break;case'◀':if(!activeInput.isContentEditable){let start=activeInput.selectionStart||0;if(start>0)activeInput.selectionStart=activeInput.selectionEnd=start-1;}break;case'▶':if(!activeInput.isContentEditable){let end=activeInput.selectionEnd||0;let valLen=(activeInput.value||'').length;if(end<valLen)activeInput.selectionStart=activeInput.selectionEnd=end+1;}break;case'⏎':if(activeInput.isContentEditable){document.execCommand('insertParagraph',false,null);activeInput.dispatchEvent(new Event('input',{bubbles:true,composed:true}));}else if(activeInput.tagName==='TEXTAREA'){insertText('\\n');activeInput.dispatchEvent(new Event('input',{bubbles:true,composed:true}));activeInput.dispatchEvent(new Event('change',{bubbles:true,composed:true}));}else{const evInit={key:'Enter',code:'Enter',keyCode:13,which:13,bubbles:true,composed:true,cancelable:true};activeInput.dispatchEvent(new KeyboardEvent('keydown',evInit));activeInput.dispatchEvent(new KeyboardEvent('keypress',evInit));activeInput.dispatchEvent(new KeyboardEvent('keyup',evInit));hideKeyboard();}break;default:if(key){insertText(key);if(isShifted){isShifted=false;currentLayout='default';renderKeyboard();}}break;}if(key!=='⏎'&&key!=='▼'){activeInput.dispatchEvent(new Event('input',{bubbles:true,composed:true}));activeInput.dispatchEvent(new Event('change',{bubbles:true,composed:true}));}}function showKeyboard(inputElement){console.log('[VKB] showKeyboard triggered for:',inputElement);activeInput=inputElement;renderKeyboard();window.__vkbOpeningShield=Date.now();if(keyboardContainer.showPopover){if(keyboardContainer.matches(':popover-open')){keyboardContainer.hidePopover();}keyboardContainer.showPopover();}keyboardContainer.classList.add('vkb-visible');if(activeInput&&activeInput.scrollIntoView){activeInput.scrollIntoView({behavior:'auto',block:'center'});}}function hideKeyboard(){console.log('[VKB] hideKeyboard triggered. Activating ghost-click shield.');window.__vkbClosingShield=Date.now();if(keyboardContainer){keyboardContainer.classList.remove('vkb-visible');if(keyboardContainer.hidePopover&&keyboardContainer.matches(':popover-open')){keyboardContainer.hidePopover();}}if(activeInput&&activeInput.blur){activeInput.blur();}activeInput=null;isShifted=false;currentLayout='default';}const validTypes=['text','email','number','password','search','tel','url'];function resolveInputFromPath(path){for(let i=0;i<path.length;i++){let el=path[i];if(!el||!el.tagName)continue;let t=el.tagName.toUpperCase();if(t==='INPUT'&&validTypes.includes(el.type)){return el;}if(t==='TEXTAREA'||el.isContentEditable||(el.classList&&el.classList.contains('cm-content'))){return el;}if(t==='HA-TEXTFIELD'||t==='HA-SEARCH-INPUT'||t==='HA-CODE-EDITOR'||t==='HA-SELECTOR-TEXT'){let inner=el.shadowRoot?el.shadowRoot.querySelector('input, textarea, [contenteditable=\"true\"], .cm-content'):null;if(inner){return inner;}}}return null;}function checkAndShowKeyboard(e){const path=e.composedPath?e.composedPath():[e.target];const targetInput=resolveInputFromPath(path);if(targetInput){console.log('[VKB] Valid DOM element found via',e.type,':',targetInput);if(ensureDOM()){const isVisible=keyboardContainer&&keyboardContainer.classList.contains('vkb-visible');if(activeInput!==targetInput||!isVisible){showKeyboard(targetInput);}else{console.log('[VKB] Element already active and visible. Ignoring.');}}}else{if(e.type==='focusin')console.log('[VKB] focusin ignored: Target is not a valid input.');}}document.addEventListener('focusin',checkAndShowKeyboard,true);document.addEventListener('click',checkAndShowKeyboard,true);const interactionEvents=['pointerdown','pointerup','mousedown','mouseup','click','touchstart','touchend'];interactionEvents.forEach(ev=>{document.addEventListener(ev,function(e){if(window.__vkbClosingShield&&(Date.now()-window.__vkbClosingShield<400)){e.preventDefault();e.stopPropagation();e.stopImmediatePropagation();return;}if(keyboardContainer&&keyboardContainer.classList.contains('vkb-visible')){let x=e.clientX;let y=e.clientY;if(x===undefined&&e.changedTouches&&e.changedTouches.length>0){x=e.changedTouches[0].clientX;y=e.changedTouches[0].clientY;}if(x===undefined||y===undefined)return;const rect=keyboardContainer.getBoundingClientRect();if(y>=rect.top&&y<=rect.bottom&&x>=rect.left&&x<=rect.right){e.preventDefault();e.stopPropagation();e.stopImmediatePropagation();if(window.__vkbOpeningShield&&(Date.now()-window.__vkbOpeningShield<400))return;if(['pointerdown','touchstart','mousedown','click'].includes(ev)){if(window.__vkbLastTap&&(Date.now()-window.__vkbLastTap<250))return;window.__vkbLastTap=Date.now();const keys=keyboardContainer.querySelectorAll('.vkb-key');let foundKey=null;for(let i=0;i<keys.length;i++){const kRect=keys[i].getBoundingClientRect();if(y>=kRect.top&&y<=kRect.bottom&&x>=kRect.left&&x<=kRect.right){foundKey=keys[i];break;}}if(foundKey){const key=foundKey.dataset.key;foundKey.style.background='#555';setTimeout(()=>{foundKey.style.background='';},100);processKey(key);}}return;}if(ev==='pointerdown'){const path=e.composedPath?e.composedPath():[e.target];const clickedOnInput=resolveInputFromPath(path)!==null;if(!clickedOnInput){console.log('[VKB] Pointer down outside. Hiding.');hideKeyboard();}else{console.log('[VKB] Pointer down on input. Staying open.');}}}},true);});console.log('[VKB] Initialization complete (Fully Scalable, Ghost-Click Shield Active).');})();";
+  await session.send('Page.addScriptToEvaluateOnNewDocument', { source: kioskKeyboardScript });
+
   await session.send('Page.startScreencast', {
     format: 'png',
     maxWidth: cfg.width,
@@ -161,6 +165,25 @@ export async function ensureDeviceAsync(id: string, cfg: DeviceConfig): Promise<
     }
   });
 
+  // Function to deal with URL changing via either full page refresh or single page # follow
+  const handleNavigation = (url: string) => {
+    if (newDevice.url !== url) {
+      newDevice.url = url;
+      broadcaster.sendCurrentURL(newDevice.deviceId, url);
+      console.log(`[device] URL changed to: ${url}`);
+    }
+  };
+  // Triggered on full page loads
+  session.on('Page.frameNavigated', (evt: any) => {
+    if (!evt.frame.parentId) { // Only track the main frame, ignore iframes
+      handleNavigation(evt.frame.url);
+    }
+  });
+  // Triggered on Single Page App (SPA) hash or history API changes
+  session.on('Page.navigatedWithinDocument', (evt: any) => {
+    handleNavigation(evt.url);
+  });
+
   return newDevice;
 }
 
@@ -197,4 +220,4 @@ async function deleteDeviceAsync(device: DeviceSession) {
 
   try { await device.cdp.send("Page.stopScreencast").catch(() => { }); } catch { }
   try { await root?.send("Target.closeTarget", { targetId: device.id }); } catch { }
-}
+}

src/protocol.ts

@@ -16,6 +16,8 @@
 // Keepalive message:
 //   [type u8=5][ver u8=1]
 //
+// Current URL packet:
+//   [type u8][ver u8][len u32][url utf8 bytes...]
 
 export const PROTOCOL_VERSION = 1 as const;
 
@@ -26,6 +28,7 @@ export enum MsgType {
   FrameStats  = 3,
   OpenURL     = 4,
   Keepalive   = 5,
+  CurrentURL  = 6, // <-- Current URL packet
 }
 
 export enum Encoding {
@@ -75,9 +78,21 @@ export const TILE_HEADER_BYTES  = 2 + 2 + 2 + 2 + 4;      // 12
 export const TOUCH_BYTES        = 1 + 1 + 1 + 1 + 2 + 2;  // 8
 export const FRAME_STATS_BYTES  = 1 + 1 + 4 + 4;          // 10
 export const OPENURL_HEADER_BYTES = 1 + 1 + 2 + 4;        // 8
+export const CURRENTURL_HEADER_BYTES = 1 + 1 + 4;         // 6 bytes: [type u8][ver u8][len u32] - Current URL header
 
 const clampU16 = (v: number) => (v < 0 ? 0 : v > 0xffff ? 0xffff : v|0);
 
+// Current URL packet
+export function buildCurrentURLPacket(url: string): Buffer {
+  const urlBuf = Buffer.from(url, "utf8");
+  const buf = Buffer.alloc(CURRENTURL_HEADER_BYTES + urlBuf.length);
+  buf.writeUInt8(MsgType.CurrentURL, 0);
+  buf.writeUInt8(PROTOCOL_VERSION, 1);
+  buf.writeUInt32LE(urlBuf.length, 2);
+  urlBuf.copy(buf, CURRENTURL_HEADER_BYTES);
+  return buf;
+}
+
 export function buildTouchPacket(kind: TouchKind, x: number, y: number, pointerId = 0): Buffer {
   const buf = Buffer.alloc(TOUCH_BYTES);
   buf.writeUInt8(MsgType.Touch, 0);