Dev Spy

@rebindv0.2.0

See the pointer, window, and color under it while you write a script, record observations and macros, and try AI calls.

Files

Version 0.2.0 (current)

main.luau

-- Dev Spy: a HUD for writing Rebind scripts. It shows the pointer, monitor,
-- window, the color under the pointer and the input Rebind sees; records
-- observations (every input plus screenshots) and macros (keys, mouse and
-- controller only) as JSON lines; and runs rb.ai calls from a menu.
-- rebind.toml owns the metadata.

local cfg = rb.ui.schema({
  -- one chord opens the menu; the other actions live in it and can be given
  -- keys here. Letters, not F-keys: Mac laptops send media keys for F-keys
  -- unless fn is held, and software mode does not see media keys.
  menu_key = rb.ui.keybind("Ctrl+Alt+D", {
    label = "Open the menu",
    tab = "Keys",
  }),
  hud_key = rb.ui.keybind("", { label = "Hide or show the HUD", tab = "Keys" }),
  observe_key = rb.ui.keybind("", {
    label = "Start or stop an observation",
    tab = "Keys",
  }),
  record_key = rb.ui.keybind("", {
    label = "Start or stop recording a macro",
    tab = "Keys",
  }),
  copy_key = rb.ui.keybind("", {
    label = "Copy pointer and color",
    tooltip = "Puts rb.hid.move_to(x, y) and the color on the clipboard",
    tab = "Keys",
  }),

  theme = rb.ui.select(
    "rebind",
    { "rebind", "rebind-light", "catppuccin", "tokyo-night", "contrast" },
    { label = "Overlay theme", tab = "HUD" }
  ),
  refresh_ms = rb.ui.slider(100, {
    label = "HUD refresh",
    min = 100,
    max = 1000,
    step = 100,
    suffix = " ms",
    tab = "HUD",
  }),

  frame_ms = rb.ui.slider(1, {
    label = "Screenshot after idle for",
    tooltip = "Screenshots follow clicks, keys, scrolling, movement and window"
      .. " changes. This adds one after a quiet stretch; 0 turns it off.",
    min = 0,
    max = 30,
    step = 1,
    suffix = " s",
    tab = "Sessions",
  }),
  frame_edge = rb.ui.slider(1024, {
    label = "Screenshot size (long edge)",
    min = 256,
    max = 2048,
    step = 128,
    suffix = " px",
    tab = "Sessions",
  }),
  max_minutes = rb.ui.slider(30, {
    label = "Stop a recording after",
    min = 1,
    max = 240,
    suffix = " min",
    tab = "Sessions",
  }),
  macro_moves = rb.ui.toggle(true, {
    label = "Macros record mouse movement",
    tab = "Sessions",
  }),
  macro_scroll = rb.ui.toggle(true, {
    label = "Macros record scrolling",
    tab = "Sessions",
  }),
  macro_pad = rb.ui.toggle(true, {
    label = "Macros record the controller",
    tooltip = "Buttons replay; stick and trigger movement is kept in the log only",
    tab = "Sessions",
  }),
  goal = rb.ui.text("", {
    label = "What this recording is for",
    placeholder = "for example: fish until the bag is full",
    tooltip = "Saved at the start of each observation and macro recording",
    tab = "Sessions",
  }),
  save_name = rb.ui.text("", {
    label = "Name for the next save",
    placeholder = "empty: the time the recording started",
    max_length = 64,
    tab = "Sessions",
  }),

  ai_question = rb.ui.text("Is a dialog open?", {
    label = "Decide: question",
    tab = "AI",
  }),
  ai_choices = rb.ui.text("yes, no", {
    label = "Decide: choices",
    tooltip = "Separated by commas",
    tab = "AI",
  }),
  ai_find = rb.ui.text("the Save button", {
    label = "Find: what to look for",
    tab = "AI",
  }),
  ai_read = rb.ui.text("the text under the pointer", {
    label = "Read: what to read",
    tab = "AI",
  }),
  ai_window = rb.ui.text("", {
    label = "Window to look at",
    placeholder = "empty: the display under the pointer",
    tooltip = "A window title, or an app name on macOS. Read in window uses"
      .. " the active window when this is empty.",
    tab = "AI",
  }),
})

local session = require("session")
local probe = require("probe")
local hud = require("hud")
local console = require("ai")

console.init(cfg, rb.ai, hud, session)

local KEYS = { "menu_key", "hud_key", "observe_key", "record_key", "copy_key" }

-- the spy's own bindings by key, each with the modifiers its chord needs,
-- so a binding such as Ctrl+S leaves a plain S in the log; read when the
-- script starts, as its binds are (a changed key restarts the script)
local own = {}
local function read_own()
  table.clear(own)
  for _, id in KEYS do
    local parts = string.split(cfg[id], "+")
    local key = string.lower(parts[#parts])
    if key ~= "" then
      local mods = {}
      for i = 1, #parts - 1 do
        -- LCtrl and RCtrl count as ctrl, as rb.input.modifiers reports them
        local name = string.gsub(string.lower(parts[i]), "^[lr]", "")
        table.insert(mods, name)
      end
      own[key] = own[key] or {}
      table.insert(own[key], mods)
    end
  end
end

local function is_own(k)
  local chords = own[k]
  if chords == nil then
    return false
  end
  local held = rb.input.modifiers()
  for _, mods in chords do
    local all = true
    for _, m in mods do
      all = all and held[m] == true
    end
    if all then
      return true
    end
  end
  return false
end

local MENU_KEYS = { up = true, down = true, enter = true, escape = true }
-- keys whose press was left out, so their release is too
local skipped = {}

local menu = nil
local menu_size = 0
local menu_at = 0
local timer = nil
local last_tick = 0
local last_draw = -math.huge
-- the clocks' whole seconds at the last draw
local drawn_up, drawn_rec = nil, nil
local last_window, last_monitor = nil, nil
local logged_axes = {}
local motion_hooks = nil

-- keys the open menu takes: Up, Down, Enter, Escape and its item numbers
local function menu_key(k)
  if menu == nil then
    return false
  end
  local n = tonumber(k)
  return MENU_KEYS[k] or (n ~= nil and n >= 1 and n <= math.min(menu_size, 9))
end

local function short(text)
  text = tostring(text)
  if (utf8.len(text) or #text) <= 28 then
    return text
  end
  return string.sub(text, 1, (utf8.offset(text, 28) or 29) - 1) .. "…"
end

local function chars(text)
  local n = utf8.len(text) or #text
  return n == 0 and "empty" or n .. (n == 1 and " character" or " characters")
end

local function open(items, title, subtitle)
  for _, item in items do
    if item.hint ~= nil then
      item.hint = short(item.hint)
    end
  end
  local ok, m, why = pcall(rb.overlay.menu, items, {
    title = title,
    subtitle = subtitle,
    theme = cfg.theme,
    on_close = function()
      menu = nil
    end,
  })
  if not ok or m == nil then
    hud.toast(tostring(ok and why or m), "menu not shown")
    return
  end
  menu, menu_size = m, #items
end

local function redraw()
  last_draw = -math.huge
end

-- movement and scrolling are watched only while recording: each hook costs
-- every mouse event a call into the script
local function watch_motion(on)
  if on and motion_hooks == nil then
    motion_hooks = {
      rb.on("input_move", function(dx, dy)
        session.move(dx, dy)
      end),
      rb.on("input_scroll", function(delta, device)
        probe.input("scroll " .. delta, device)
        session.scroll(delta, device)
      end),
    }
  elseif not on and motion_hooks ~= nil then
    for _, hook in motion_hooks do
      hook:stop()
    end
    motion_hooks = nil
  end
end

local function stop(reason)
  watch_motion(false)
  if session.stop(reason) ~= nil then
    redraw()
  end
end

local function toggle(mode)
  local s = session.live
  if s == nil then
    local started, err = session.start(mode)
    if started == nil then
      hud.toast(err, "not started")
      return
    end
    watch_motion(true)
    -- a stick already moved when recording starts is logged on the next tick
    table.clear(logged_axes)
    redraw()
  elseif s.mode == mode then
    stop("stopped")
  else
    local running = s.mode == "observe" and "An observation"
      or "A macro recording"
    hud.toast(running .. " is running. Stop it first.", "busy")
  end
end

local function toggle_hud()
  hud.hidden = not hud.hidden
  redraw()
end

local function copy()
  local r = probe.read(true, hud.covers, false)
  if r.covered then
    hud.toast("Move the pointer off Dev Spy's HUD to copy it.", "not copied")
    return
  end
  local text =
    string.format("rb.hid.move_to(%d, %d) -- #%s", r.x, r.y, r.hex or "??????")
  local ok, err = pcall(rb.clipboard.set, text)
  hud.toast(ok and text or tostring(err), ok and "copied" or "not copied")
end

-- the full path of the data folder that @data/ names
local function copy_data()
  local found, dir = pcall(rb.file.data_dir)
  if not found then
    hud.toast(tostring(dir), "not copied")
    return
  end
  local ok, err = pcall(rb.clipboard.set, dir)
  hud.toast(ok and dir or tostring(err), ok and "copied" or "not copied")
end

local function total(s)
  return hud.size((s.log_bytes or 0) + (s.frame_bytes or 0))
end

local function summary(s)
  local parts = {}
  if s.events ~= nil then
    table.insert(parts, hud.commas(s.events) .. " events")
  end
  if s.mode == "observe" then
    table.insert(parts, hud.commas(s.frames) .. " screenshots")
  end
  table.insert(parts, total(s))
  return table.concat(parts, "  ")
end

local main_menu, save_menu

-- moving a folder and writing a macro run outside the key that asked for
-- them, so the key returns at once
local function save(s, target, name)
  rb.run(function()
    local ok, path, script, why = pcall(session.save, s, target, name)
    if not ok then
      hud.toast(tostring(path), "not saved")
      return
    end
    if cfg.save_name ~= "" then
      cfg.save_name = ""
    end
    if target == "macros" then
      local first = script or ".luau not written: " .. tostring(why)
      hud.toast(first .. "\n" .. path .. "/  " .. summary(s), "saved")
    else
      hud.toast(path .. "/\n" .. summary(s), "saved")
    end
    redraw()
  end)
end

local function discard_menu(s)
  open({
    { label = "Keep it", action = save_menu },
    {
      label = "Discard for good",
      hint = total(s),
      action = function()
        rb.run(function()
          session.discard(s)
          hud.toast(s.dir .. " deleted", "discarded")
          redraw()
        end)
      end,
    },
  }, "Discard recording", s.id)
end

save_menu = function()
  local s = session.unsaved[#session.unsaved]
  if s == nil then
    return main_menu()
  end
  -- a name from the clipboard belongs to this recording only
  local name = session.clean(s.name or cfg.save_name)
  if name == "" then
    name = s.id
  end
  local ok, text = pcall(rb.clipboard.get)
  local clip = session.clean(ok and text or "")
  open({
    {
      label = "Save to observations/",
      hint = session.free("observations", name) .. "/",
      action = function()
        save(s, "observations", name)
      end,
    },
    {
      label = "Save to macros/",
      hint = session.free("macros", name) .. ".luau",
      action = function()
        save(s, "macros", name)
      end,
    },
    {
      -- the length only: the clipboard may hold something private
      label = "Name from clipboard",
      hint = chars(clip),
      action = function()
        if clip ~= "" then
          s.name = clip
        end
        save_menu()
      end,
    },
    {
      label = "Discard",
      hint = total(s),
      action = function()
        discard_menu(s)
      end,
    },
    {
      label = "Back",
      action = function()
        main_menu(true)
      end,
    },
  }, "Save recording", name)
end

local function ai_menu()
  open(
    console.items(function()
      main_menu(true)
    end, function()
      ai_menu()
    end),
    "AI console",
    "rb.ai"
  )
end

-- an unsaved recording opens Save first, unless Back came from there
main_menu = function(from_save)
  local s = session.live
  if s == nil and not from_save and #session.unsaved > 0 then
    return save_menu()
  end
  local items = {}
  local subtitle = "idle"
  if s == nil then
    table.insert(items, {
      label = "Start observation",
      hint = cfg.observe_key,
      action = function()
        toggle("observe")
      end,
    })
    table.insert(items, {
      label = "Record macro",
      hint = cfg.record_key,
      action = function()
        toggle("macro")
      end,
    })
  else
    local observe = s.mode == "observe"
    subtitle = (observe and "observing " or "recording ")
      .. hud.clock(rb.system.time() - s.started)
    -- a mark is timed when the menu opened, not when an item is picked
    local function mark(label, shown)
      return function()
        -- a recording stopped or started while the menu was open
        if session.live ~= s then
          return
        end
        local at = menu_at - s.started
        session.note({ kind = "mark", label = label, at = at })
        s.marks = (s.marks or 0) + 1
        hud.toast(
          "at " .. hud.clock(at) .. (shown and "  " .. shown or ""),
          "marked"
        )
      end
    end
    local ok, text = pcall(rb.clipboard.get)
    local clip = session.label(ok and text or "")
    table.insert(items, { label = "Mark this moment", action = mark(nil) })
    table.insert(
      items,
      { label = "Mark success", action = mark("success", "success") }
    )
    table.insert(
      items,
      { label = "Mark failure", action = mark("failure", "failure") }
    )
    table.insert(items, {
      label = "Mark with clipboard text",
      hint = chars(clip),
      action = mark(
        clip ~= "" and clip or nil,
        clip ~= "" and chars(clip) or nil
      ),
    })
    table.insert(items, {
      label = observe and "Stop observation" or "Stop recording",
      hint = observe and cfg.observe_key or cfg.record_key,
      action = function()
        stop("stopped")
      end,
    })
  end
  table.insert(items, {
    label = hud.hidden and "Show HUD" or "Hide HUD",
    hint = cfg.hud_key,
    action = toggle_hud,
  })
  table.insert(items, { label = "AI console", hint = "Pro", action = ai_menu })
  table.insert(items, {
    label = "Copy pointer and color",
    hint = cfg.copy_key,
    action = copy,
  })
  table.insert(items, { label = "Copy data folder path", action = copy_data })
  for _, item in items do
    if item.hint == "" then
      item.hint = nil
    end
  end
  open(items, "Dev Spy", subtitle)
end

local function tick()
  local t = rb.system.time()
  probe.interval, last_tick = t - last_tick, t
  local live = session.live
  if live ~= nil then
    local why = session.tick()
    if why ~= nil then
      stop(why)
      hud.toast("recording stopped: " .. why, "stopped")
    end
  end
  -- with nothing recording and the HUD hidden, the spy reads nothing
  -- drawing also runs while a surface is still up, so hiding the HUD or
  -- saving the last recording takes it down
  local shown = not hud.hidden
    or #session.unsaved > 0
    or live ~= nil
    or hud.showing()
  -- also when a clock reaches a new second, so none is skipped at a slow
  -- refresh
  local up, rec = hud.seconds(t)
  local draw = shown
    and (t - last_draw >= cfg.refresh_ms or up ~= drawn_up or rec ~= drawn_rec)
  if session.live ~= nil or draw then
    -- the color under the pointer only for the HUD, and never under the
    -- spy's own panels, where a read costs 10 to 25 ms on macOS
    local r = probe.read(draw and not hud.hidden, hud.covers, menu ~= nil)
    if session.live ~= nil then
      -- focus and monitor changes and controller motion go into a recording
      local w = r.window
      local window_key =
        table.concat({ w.title, w.process, w.x, w.y, w.width, w.height }, "\0")
      if window_key ~= last_window then
        last_window = window_key
        session.window(w)
      end
      local monitor = r.monitor and r.monitor.index
      if monitor ~= last_monitor then
        last_monitor = monitor
        if r.monitor ~= nil then
          session.display(r.monitor)
        end
      end
      for name, v in r.axes or {} do
        local last = logged_axes[name] or 0
        -- a stick back at rest is always logged, as the relay's observer does
        if math.abs(v - last) >= 0.05 or (v == 0 and last ~= 0) then
          logged_axes[name] = v
          session.axis(name, v)
        end
      end
    end
    if draw then
      last_draw, drawn_up, drawn_rec = t, up, rec
      hud.draw(r)
    end
  end
  hud.ai_tick(t)
end

-- the hooks only watch: input always passes through

local MODIFIER_KEYS =
  { "LCtrl", "RCtrl", "LShift", "RShift", "LAlt", "RAlt", "LWin", "RWin" }

rb.on("input_down", function(key, device)
  local k = string.lower(key)
  if is_own(k) then
    skipped[k] = true
    -- the chord's modifiers were logged as they went down; a saved macro
    -- leaves them out, and the log says which they were
    local held = {}
    for _, name in MODIFIER_KEYS do
      if rb.input.is_down(name) then
        table.insert(held, name)
      end
    end
    if #held > 0 then
      session.note({ kind = "own", keys = held })
    end
    return
  end
  if menu_key(k) then
    skipped[k] = true
    return
  end
  probe.input("down " .. key, device)
  session.key(true, key, nil, device)
end)

rb.on("input_up", function(key, held, device)
  local k = string.lower(key)
  if skipped[k] then
    skipped[k] = nil
    return
  end
  probe.input("up " .. key, device)
  session.key(false, key, held, device)
end)

local function bind(id, fn)
  if cfg[id] ~= "" then
    rb.bind(cfg[id], fn)
  end
end

rb.on("start", function()
  read_own()
  hud.init(cfg)
  session.init(cfg, hud.toast)
  local left = session.recover()
  if left > 0 then
    hud.toast(
      left
        .. " unsaved recording"
        .. (left > 1 and "s" or "")
        .. " from before\n"
        .. cfg.menu_key
        .. " to save",
      "not saved"
    )
  end
  bind("menu_key", function()
    if menu ~= nil then
      menu.close()
    else
      menu_at = rb.system.time()
      main_menu()
    end
  end)
  bind("hud_key", toggle_hud)
  bind("observe_key", function()
    toggle("observe")
  end)
  bind("record_key", function()
    toggle("macro")
  end)
  bind("copy_key", copy)
  last_tick = rb.system.time()
  timer = rb.timer.every(100, tick)
  tick()
end)

rb.on("stop", function()
  if timer ~= nil then
    timer:stop()
    timer = nil
  end
  session.stop("script stopped")
end)