Add Lua::set_vector_metatable() method (unstable)

src/lua.rs

@@ -1795,6 +1795,27 @@ impl Lua {
         unsafe { self.make_userdata(UserDataCell::new(UserDataProxy::<T>(PhantomData))) }
     }
 
+    /// Sets the metatable for a Luau builtin vector type.
+    #[cfg(any(all(feature = "luau", feature = "unstable"), doc))]
+    #[cfg_attr(docsrs, doc(cfg(all(feature = "luau", feature = "unstable"))))]
+    pub fn set_vector_metatable(&self, metatable: Option<Table>) {
+        unsafe {
+            let state = self.state();
+            let _sg = StackGuard::new(state);
+            assert_stack(state, 2);
+
+            #[cfg(not(feature = "luau-vector4"))]
+            ffi::lua_pushvector(state, 0., 0., 0.);
+            #[cfg(feature = "luau-vector4")]
+            ffi::lua_pushvector(state, 0., 0., 0., 0.);
+            match metatable {
+                Some(metatable) => self.push_ref(&metatable.0),
+                None => ffi::lua_pushnil(state),
+            };
+            ffi::lua_setmetatable(state, -2);
+        }
+    }
+
     /// Returns a handle to the global environment.
     pub fn globals(&self) -> Table {
         let state = self.state();

tests/luau.rs

@@ -130,6 +130,48 @@ fn test_vectors() -> Result<()> {
     Ok(())
 }
 
+#[cfg(all(not(feature = "luau-vector4"), feature = "unstable"))]
+#[test]
+fn test_vector_metatable() -> Result<()> {
+    let lua = Lua::new();
+
+    let vector_mt = lua
+        .load(
+            r#"
+            {
+                __index = {
+                    new = vector,
+
+                    product = function(a, b)
+                        return vector(a.x * b.x, a.y * b.y, a.z * b.z)
+                    end
+                }
+            }
+    "#,
+        )
+        .eval::<Table>()?;
+    vector_mt.set_metatable(Some(vector_mt.clone()));
+    lua.set_vector_metatable(Some(vector_mt.clone()));
+    lua.globals().set("Vector3", vector_mt)?;
+
+    let compiler = Compiler::new()
+        .set_vector_lib("Vector3")
+        .set_vector_ctor("new");
+
+    // Test vector methods (fastcall)
+    lua.load(
+        r#"
+        local v = Vector3.new(1, 2, 3)
+        local v2 = v:product(Vector3.new(2, 3, 4))
+        assert(v2.x == 2 and v2.y == 6 and v2.z == 12)
+    "#,
+    )
+    .set_compiler(compiler)
+    .exec()?;
+
+    Ok(())
+}
+
 #[test]
 fn test_readonly_table() -> Result<()> {
     let lua = Lua::new();