fixed camera projection + model culling

camera.lua

@@ -5,10 +5,11 @@ require("kara3d.transform")
 camera = {class = "camera"}
 camera.__index = camera
 
--- Creates a camera at transform t with aspect ratio a, horizontal fov f,
+-- Creates a camera at transform t with horizontal fov f,
 -- near plane distance nd and far plane distance fd
 function camera.new(t, f, nd, fd)
-  local c = {transform = t, fov = f or 80 / 180 * math.pi,
+  local xres, yres, _, _ = aegisub.video_size()
+  local c = {transform = t, aspect = xres / yres, fov = f or 80 / 180 * math.pi,
     near = nd or 1, far = fd or 1000000000}
   setmetatable(c, camera)
   return c
@@ -21,15 +22,21 @@ end
 
 -- ********** INSTANCE FUNCTIONS **********
 
--- Returns this camera's projection * view matrix
-function camera:matrix()
-  local s = 1 / math.tan(self.fov / 2)
-  local n, f = self.near, self.far
+-- Returns this camera's projection matrix
+function camera:projection()
   local xres, yres, _, _ = aegisub.video_size()
+  local sx = 1 / math.atan(self.fov / 2)
+  local sy = self.aspect / math.atan(self.fov / 2)
+  local n, f = self.near, self.far
   local m = matrix.new(4, 4,
-    {s, 0,             0,             0,
-     0, s,             0,             0,
+    {sx, 0,            0,             0,
+     0, sy,            0,             0,
      0, 0, (n+f) / (n-f), 2*n*f / (n-f),
-     0, 0,      1 / yres,             0})
-  return m * self.transform:inv_matrix(true)
+     0, 0,            -1,             0})
+  return m
+end
+
+-- Returns this camera's view matrix
+function camera:view()
+  return self.transform:inv_matrix(true)
 end

quaternion.lua

@@ -32,10 +32,10 @@ function quaternion.identity() return quaternion.new() end
 function quaternion.mul_quaternion(q1, q2)
   local b, c, d, a = q1:normalized():unpack()
   local f, g, h, e = q2:normalized():unpack()
-  return quaternion.new(a*e - b*f - c*g - d*h,
-                        a*f + b*e + c*h - d*g,
+  return quaternion.new(a*f + b*e + c*h - d*g,
                         a*g + c*e + d*f - b*h,
-                        a*h + d*e + b*g - c*f)
+                        a*h + d*e + b*g - c*f,
+                        a*e - b*f - c*g - d*h)
 end
 
 -- Returns a vector equal to the vector v (size >= 3)
@@ -93,7 +93,7 @@ end
 
 -- Returns the inverse of this quaternion
 function quaternion:inv()
-  return quaternion.new(-self.x, -self.y, -self.z, -self.w)
+  return quaternion.new(-self.x, -self.y, -self.z, self.w)
 end
 
 -- Returns the 4 coordinates of this quaternion

shape.lua

@@ -38,44 +38,51 @@ end
 -- The boolean cull determines if the faces facing away from the screen are
 -- drawn or not
 function shape:draw(subs, line, tags, cull, cam)
-  local l = table.copy(line)
   local p = self.transform:position(true)
-  local r = self.transform:rotation(true)
+  local tm = self.transform:matrix(true)
   local c = vector.cardinal(4, 4)
-  local layer = line.layer
 
-  local pv = cam and cam:matrix() or matrix.identity(4, 4)
-  local cam_inv_rot = cam and cam.transform:rotation(true):inv() or quaternion.identity()
+  local proj = cam and cam:projection() or matrix.identity(4, 4)
+  local view = cam and cam:view() or matrix.identity(4, 4)
+  local camp = cam and cam.transform:position(true) or vector.zero(3)
+
   local xres, yres, _, _ = aegisub.video_size()
 
-  local m = pv * self.transform:matrix(true)
+  local m = proj * view * tm
 
   for i = 1, #self.faces do
     local face = self.faces[i]
-    local n = quaternion.mul_vector(r, face.normal)
-    n = quaternion.mul_vector(cam_inv_rot, n)
+    local vs = face.vertices
+
+    local v = m * (vs[#vs] + c)
+    v = (1 / v:w()) * v
+
+    local n = (tm * (face.normal + c) - p):set_size(3)
+    local c_to_v = (tm * (vs[#vs] + c) - camp):set_size(3):normalize()
 
-    if not (cull and n:z() > 0) then
-      local vs = face.vertices
+    if not (cull and vector.dot(c_to_v, n) > 0) then
 
-      local v = m * (vs[#vs] + c)
-      v = (1 / v:w()) * v
-      local sum = vector.zero(4)
-      local str = (tags and "{" .. tags .. "}" or "")
+      local l = table.copy(line)
+      l.text = (tags and "{" .. tags .. "}" or "")
         .. (face.tags and "{" .. face.tags .. "}" or "")
         .. "{\\an7\\pos(" .. (xres / 2) .. ", " .. (yres / 2) .. ")\\p1}"
-        .. "m " .. v:x() .. " " .. v:y() .. " "
+        .. "m " .. (v:x() * xres / 2) .. " " .. (-v:y() * yres / 2) .. " "
+
+      local face_center = vector.zero(4)
 
       for i = 1, #vs do
         v = m * (vs[i] + c)
-        sum = sum + v
+        face_center = face_center + v
         v = (1 / v:w()) * v
-        str = str .. "l " .. v:x() .. " " .. v:y() .. " "
+        l.text = l.text .. "l " .. ( v:x() * xres / 2) .. " "
+                                .. (-v:y() * yres / 2) .. " "
       end
 
-      l.text = str
+      face_center = (1 / #vs) * face_center
+
+      l.text = l.text
       -- layers must be between 0 and 9999999999
-      l.layer = 500000000 + math.floor((p + sum):z() / #vs)
+      l.layer = math.floor(-face_center:z())
       l.effect = "fx"
       subs.append(l)
     end

transform.lua

@@ -28,9 +28,21 @@ end
 
 -- ********** INSTANCE FUNCTIONS **********
 
--- Rotates this transform by angle around axis
+function transform:right()
+  return (self:rotation(true):to_matrix() * vector.new({1, 0, 0, 1})):set_size(3):normalize()
+end
+
+function transform:up()
+  return (self:rotation(true):to_matrix() * vector.new({0, 1, 0, 1})):set_size(3):normalize()
+end
+
+function transform:forward()
+  return (self:rotation(true):to_matrix() * vector.new({0, 0, 1, 1})):set_size(3):normalize()
+end
+
+-- Rotates this transform by angle around axis (local)
 function transform:rotate(axis, angle)
-  self.rot = quaternion.axis_angle(axis, angle) * self.rot
+  self.rot = quaternion.mul_quaternion(quaternion.axis_angle(axis, angle), self.rot)
   return self
 end
 
@@ -95,7 +107,7 @@ function transform:inv_matrix(global)
      0, 1, 0, -self.pos:get(2),
      0, 0, 1, -self.pos:get(3),
      0, 0, 0,               1})
-  local rotation = self.rot:normalized():inv():to_matrix()
+  local rotation = self.rot:inv():normalized():to_matrix()
   local scale = matrix.new(4, 4,
     {1/self.scl:get(1),                 0,                 0, 0,
                      0, 1/self.scl:get(2),                 0, 0,