How can I transform a world view dir into tangent space in a Cg only shader

Hello, I've been trying to figure this one out for a while now and I can't seem to make it work. The only place it fails is when I rotate the object in any way.

My plan was to write a custom Cg shader that would transform the viewDir into tangent space so I could do my calculation there. This method is supposed to be much faster than transforming the tangent space normal into world space, and less complicated, so I thought :).

Here was my line of thinking. I create a TBN matrix with TANGENT_SPACE_ROTATION and multiply it with unity_ObjectToWorld so that the TBN will be in world space. Then I multiply the world space view dir UNITY_MATRIX_V[2].xyz by the transpose of that matrix to bring the view direction into tangent space.

Update: Updated to working code.

Here's the code:

 Shader "CgRimShader"
 {
     Properties
     {
         _Color("Color", COLOR) = (0,0,0,1)
         _RimColor("Rim Color", COLOR) = (1,1,1,1)
         _RimPower("Rim Power", Float) = 2
         _BumpMap ("Normalmap", 2D) = "bump" {}
     }
     SubShader
     {
         Tags {"RenderType"="Opaque" }
         LOD 100
 
         Pass
         {
             CGPROGRAM
             #pragma vertex vert
             #pragma fragment frag
             
             #include "UnityCG.cginc"
 
             struct appdata
             {
                 float4 vertex   : POSITION;
                 float2 uv       : TEXCOORD0;
                 float3 normal   : NORMAL;
                 float4 tangent  : TANGENT;
             };
 
             struct v2f
             {
                 float4 vertex   : SV_POSITION;
                 float2 uv       : TEXCOORD0;
                 float3 viewDir  : TEXCOORD1;
             };
 
             sampler2D   _BumpMap;
             float4      _BumpMap_ST;
             float4      _Color;
             float4      _RimColor;
             float       _RimPower;
             
             v2f vert (appdata v)
             {
                 v2f o;
 
                 o.vertex = UnityObjectToClipPos(v.vertex);
                 o.uv = TRANSFORM_TEX(v.uv, _BumpMap);
                             o.viewDir = COMPUTE_VIEW_NORMAL;
 
                 return o;
             }
             
             fixed4 frag (v2f i) : SV_Target
             {
                 float3 Normal = UnpackNormal(tex2D(_BumpMap, i.uv));
                 float3 ViewDir = normalize(i.viewDir);
                 float NdotV = max(0, dot(Normal, ViewDir));
 
                 fixed4 col = _Color + pow(1-NdotV, _RimPower) * _RimColor;
 
                 return col;
             }
             ENDCG
         }
     }
 }