Normal vectors not interpolating in custom lighting surface shaders

I am messing around with custom lighting in surface shaders, but it seems that I can't make my specular highlights as smooth as BlinnPhong specular. If I put my shader on a basic sphere, the reflection acts on the rough geometry, while BlinnPhong seems to interpolate the normal vectors so the lighting appears smoother. I tried to use the BlinnPhong function from lighting.cginc, but the interpolation seems to come from elsewhere. I succeeded to get some interpolation by unpacking a blank normal map, but it seems to create glitches on spheres.

here's my shader:

 _MainTex("Base (RGB), Spec(A)", 2D) = "bump" {}
 _NormalMap ("Normal", 2D) = "bump" {} //Texture 2D bump par défaut, pour la normal
 _Shininess ("Shininess", Range (0.01, 1)) = 0.078125
 _SpecColor ("Specular Color", Color) = (0.5, 0.5, 0.5, 1)
 _LC("Lit Center", Color)  = (1,1,1,1)
 _UC("Unlit Center", Color) = (1,1,1,1)
 _LR("Lit Rim", Color)= (1,1,1,1)
 _UR("Unlit Rim", Color) = (1,1,1,1)
 _RimPower("RimPower",Float)=0.0
     }
     
     SubShader 
     {
     Tags { "RenderType"="Opaque"}
     LOD 200
 
         CGPROGRAM
 #pragma surface surf Ramp
 #pragma target 3.0
 
 
 sampler2D _MainTex;
 half _Shininess;
 sampler2D _NormalMap;
 float4 _LC;
 float4 _UC;
 float4 _LR;
 float4 _UR;
 float _RimPower;
 
 struct Input {
     float2 uv_MainTex;
 };
 
 half4 LightingRamp (SurfaceOutput s, float3 lightDir, half3 viewDir, half atten)
             {
             half3 h = normalize (lightDir + viewDir);
             float nh = max (0, dot (s.Normal, h));
             float spec = pow (nh, s.Specular*128.0) * s.Gloss;
             float NdotL=dot(s.Normal, lightDir);
             float NdotE=pow(saturate(dot(s.Normal, viewDir)),_RimPower);
             float diff=(NdotL*0.5)+0.5;
             float4 c;
             c.rgb=(s.Albedo*_LightColor0.rgb*lerp(lerp(_UR,_UC,NdotE),lerp(_LR,_LC,NdotE),diff)+_LightColor0.rgb * _SpecColor.rgb * spec)*atten*2;
             c.a=1;
             return c;
             }
             
 
             
             
 void surf (Input IN, inout SurfaceOutput o) 
         {
             float4 c=tex2D(_MainTex,IN.uv_MainTex);
             o.Albedo=c.rgb;
             o.Alpha=c.a;
             o.Gloss=c.a;
             o.Specular = _Shininess;
             o.Normal = UnpackNormal(tex2D(_NormalMap, IN.uv_MainTex));
         }
         ENDCG
     }
     FallBack "Specular"
 }