Vertex displacement shader not updating shadows correctly

I have a vertex displacement shader that bends an object. It doesn't seem to handle light and shadows correctly though, like what it sent to the light data isn't updated correctly after the displacement. This can create invisible "ghosts" that occludes lights and shadows. How do I update it to match the actual model?

Here's my current code:

 Shader "City/Vertex colored animated" {
      Properties {
          _Color ("Color", Color) = (1,1,1,1)
          _MainTex ("Albedo (RGB)", 2D) = "white" {}
          _Smoothness ("Smoothness", Range(0,1)) = 0.5
          _Stretch ("Stretch", Range(-0.5,0.5)) = 0 
          _Bend ("Bend", Range(-0.5,0.5)) = 0 
          _SubmeshOffset ("Submesh Offset", Vector) = (0,0,0,0)
 
      }
      SubShader {
          Tags { 
              "RenderType"="Opaque" 
          }
          LOD 200
          
          CGPROGRAM
          #pragma surface surf Standard vertex:vert addshadow 
          #pragma target 3.0
 
          struct Input {
              float2 uv_MainTex;
              float3 vertexColor; // Vertex color stored here by vert() method
          };
 
          float _Stretch;
          float _Bend;
          float3 _SubmeshOffset;
  
          void vert (inout appdata_full v, out Input o) {
              UNITY_INITIALIZE_OUTPUT(Input,o);
              // Save the Vertex Color in the Input for the surf() method
              o.vertexColor = v.color; 
 
              float3 wpos = v.vertex.xyz;
 
              //vertex displacing
              float y = wpos.y + _SubmeshOffset.y;
              float x = wpos.x + _SubmeshOffset.x;
 
              wpos.y += y * _Stretch - (x * y) * _Bend;
              wpos.x += (wpos.x + _SubmeshOffset.x) * -_Stretch + y * y * _Bend;
              wpos.z += (wpos.z + _SubmeshOffset.z) * -_Stretch;
 
              v.vertex.xyz = wpos.xyz;
          }
  
          sampler2D _MainTex;
          half _Smoothness;
          fixed4 _Color;
  
          void surf (Input IN, inout SurfaceOutputStandard o) {
              // Albedo comes from a texture tinted by color
              fixed4 c = tex2D (_MainTex, IN.uv_MainTex) * _Color;
               // Combine normal color with the vertex color
              o.Albedo = c.rgb * IN.vertexColor;
              o.Smoothness = _Smoothness;
              o.Alpha = c.a;
 
          }
          ENDCG
     }
 }