How can I control a shader's colormask with a texture?

I have a skinned clothing mesh parented to a character. Based on the alpha channel of the texture certain parts of the clothing mesh get hidden. I'm creating a shader for making the clothing "invisible". The shader is queued to geometry but renders transparent and with a colormask A; which means that when the transparency of the clothing mesh is lowered it also hides the character geometry underneath it, preventing the invisible clothing from making the character nude!

The problem is that at the moment the entire clothing mesh is made invisible not just the clothing rendered by the diffuse map. This makes parts of the characters body geometry which should be poking through, strangely hidden. What I need is to have the invisible parts controlled by the alpha channel of the texture i.e. all the parts that are black in the alpha channel should not be invisible, and the rest should. I'm not sure if that means controlling the render queue or the value of the colormask? My only guess is that 2 separate passes would be needed, so that the invisible and visible parts of the same geometry could be rendered differently.

Here is the shader code:

 Shader "Player/Invisible" {
 Properties {
     _Color ("Main Color", Color) = (1,1,1,1)
     _SpecColor ("Specular Color", Color) = (0.5, 0.5, 0.5, 0)
     _Shininess ("Shininess", Range (0.01, 1)) = 0.078125
     _MainTex ("Base (RGB) TransGloss (A)", 2D) = "white" {}
     _BumpMap ("Normal Map", 2D) = "bump" {}
 }
  
 SubShader {
     Tags { "Queue"="Geometry" "IgnoreProjector"="False" "RenderType"="Transparent" }
     LOD 400
  
 //For getting rid of Z-fighting
 Pass {
         //ZWrite On
         ColorMask A
         //AlphaTest GEqual 0.01
     }
  
 CGPROGRAM
 #pragma surface surf BlinnPhong alpha
 #pragma exclude_renderers flash
  
 sampler2D _MainTex;
 sampler2D _BumpMap;
 fixed4 _Color;
 half _Shininess;
  
 struct Input {
     float2 uv_MainTex;
     float2 uv_BumpMap;
 };
  
 void surf (Input IN, inout SurfaceOutput o) {
     fixed4 tex = tex2D(_MainTex, IN.uv_MainTex);
     o.Albedo = tex.rgb * _Color.rgb;
     o.Gloss = tex.a;
     o.Alpha = tex.a * _Color.a;
     o.Specular = _Shininess;
     o.Normal = UnpackNormal(tex2D(_BumpMap, IN.uv_BumpMap));
 } 
 ENDCG  
 }
  
 FallBack "Transparent/Bumped Diffuse"
 }