How to change RGB without changing alpha in sprite frag shader?
I'm adding fog to a custom sprite vert+surf shader, rendered with a SpriteRenderer. It's as easy as removing #pragma nofog, but the result applies the fog color to transparent portions of my sprites, resulting in semi-visible boxes around them.
I traced this to Unity's UNITY_FOG_LERP_COLOR, called at the end of the Unity generated frag shader (frag_surf):
#define UNITY_FOG_LERP_COLOR(col,fogCol,fogFac) col.rgb = lerp((fogCol).rgb, (col).rgb, saturate(fogFac))
If I change it to:
col.r = 1;
Now it only sets the red channel to 1. It shouldn't affect other colors, or the alpha, right? But the result still adds semi-visible red boxes around my sprites (pictured above). Does Unity somehow embed alpha information into the R channel? How do I change RGB in the frag shader without affecting alpha?
I don't want to use clip. Multiplying col.rgb by col.a after applying the fog looks close to correct, but adds a faint dark outline.
My shader:
Shader "Northway/SpritesDiffuseWave" {
Properties {
[PerRendererData] _MainTex ("Sprite Texture", 2D) = "white" {}
}
SubShader {
Tags {
"Queue"="Geometry"
"IgnoreProjector"="True"
"RenderType"="Transparent"
"PreviewType"="Plane"
"CanUseSpriteAtlas"="True"
}
Cull Off
Lighting Off
ZWrite Off
Blend One OneMinusSrcAlpha
CGPROGRAM
#pragma surface surf SimpleLambert vertex:vert nolightmap nodynlightmap keepalpha noinstancing //nofog
#include "UnitySprites.cginc"
#undef UNITY_FOG_LERP_COLOR
#define UNITY_FOG_LERP_COLOR(col,fogCol,fogFac) col.r = 1;
struct Input {
float2 uv_MainTex;
fixed4 color;
};
half4 LightingSimpleLambert(SurfaceOutput s, half3 lightDir, half atten) {
half4 c;
c.rgb = s.Albedo * _LightColor0.rgb * atten * 0.60;
c.a = s.Alpha;
return c;
}
void vert (inout appdata_full v, out Input o) {
UNITY_INITIALIZE_OUTPUT(Input, o);
//... vertex shader code
}
void surf (Input IN, inout SurfaceOutput o) {
fixed4 color = tex2D(_MainTex, IN.uv_MainTex) * IN.color;
o.Albedo = color.rgb * color.a;
o.Alpha = color.a;
}
ENDCG
}
Fallback "Transparent/VertexLit"
}
Answer by sarahnorthway · Sep 19, 2018 at 09:34 PM
I was on the right track with multiplying rgb by alpha; I needed to multiply the FOG rgb by alpha before I lerp it in. Presumably col.rgb already has the alpha information baked in, but fogCol.rgb does not.
#undef UNITY_FOG_LERP_COLOR
#define UNITY_FOG_LERP_COLOR(col,fogCol,fogFac) col.rgb = lerp((fogCol).rgb * col.a, (col).rgb, saturate(fogFac))
This works with #pragma keepalpha or alpha:premul but not alpha:blend.