From 4256641da483a6a03fded2dffb974243690955a0 Mon Sep 17 00:00:00 2001
From: wwylele <wwylele@gmail.com>
Date: Sun, 1 Apr 2018 18:22:14 +0300
Subject: [PATCH] gl_rasterizer/lighting: implement shadow attenuation

---
 .../renderer_opengl/gl_shader_gen.cpp         | 44 ++++++++++++++++++-
 .../renderer_opengl/gl_shader_gen.h           |  8 ++++
 2 files changed, 50 insertions(+), 2 deletions(-)

diff --git a/src/video_core/renderer_opengl/gl_shader_gen.cpp b/src/video_core/renderer_opengl/gl_shader_gen.cpp
index 8e0964496..9c0f10a41 100644
--- a/src/video_core/renderer_opengl/gl_shader_gen.cpp
+++ b/src/video_core/renderer_opengl/gl_shader_gen.cpp
@@ -115,6 +115,7 @@ PicaShaderConfig PicaShaderConfig::BuildFromRegs(const Pica::Regs& regs) {
             !regs.lighting.IsDistAttenDisabled(num);
         state.lighting.light[light_index].spot_atten_enable =
             !regs.lighting.IsSpotAttenDisabled(num);
+        state.lighting.light[light_index].shadow_enable = !regs.lighting.IsShadowDisabled(num);
     }
 
     state.lighting.lut_d0.enable = regs.lighting.config1.disable_lut_d0 == 0;
@@ -161,6 +162,13 @@ PicaShaderConfig PicaShaderConfig::BuildFromRegs(const Pica::Regs& regs) {
     state.lighting.bump_renorm = regs.lighting.config0.disable_bump_renorm == 0;
     state.lighting.clamp_highlights = regs.lighting.config0.clamp_highlights != 0;
 
+    state.lighting.enable_shadow = regs.lighting.config0.enable_shadow != 0;
+    state.lighting.shadow_primary = regs.lighting.config0.shadow_primary != 0;
+    state.lighting.shadow_secondary = regs.lighting.config0.shadow_secondary != 0;
+    state.lighting.shadow_invert = regs.lighting.config0.shadow_invert != 0;
+    state.lighting.shadow_alpha = regs.lighting.config0.shadow_alpha != 0;
+    state.lighting.shadow_selector = regs.lighting.config0.shadow_selector;
+
     state.proctex.enable = regs.texturing.main_config.texture3_enable;
     if (state.proctex.enable) {
         state.proctex.coord = regs.texturing.main_config.texture3_coordinates;
@@ -203,6 +211,11 @@ static std::string SampleTexture(const PicaShaderConfig& config, unsigned textur
             return "textureProj(tex[0], vec3(texcoord[0], texcoord0_w))";
         case TexturingRegs::TextureConfig::TextureCube:
             return "texture(tex_cube, vec3(texcoord[0], texcoord0_w))";
+        case TexturingRegs::TextureConfig::Shadow2D:
+        case TexturingRegs::TextureConfig::ShadowCube:
+            NGLOG_CRITICAL(HW_GPU, "Unhandled shadow texture");
+            UNIMPLEMENTED();
+            return "vec4(1.0)"; // stubbed to avoid rendering with wrong shadow
         default:
             LOG_CRITICAL(HW_GPU, "Unhandled texture type %x",
                          static_cast<int>(state.texture0_type));
@@ -606,6 +619,17 @@ static void WriteLighting(std::string& out, const PicaShaderConfig& config) {
     out += "vec3 normal = quaternion_rotate(normalized_normquat, surface_normal);\n";
     out += "vec3 tangent = quaternion_rotate(normalized_normquat, surface_tangent);\n";
 
+    if (lighting.enable_shadow) {
+        std::string shadow_texture = SampleTexture(config, lighting.shadow_selector);
+        if (lighting.shadow_invert) {
+            out += "vec4 shadow = vec4(1.0) - " + shadow_texture + ";\n";
+        } else {
+            out += "vec4 shadow = " + shadow_texture + ";\n";
+        }
+    } else {
+        out += "vec4 shadow = vec4(1.0);\n";
+    }
+
     // Samples the specified lookup table for specular lighting
     auto GetLutValue = [&lighting](LightingRegs::LightingSampler sampler, unsigned light_num,
                                    LightingRegs::LightingLutInput input, bool abs) {
@@ -814,13 +838,29 @@ static void WriteLighting(std::string& out, const PicaShaderConfig& config) {
             }
         }
 
+        bool shadow_primary_enable = lighting.shadow_primary && light_config.shadow_enable;
+        bool shadow_secondary_enable = lighting.shadow_secondary && light_config.shadow_enable;
+        std::string shadow_primary = shadow_primary_enable ? " * shadow.rgb" : "";
+        std::string shadow_secondary = shadow_secondary_enable ? " * shadow.rgb" : "";
+
         // Compute primary fragment color (diffuse lighting) function
         out += "diffuse_sum.rgb += ((" + light_src + ".diffuse * dot_product) + " + light_src +
-               ".ambient) * " + dist_atten + " * " + spot_atten + ";\n";
+               ".ambient) * " + dist_atten + " * " + spot_atten + shadow_primary + ";\n";
 
         // Compute secondary fragment color (specular lighting) function
         out += "specular_sum.rgb += (" + specular_0 + " + " + specular_1 +
-               ") * clamp_highlights * " + dist_atten + " * " + spot_atten + ";\n";
+               ") * clamp_highlights * " + dist_atten + " * " + spot_atten + shadow_secondary +
+               ";\n";
+    }
+
+    // Apply shadow attenuation to alpha components if enabled
+    if (lighting.shadow_alpha) {
+        if (lighting.enable_primary_alpha) {
+            out += "diffuse_sum.a *= shadow.a;\n";
+        }
+        if (lighting.enable_secondary_alpha) {
+            out += "specular_sum.a *= shadow.a;\n";
+        }
     }
 
     // Sum final lighting result
diff --git a/src/video_core/renderer_opengl/gl_shader_gen.h b/src/video_core/renderer_opengl/gl_shader_gen.h
index 12bda326f..929c3c015 100644
--- a/src/video_core/renderer_opengl/gl_shader_gen.h
+++ b/src/video_core/renderer_opengl/gl_shader_gen.h
@@ -66,6 +66,7 @@ struct PicaShaderConfigState {
             bool spot_atten_enable;
             bool geometric_factor_0;
             bool geometric_factor_1;
+            bool shadow_enable;
         } light[8];
 
         bool enable;
@@ -79,6 +80,13 @@ struct PicaShaderConfigState {
         bool enable_primary_alpha;
         bool enable_secondary_alpha;
 
+        bool enable_shadow;
+        bool shadow_primary;
+        bool shadow_secondary;
+        bool shadow_invert;
+        bool shadow_alpha;
+        unsigned shadow_selector;
+
         struct {
             bool enable;
             bool abs_input;