RenderState support for two sided Stencil Buffer operations (in order to support Shadow Volumes in the future)

git-svn-id: https://jmonkeyengine.googlecode.com/svn/trunk@7286 75d07b2b-3a1a-0410-a2c5-0572b91ccdca

engine/src/core/com/jme3/material/RenderState.java

@@ -125,11 +125,34 @@ public class RenderState implements Cloneable, Savable {
          */
         Back,
         /**
-         * Cull both front and back faces. 
+         * Cull both front and back faces.
          */
         FrontAndBack
     }
 
+
+    public enum StencilOperation {
+        Keep, //keep the current value
+        Zero, //set the value to 0
+        Replace,  //sets the buffer to
+        Increment,
+        IncrementWrap,
+        Decrement,
+        DecrementWrap,
+        Invert
+    }
+
+    public enum StencilFunction {
+        Never,
+        Less,
+        LessEqual,
+        Greater,
+        GreaterEqual,
+        Equal,
+        NotEqual,
+        Always
+    }
+
     static {
         NULL.cullMode = FaceCullMode.Off;
         NULL.depthTest = false;
@@ -147,7 +170,7 @@ public class RenderState implements Cloneable, Savable {
         ADDITIONAL.applyAlphaFallOff = false;
         ADDITIONAL.applyPolyOffset = false;
     }
-    
+
     boolean pointSprite = false;
     boolean applyPointSprite = true;
     boolean wireframe = false;
@@ -170,6 +193,15 @@ public class RenderState implements Cloneable, Savable {
     boolean applyPolyOffset = true;
     float offsetFactor = 0;
     float offsetUnits = 0;
+    boolean stencilTest = false;
+    StencilOperation frontStencilStencilFailOperation = StencilOperation.Keep;
+    StencilOperation frontStencilDepthFailOperation = StencilOperation.Keep;
+    StencilOperation frontStencilDepthPassOperation = StencilOperation.Keep;
+    StencilOperation backStencilStencilFailOperation = StencilOperation.Keep;
+    StencilOperation backStencilDepthFailOperation = StencilOperation.Keep;
+    StencilOperation backStencilDepthPassOperation = StencilOperation.Keep;
+    StencilFunction frontStencilFunction = StencilFunction.Always;
+    StencilFunction backStencilFunction = StencilFunction.Always;
 
     public void write(JmeExporter ex) throws IOException {
         OutputCapsule oc = ex.getCapsule(this);
@@ -185,6 +217,15 @@ public class RenderState implements Cloneable, Savable {
         oc.write(offsetEnabled, "offsetEnabled", false);
         oc.write(offsetFactor, "offsetFactor", 0);
         oc.write(offsetUnits, "offsetUnits", 0);
+        oc.write(stencilTest, "stencilTest", false);
+        oc.write(frontStencilStencilFailOperation, "frontStencilStencilFailOperation", StencilOperation.Keep);
+        oc.write(frontStencilDepthFailOperation, "frontStencilDepthFailOperation", StencilOperation.Keep);
+        oc.write(frontStencilDepthPassOperation, "frontStencilDepthPassOperation", StencilOperation.Keep);
+        oc.write(backStencilStencilFailOperation, "frontStencilStencilFailOperation", StencilOperation.Keep);
+        oc.write(backStencilDepthFailOperation, "backStencilDepthFailOperation", StencilOperation.Keep);
+        oc.write(backStencilDepthPassOperation, "backStencilDepthPassOperation", StencilOperation.Keep);
+        oc.write(frontStencilFunction, "frontStencilFunction", StencilFunction.Always);
+        oc.write(backStencilFunction, "backStencilFunction", StencilFunction.Always);
     }
 
     public void read(JmeImporter im) throws IOException {
@@ -201,6 +242,15 @@ public class RenderState implements Cloneable, Savable {
         offsetEnabled = ic.readBoolean("offsetEnabled", false);
         offsetFactor = ic.readFloat("offsetFactor", 0);
         offsetUnits = ic.readFloat("offsetUnits", 0);
+        stencilTest = ic.readBoolean("stencilTest", false);
+        frontStencilStencilFailOperation = ic.readEnum("frontStencilStencilFailOperation", StencilOperation.class, StencilOperation.Keep);
+        frontStencilDepthFailOperation = ic.readEnum("frontStencilDepthFailOperation", StencilOperation.class, StencilOperation.Keep);
+        frontStencilDepthPassOperation = ic.readEnum("frontStencilDepthPassOperation", StencilOperation.class, StencilOperation.Keep);
+        backStencilStencilFailOperation = ic.readEnum("backStencilStencilFailOperation", StencilOperation.class, StencilOperation.Keep);
+        backStencilDepthFailOperation = ic.readEnum("backStencilDepthFailOperation", StencilOperation.class, StencilOperation.Keep);
+        backStencilDepthPassOperation = ic.readEnum("backStencilDepthPassOperation", StencilOperation.class, StencilOperation.Keep);
+        frontStencilFunction = ic.readEnum("frontStencilFunction", StencilFunction.class, StencilFunction.Always);
+        backStencilFunction = ic.readEnum("backStencilFunction", StencilFunction.class, StencilFunction.Always);
     }
 
     @Override
@@ -282,6 +332,41 @@ public class RenderState implements Cloneable, Savable {
         offsetUnits = units;
     }
 
+    public void setStencil(boolean enabled,
+            StencilOperation _frontStencilStencilFailOperation,
+            StencilOperation _frontStencilDepthFailOperation,
+            StencilOperation _frontStencilDepthPassOperation,
+            StencilOperation _backStencilStencilFailOperation,
+            StencilOperation _backStencilDepthFailOperation,
+            StencilOperation _backStencilDepthPassOperation,
+            StencilFunction _frontStencilFunction,
+            StencilFunction _backStencilFunction){
+
+        stencilTest = enabled;
+        this.frontStencilStencilFailOperation = _frontStencilStencilFailOperation;
+        this.frontStencilDepthFailOperation = _frontStencilDepthFailOperation;
+        this.frontStencilDepthPassOperation = _frontStencilDepthPassOperation;
+        this.backStencilStencilFailOperation = _backStencilStencilFailOperation;
+        this.backStencilDepthFailOperation = _backStencilDepthFailOperation;
+        this.backStencilDepthPassOperation = _backStencilDepthPassOperation;
+        this.frontStencilFunction = _frontStencilFunction;
+        this.backStencilFunction = _backStencilFunction;
+    }
+
+    public boolean isStencilTest() {
+        return stencilTest;
+    }
+
+    public StencilOperation getFrontStencilStencilFailOperation(){ return frontStencilStencilFailOperation; }
+    public StencilOperation getFrontStencilDepthFailOperation(){ return frontStencilDepthFailOperation; }
+    public StencilOperation getFrontStencilDepthPassOperation(){ return frontStencilDepthPassOperation; }
+    public StencilOperation getBackStencilStencilFailOperation(){ return backStencilStencilFailOperation; }
+    public StencilOperation getBackStencilDepthFailOperation(){ return backStencilDepthFailOperation; }
+    public StencilOperation getBackStencilDepthPassOperation(){ return backStencilDepthPassOperation; }
+
+    public StencilFunction getFrontStencilFunction(){ return frontStencilFunction; }
+    public StencilFunction getBackStencilFunction(){ return backStencilFunction; }
+
     public void setFaceCullMode(FaceCullMode cullMode) {
         applyCullMode = true;
         this.cullMode = cullMode;

engine/src/core/com/jme3/renderer/RenderContext.java

@@ -42,7 +42,7 @@ import com.jme3.texture.Texture;
  * internally to reduce state changes. NOTE: This class is specific to OpenGL.
  */
 public class RenderContext {
-    
+
     /**
      * If back-face culling is enabled.
      */
@@ -139,6 +139,19 @@ public class RenderContext {
 
     public int boundTextureUnit = 0;
 
+    /**
+     * Stencil Buffer state
+     */
+    public boolean stencilTest = false;
+    public RenderState.StencilOperation frontStencilStencilFailOperation = RenderState.StencilOperation.Keep;
+    public RenderState.StencilOperation frontStencilDepthFailOperation = RenderState.StencilOperation.Keep;
+    public RenderState.StencilOperation frontStencilDepthPassOperation = RenderState.StencilOperation.Keep;
+    public RenderState.StencilOperation backStencilStencilFailOperation = RenderState.StencilOperation.Keep;
+    public RenderState.StencilOperation backStencilDepthFailOperation = RenderState.StencilOperation.Keep;
+    public RenderState.StencilOperation backStencilDepthPassOperation = RenderState.StencilOperation.Keep;
+    public RenderState.StencilFunction frontStencilFunction = RenderState.StencilFunction.Always;
+    public RenderState.StencilFunction backStencilFunction = RenderState.StencilFunction.Always;
+
     /**
      * Vertex attribs currently bound and enabled. If a slot is null, then
      * it is disabled.

engine/src/lwjgl-ogl/com/jme3/renderer/lwjgl/LwjglGL1Renderer.java

@@ -324,6 +324,9 @@ public class LwjglGL1Renderer implements GL1Renderer {
             context.blendMode = state.getBlendMode();
         }
 
+        if(state.isStencilTest())
+            throw new UnsupportedOperationException("OpenGL 1.1 doesn't support two sided stencil operations.");
+
     }
 
     public void setViewPort(int x, int y, int w, int h) {
@@ -520,7 +523,7 @@ public class LwjglGL1Renderer implements GL1Renderer {
                 if (!FastMath.isPowerOfTwo(img.getWidth())
                     || !FastMath.isPowerOfTwo(img.getHeight())
                     || img.getWidth() != img.getHeight()){
-                    
+
                     // Resize texture to Power-of-2 size
                     MipMapGenerator.resizeToPowerOf2(img);
 
@@ -685,7 +688,7 @@ public class LwjglGL1Renderer implements GL1Renderer {
     public void drawTriangleArray(Mesh.Mode mode, int count, int vertCount) {
         if (count > 1)
             throw new UnsupportedOperationException();
-        
+
         glDrawArrays(convertElementMode(mode), 0, vertCount);
     }
 
@@ -800,7 +803,7 @@ public class LwjglGL1Renderer implements GL1Renderer {
                 drawElements(elMode,
                              fmt,
                              indexData);
-                
+
                 curOffset += elementLength;
             }*/
         } else {
@@ -810,7 +813,7 @@ public class LwjglGL1Renderer implements GL1Renderer {
         }
     }
 
-    
+
 
     public void clearVertexAttribs() {
         for (int i = 0; i < 16; i++) {
@@ -895,7 +898,7 @@ public class LwjglGL1Renderer implements GL1Renderer {
         }
 
         statistics.onMeshDrawn(mesh, lod);
-        
+
 //        if (!dynamic) {
             // dealing with a static object, generate display list
 //            renderMeshDisplayList(mesh);

engine/src/lwjgl-ogl/com/jme3/renderer/lwjgl/LwjglRenderer.java

@@ -170,7 +170,7 @@ public class LwjglRenderer implements Renderer {
         return caps;
     }
 
-    @SuppressWarnings("fallthrough") 
+    @SuppressWarnings("fallthrough")
     public void initialize() {
         ContextCapabilities ctxCaps = GLContext.getCapabilities();
         if (ctxCaps.OpenGL20) {
@@ -471,7 +471,7 @@ public class LwjglRenderer implements Renderer {
 
         if (state.isDepthTest() && !context.depthTestEnabled) {
             glEnable(GL_DEPTH_TEST);
-            glDepthFunc(GL_LEQUAL);
+            glDepthFunc(GL_LESS);
             context.depthTestEnabled = true;
         } else if (!state.isDepthTest() && context.depthTestEnabled) {
             glDisable(GL_DEPTH_TEST);
@@ -603,6 +603,97 @@ public class LwjglRenderer implements Renderer {
             context.blendMode = state.getBlendMode();
         }
 
+        if(context.stencilTest!=state.isStencilTest()
+                || context.frontStencilStencilFailOperation!=state.getFrontStencilStencilFailOperation()
+                || context.frontStencilDepthFailOperation!=state.getFrontStencilDepthFailOperation()
+                || context.frontStencilDepthPassOperation!=state.getFrontStencilDepthPassOperation()
+                || context.backStencilStencilFailOperation!=state.getBackStencilStencilFailOperation()
+                || context.backStencilDepthFailOperation!=state.getBackStencilDepthFailOperation()
+                || context.backStencilDepthPassOperation!=state.getBackStencilDepthPassOperation()
+                || context.frontStencilFunction!=state.getFrontStencilFunction()
+                || context.backStencilFunction!=state.getBackStencilFunction()
+                ){
+
+            context.frontStencilStencilFailOperation=state.getFrontStencilStencilFailOperation();   //terrible looking, I know
+            context.frontStencilDepthFailOperation=state.getFrontStencilDepthFailOperation();
+            context.frontStencilDepthPassOperation=state.getFrontStencilDepthPassOperation();
+            context.backStencilStencilFailOperation=state.getBackStencilStencilFailOperation();
+            context.backStencilDepthFailOperation=state.getBackStencilDepthFailOperation();
+            context.backStencilDepthPassOperation=state.getBackStencilDepthPassOperation();
+            context.frontStencilFunction=state.getFrontStencilFunction();
+            context.backStencilFunction=state.getBackStencilFunction();
+
+            if(state.isStencilTest()){
+                glEnable(GL_STENCIL_TEST);
+                glStencilOpSeparate(GL_FRONT,
+                        glStencilOpFromStencilOp(state.getFrontStencilStencilFailOperation()),
+                        glStencilOpFromStencilOp(state.getFrontStencilDepthFailOperation()),
+                        glStencilOpFromStencilOp(state.getFrontStencilDepthPassOperation())
+                        );
+                glStencilOpSeparate(GL_BACK,
+                        glStencilOpFromStencilOp(state.getBackStencilStencilFailOperation()),
+                        glStencilOpFromStencilOp(state.getBackStencilDepthFailOperation()),
+                        glStencilOpFromStencilOp(state.getBackStencilDepthPassOperation())
+                        );
+                glStencilFuncSeparate(GL_FRONT,
+                        glStencilFuncFromStencilFunc(state.getFrontStencilFunction()),
+                        0,Integer.MAX_VALUE
+                        );
+                glStencilFuncSeparate(GL_BACK,
+                        glStencilFuncFromStencilFunc(state.getBackStencilFunction()),
+                        0,Integer.MAX_VALUE
+                        );
+            }else{
+                glDisable(GL_STENCIL_TEST);
+            }
+        }
+
+    }
+
+    private int glStencilOpFromStencilOp(RenderState.StencilOperation s){
+        switch(s){
+        case Keep:
+            return GL_KEEP;
+        case Zero:
+            return GL_ZERO;
+        case Replace:
+            return GL_REPLACE;
+        case Increment:
+            return GL_INCR;
+        case IncrementWrap:
+            return GL_INCR_WRAP;
+        case Decrement:
+            return GL_DECR;
+        case DecrementWrap:
+            return GL_DECR_WRAP;
+        case Invert:
+            return GL_INVERT;
+        default:
+            throw new UnsupportedOperationException("Unrecognized front stencil operation: " + s);
+        }  //end switch
+    }
+
+    private int glStencilFuncFromStencilFunc(RenderState.StencilFunction s){
+        switch(s){
+        case Never:
+            return GL_NEVER;
+        case Less:
+            return GL_LESS;
+        case LessEqual:
+            return GL_LEQUAL;
+        case Greater:
+            return GL_GREATER;
+        case GreaterEqual:
+            return GL_GEQUAL;
+        case Equal:
+            return GL_EQUAL;
+        case NotEqual:
+            return GL_NOTEQUAL;
+        case Always:
+            return GL_ALWAYS;
+        default:
+            throw new UnsupportedOperationException("Unrecognized front stencil functin: " + s);
+        }  //end switch
     }
 
     /*********************************************************************\
@@ -1524,7 +1615,7 @@ public class LwjglRenderer implements Renderer {
         }
     }
 
-    @SuppressWarnings("fallthrough") 
+    @SuppressWarnings("fallthrough")
     private void setupTextureParams(Texture tex) {
         Image image = tex.getImage();
         int target = convertTextureType(tex.getType(), image != null ? image.getMultiSamples() : 1);