Bug 716438; using flags to select a shader. r=Bas

gfx/layers/d3d10/CanvasLayerD3D10.cpp

@@ -264,53 +264,14 @@ CanvasLayerD3D10::RenderLayer()
 
   SetEffectTransformAndOpacity();
 
-  ID3D10EffectTechnique *technique;
+  PRUint8 shaderFlags = 0;
-
+  shaderFlags |= LoadMaskTexture();
-  if (LoadMaskTexture()) {
+  shaderFlags |= mDataIsPremultiplied
-    if (mDataIsPremultiplied) {
+                ? SHADER_PREMUL : SHADER_NON_PREMUL | SHADER_RGBA;
-      if (!mHasAlpha) {
+  shaderFlags |= mHasAlpha ? SHADER_RGBA : SHADER_RGB;
-        if (mFilter == gfxPattern::FILTER_NEAREST) {
+  shaderFlags |= mFilter == gfxPattern::FILTER_NEAREST
-          technique = effect()->GetTechniqueByName("RenderRGBLayerPremulPointMask");
+                ? SHADER_POINT : SHADER_LINEAR;
-        } else {
+  ID3D10EffectTechnique* technique = SelectShader(shaderFlags);
-          technique = effect()->GetTechniqueByName("RenderRGBLayerPremulMask");
-        }
-      } else {
-        if (mFilter == gfxPattern::FILTER_NEAREST) {
-          technique = effect()->GetTechniqueByName("RenderRGBALayerPremulPointMask");
-        } else {
-          technique = effect()->GetTechniqueByName("RenderRGBALayerPremulMask");
-        }
-      }
-    } else {
-      if (mFilter == gfxPattern::FILTER_NEAREST) {
-        technique = effect()->GetTechniqueByName("RenderRGBALayerNonPremulPointMask");
-      } else {
-        technique = effect()->GetTechniqueByName("RenderRGBALayerNonPremulMask");
-      }
-    }
-  } else { // no mask
-    if (mDataIsPremultiplied) {
-      if (!mHasAlpha) {
-        if (mFilter == gfxPattern::FILTER_NEAREST) {
-          technique = effect()->GetTechniqueByName("RenderRGBLayerPremulPoint");
-        } else {
-          technique = effect()->GetTechniqueByName("RenderRGBLayerPremul");
-        }
-      } else {
-        if (mFilter == gfxPattern::FILTER_NEAREST) {
-          technique = effect()->GetTechniqueByName("RenderRGBALayerPremulPoint");
-        } else {
-          technique = effect()->GetTechniqueByName("RenderRGBALayerPremul");
-        }
-      }
-    } else {
-      if (mFilter == gfxPattern::FILTER_NEAREST) {
-        technique = effect()->GetTechniqueByName("RenderRGBALayerNonPremulPoint");
-      } else {
-        technique = effect()->GetTechniqueByName("RenderRGBALayerNonPremul");
-      }
-    }
-  }
 
   if (mSRView) {
     effect()->GetVariableByName("tRGB")->AsShaderResource()->SetResource(mSRView);

gfx/layers/d3d10/ColorLayerD3D10.cpp

@@ -73,12 +73,7 @@ ColorLayerD3D10::RenderLayer()
   effect()->GetVariableByName("mLayerTransform")->SetRawValue(raw, 0, 64);
   effect()->GetVariableByName("fLayerColor")->AsVector()->SetFloatVector(color);
 
-  ID3D10EffectTechnique *technique;
+  ID3D10EffectTechnique *technique = SelectShader(SHADER_SOLID | LoadMaskTexture());
-  if (LoadMaskTexture()) {
-    technique = effect()->GetTechniqueByName("RenderSolidColorLayerMask");
-  } else {
-    technique = effect()->GetTechniqueByName("RenderSolidColorLayer");
-  }
 
   nsIntRegionRectIterator iter(mVisibleRegion);
 

gfx/layers/d3d10/ContainerLayerD3D10.cpp

@@ -316,12 +316,12 @@ ContainerLayerD3D10::RenderLayer()
     ID3D10EffectTechnique *technique;
     if (LoadMaskTexture()) {
       if (GetTransform().CanDraw2D()) {
-        technique = effect()->GetTechniqueByName("RenderRGBALayerPremulMask");
+        technique = SelectShader(SHADER_RGBA | SHADER_PREMUL | SHADER_MASK);
       } else {
-        technique = effect()->GetTechniqueByName("RenderRGBALayerPremulMask3D");
+        technique = SelectShader(SHADER_RGBA | SHADER_PREMUL | SHADER_MASK_3D);
       }
     } else {
-      technique = effect()->GetTechniqueByName("RenderRGBALayerPremul");
+        technique = SelectShader(SHADER_RGBA | SHADER_PREMUL | SHADER_NO_MASK);
     }
 
     effect()->GetVariableByName("vLayerQuad")->AsVector()->SetFloatVector(

gfx/layers/d3d10/ImageLayerD3D10.cpp

@@ -211,35 +211,14 @@ ImageLayerD3D10::RenderLayer()
       return;
     }
     
-    if (LoadMaskTexture()) {
+    PRUint8 shaderFlags = SHADER_PREMUL;
-      if (hasAlpha) {
+    shaderFlags |= LoadMaskTexture();
-        if (mFilter == gfxPattern::FILTER_NEAREST) {
+    shaderFlags |= hasAlpha
-          technique = effect()->GetTechniqueByName("RenderRGBALayerPremulPointMask");
+                  ? SHADER_RGBA : SHADER_RGB;
-        } else {
+    shaderFlags |= mFilter == gfxPattern::FILTER_NEAREST
-          technique = effect()->GetTechniqueByName("RenderRGBALayerPremulMask");
+                  ? SHADER_POINT : SHADER_LINEAR;
-        }
+    technique = SelectShader(shaderFlags);
-       } else {
+
-        if (mFilter == gfxPattern::FILTER_NEAREST) {
-          technique = effect()->GetTechniqueByName("RenderRGBLayerPremulPointMask");
-        } else {
-          technique = effect()->GetTechniqueByName("RenderRGBLayerPremulMask");
-        }
-       }
-     } else {
-      if (hasAlpha) {
-        if (mFilter == gfxPattern::FILTER_NEAREST) {
-          technique = effect()->GetTechniqueByName("RenderRGBALayerPremulPoint");
-        } else {
-          technique = effect()->GetTechniqueByName("RenderRGBALayerPremul");
-        }
-       } else {
-        if (mFilter == gfxPattern::FILTER_NEAREST) {
-          technique = effect()->GetTechniqueByName("RenderRGBLayerPremulPoint");
-        } else {
-          technique = effect()->GetTechniqueByName("RenderRGBLayerPremul");
-        }
-      }
-    }
 
     effect()->GetVariableByName("tRGB")->AsShaderResource()->SetResource(srView);
 
@@ -275,18 +254,12 @@ ImageLayerD3D10::RenderLayer()
       return;
     }
 
-    bool useMask = LoadMaskTexture();
-
     // TODO: At some point we should try to deal with mFilter here, you don't
     // really want to use point filtering in the case of NEAREST, since that
     // would also use point filtering for Chroma upsampling. Where most likely
     // the user would only want point filtering for final RGB image upsampling.
 
-    if (useMask) {
+    technique = SelectShader(SHADER_YCBCR | LoadMaskTexture());
-      technique = effect()->GetTechniqueByName("RenderYCbCrLayerMask");
-    } else {
-      technique = effect()->GetTechniqueByName("RenderYCbCrLayer");
-    }
 
     effect()->GetVariableByName("tY")->AsShaderResource()->SetResource(data->mYView);
     effect()->GetVariableByName("tCb")->AsShaderResource()->SetResource(data->mCbView);

gfx/layers/d3d10/LayerManagerD3D10.cpp

@@ -842,7 +842,56 @@ LayerD3D10::LayerD3D10(LayerManagerD3D10 *aManager)
 {
 }
 
-bool LayerD3D10::LoadMaskTexture()
+ID3D10EffectTechnique*
+LayerD3D10::SelectShader(PRUint8 aFlags)
+{
+  switch (aFlags) {
+  case (SHADER_RGBA | SHADER_NON_PREMUL | SHADER_LINEAR | SHADER_MASK):
+    return effect()->GetTechniqueByName("RenderRGBALayerNonPremulMask");
+  case (SHADER_RGBA | SHADER_NON_PREMUL | SHADER_LINEAR | SHADER_NO_MASK):
+    return effect()->GetTechniqueByName("RenderRGBALayerNonPremul");
+  case (SHADER_RGBA | SHADER_NON_PREMUL | SHADER_POINT | SHADER_NO_MASK):
+    return effect()->GetTechniqueByName("RenderRGBALayerNonPremulPoint");
+  case (SHADER_RGBA | SHADER_NON_PREMUL | SHADER_POINT | SHADER_MASK):
+    return effect()->GetTechniqueByName("RenderRGBALayerNonPremulPointMask");
+  case (SHADER_RGBA | SHADER_PREMUL | SHADER_LINEAR | SHADER_MASK_3D):
+    return effect()->GetTechniqueByName("RenderRGBALayerPremulMask3D");
+  case (SHADER_RGBA | SHADER_PREMUL | SHADER_LINEAR | SHADER_MASK):
+    return effect()->GetTechniqueByName("RenderRGBALayerPremulMask");
+  case (SHADER_RGBA | SHADER_PREMUL | SHADER_LINEAR | SHADER_NO_MASK):
+    return effect()->GetTechniqueByName("RenderRGBALayerPremul");
+  case (SHADER_RGBA | SHADER_PREMUL | SHADER_POINT | SHADER_MASK):
+    return effect()->GetTechniqueByName("RenderRGBALayerPremulPointMask");
+  case (SHADER_RGBA | SHADER_PREMUL | SHADER_POINT | SHADER_NO_MASK):
+    return effect()->GetTechniqueByName("RenderRGBALayerPremulPoint");
+  case (SHADER_RGB | SHADER_PREMUL | SHADER_POINT | SHADER_MASK):
+    return effect()->GetTechniqueByName("RenderRGBLayerPremulPointMask");
+  case (SHADER_RGB | SHADER_PREMUL | SHADER_POINT | SHADER_NO_MASK):
+    return effect()->GetTechniqueByName("RenderRGBLayerPremulPoint");
+  case (SHADER_RGB | SHADER_PREMUL | SHADER_LINEAR | SHADER_MASK):
+    return effect()->GetTechniqueByName("RenderRGBLayerPremulMask");
+  case (SHADER_RGB | SHADER_PREMUL | SHADER_LINEAR | SHADER_NO_MASK):
+    return effect()->GetTechniqueByName("RenderRGBLayerPremul");
+  case (SHADER_SOLID | SHADER_MASK):
+    return effect()->GetTechniqueByName("RenderSolidColorLayerMask");
+  case (SHADER_SOLID | SHADER_NO_MASK):
+    return effect()->GetTechniqueByName("RenderSolidColorLayer");
+  case (SHADER_COMPONENT_ALPHA | SHADER_MASK):
+    return effect()->GetTechniqueByName("RenderComponentAlphaLayerMask");
+  case (SHADER_COMPONENT_ALPHA | SHADER_NO_MASK):
+    return effect()->GetTechniqueByName("RenderComponentAlphaLayer");
+  case (SHADER_YCBCR | SHADER_MASK):
+    return effect()->GetTechniqueByName("RenderYCbCrLayerMask");
+  case (SHADER_YCBCR | SHADER_NO_MASK):
+    return effect()->GetTechniqueByName("RenderYCbCrLayer");
+  default:
+    NS_ERROR("Invalid shader.");
+    return nsnull;
+  }
+}
+
+PRUint8
+LayerD3D10::LoadMaskTexture()
 {
   if (Layer* maskLayer = GetLayer()->GetMaskLayer()) {
     gfxIntSize size;
@@ -850,13 +899,14 @@ bool LayerD3D10::LoadMaskTexture()
       static_cast<LayerD3D10*>(maskLayer->ImplData())->GetAsTexture(&size);
   
     if (!maskSRV) {
-      return false;
+      return SHADER_NO_MASK;
     }
 
     gfxMatrix maskTransform;
-    bool maskIs2D = maskLayer->GetTransform().CanDraw2D(&maskTransform);
+    bool maskIs2D = maskLayer->GetEffectiveTransform().CanDraw2D(&maskTransform);
     NS_ASSERTION(maskIs2D, "How did we end up with a 3D transform here?!");
-    gfxRect bounds = gfxRect(maskTransform.GetTranslation(), size);
+    gfxRect bounds = gfxRect(gfxPoint(), size);
+    bounds = maskTransform.TransformBounds(bounds);
 
     effect()->GetVariableByName("vMaskQuad")->AsVector()->SetFloatVector(
       ShaderConstantRectD3D10(
@@ -867,10 +917,10 @@ bool LayerD3D10::LoadMaskTexture()
       );
 
     effect()->GetVariableByName("tMask")->AsShaderResource()->SetResource(maskSRV);
-    return true;
+    return SHADER_MASK;
   }
 
-  return false; 
+  return SHADER_NO_MASK; 
 }
 
 WindowLayer::WindowLayer(LayerManagerD3D10* aManager)

gfx/layers/d3d10/LayerManagerD3D10.h

@@ -319,10 +319,33 @@ protected:
   /*
    * Finds a texture for this layer's mask layer (if it has one) and sets it
    * as an input to the shaders.
-   * Returns true if a texture is loaded, false if there was no mask layer, or
+   * Returns SHADER_MASK if a texture is loaded, SHADER_NO_MASK if there was no 
-   * a texture for the mask layer could not be loaded.
+   * mask layer, or a texture for the mask layer could not be loaded.
    */
-  bool LoadMaskTexture();
+  PRUint8 LoadMaskTexture();
+
+  /**
+   * Select a shader technique using a combination of the following flags.
+   * Not all combinations of flags are supported, and might cause an error,
+   * check the fx file to see which shaders exist. In particular, aFlags should
+   * include any combination of the 0x20 bit = 0 flags OR one of the 0x20 bit = 1
+   * flags. Mask flags can be used in either case.
+   */
+  ID3D10EffectTechnique* SelectShader(PRUint8 aFlags);
+  const static PRUint8 SHADER_NO_MASK = 0;
+  const static PRUint8 SHADER_MASK = 0x1;
+  const static PRUint8 SHADER_MASK_3D = 0x2;
+  // 0x20 bit = 0
+  const static PRUint8 SHADER_RGB = 0;
+  const static PRUint8 SHADER_RGBA = 0x4;
+  const static PRUint8 SHADER_NON_PREMUL = 0;
+  const static PRUint8 SHADER_PREMUL = 0x8;
+  const static PRUint8 SHADER_LINEAR = 0;
+  const static PRUint8 SHADER_POINT = 0x10;
+  // 0x20 bit = 1
+  const static PRUint8 SHADER_YCBCR = 0x20;
+  const static PRUint8 SHADER_COMPONENT_ALPHA = 0x24;
+  const static PRUint8 SHADER_SOLID = 0x28;
 
   LayerManagerD3D10 *mD3DManager;
 };

gfx/layers/d3d10/ThebesLayerD3D10.cpp

@@ -132,36 +132,19 @@ ThebesLayerD3D10::RenderLayer()
   SetEffectTransformAndOpacity();
 
   ID3D10EffectTechnique *technique;
-  if (LoadMaskTexture()) {
+  switch (mCurrentSurfaceMode) {
-    switch (mCurrentSurfaceMode) {
+  case SURFACE_COMPONENT_ALPHA:
-    case SURFACE_COMPONENT_ALPHA:
+    technique = SelectShader(SHADER_COMPONENT_ALPHA | LoadMaskTexture());
-      technique = effect()->GetTechniqueByName("RenderComponentAlphaLayerMask");
+    break;
-      break;
+  case SURFACE_OPAQUE:
-    case SURFACE_OPAQUE:
+    technique = SelectShader(SHADER_RGB | SHADER_PREMUL | LoadMaskTexture());
-      technique = effect()->GetTechniqueByName("RenderRGBLayerPremulMask");
+    break;
-      break;
+  case SURFACE_SINGLE_CHANNEL_ALPHA:
-    case SURFACE_SINGLE_CHANNEL_ALPHA:
+    technique = SelectShader(SHADER_RGBA | SHADER_PREMUL | LoadMaskTexture());
-      technique = effect()->GetTechniqueByName("RenderRGBALayerPremulMask");
+    break;
-      break;
+  default:
-    default:
+    NS_ERROR("Unknown mode");
-      NS_ERROR("Unknown mode");
+    return;
-      return;
-    }
-  } else {
-    switch (mCurrentSurfaceMode) {
-    case SURFACE_COMPONENT_ALPHA:
-      technique = effect()->GetTechniqueByName("RenderComponentAlphaLayer");
-      break;
-    case SURFACE_OPAQUE:
-      technique = effect()->GetTechniqueByName("RenderRGBLayerPremul");
-      break;
-    case SURFACE_SINGLE_CHANNEL_ALPHA:
-      technique = effect()->GetTechniqueByName("RenderRGBALayerPremul");
-      break;
-    default:
-      NS_ERROR("Unknown mode");
-      return;
-    }
   }
 
   nsIntRegionRectIterator iter(mVisibleRegion);
@@ -652,12 +635,7 @@ ShadowThebesLayerD3D10::RenderLayer()
 
   SetEffectTransformAndOpacity();
 
-  ID3D10EffectTechnique *technique;
+  ID3D10EffectTechnique *technique = SelectShader(SHADER_RGB | SHADER_PREMUL | LoadMaskTexture());
-  if (LoadMaskTexture()) {
-    technique = effect()->GetTechniqueByName("RenderRGBLayerPremulMask");
-  } else {
-    technique = effect()->GetTechniqueByName("RenderRGBLayerPremul");
-  }
 
   effect()->GetVariableByName("tRGB")->AsShaderResource()->SetResource(srView);