I have been working on a metaball compute shader for roughly 3 months now, and the only issue I have to resolve is noise in the output Dest[id.xy]. I also asked GPT 4 about the crux of the issue and how to fix it, to no avail. I really need help from someone who has substantial experience with compute shaders. So the issue is the noise as you can see:
MainCameraController.cs
using System.Collections;
using System.Collections.Generic;
using UnityEngine;
[RequireComponent(typeof(Camera))]
public class MainCameraController : MonoBehaviour
{
//Game Object Manager
public GameObject objectManager;
private ObjectManagerController objectManagerController;
//Shader
private ComputeShader metaballComputeShader;
private Camera _mainCamera;
public Camera _depthCamera;
public Camera _operationCamera;
//Compute Shader variables
private int threadsPerGroup = 8;
private int kernelHandle = 0;
//List to maintain game objects
private List<GameObject> colorSlimes = new List<GameObject>();
private RenderTexture depthTextureArray;
private RenderTexture colorTextureArray;
void Awake() {
//depthTextureArray = new Texture2DArray(_mainCamera.pixelWidth, _mainCamera.pixelWidth, 1, TextureFormat.ARGB32, true, false);
//colorTextureArray = new Texture2DArray(_mainCamera.pixelWidth, _mainCamera.pixelWidth, 1, TextureFormat.ARGB32, true, false);
Debug.Log("Awake for Main camera.");
objectManagerController = objectManager.GetComponent<ObjectManagerController>();
//Initialize texture arrays
colorTextureArray = objectManagerController.colorTextureArrayGlobal;
depthTextureArray = objectManagerController.depthTextureArrayGlobal;
//Initialize Shader
metaballComputeShader = objectManagerController.metaballComputeShaderGlobal;
//Initialize color slime List
colorSlimes = objectManagerController.colorSlimesGlobal;
}
void Start()
{
kernelHandle = metaballComputeShader.FindKernel("CSMain");
}
// private void Reset()
// {
// _mainCamera = GetComponent<Camera>();
// }
// private void OnEnable() {
// Debug.Log("OnEnable for main camera.");
// if (!_mainCamera) _mainCamera = GetComponent<Camera>();
// if (_mainCamera) _mainCamera.depthTextureMode |= DepthTextureMode.Depth;
// _mainCamera.transparencySortMode = TransparencySortMode.Orthographic;
// }
// private void OnDisable()
// {
// if (_mainCamera) _mainCamera.depthTextureMode = DepthTextureMode.None;
// }
// private void OnDestroy()
// {
// if (_mainCamera) _mainCamera.depthTextureMode = DepthTextureMode.None;
// }
//Apply metaball effect
public void OnRenderImage(RenderTexture src, RenderTexture dest) {
Debug.Log("Metaball Shader");
//Thread control
int threadGroupX = Mathf.CeilToInt(src.width / (float)threadsPerGroup);
int threadGroupY = Mathf.CeilToInt(src.height / (float)threadsPerGroup);
//depthTextureArray.dimension = UnityEngine.Rendering.TextureDimension.Tex2DArray;
//depthTextureArray.enableRandomWrite = true;
//depthTextureArray.volumeDepth = colorSlimes.Count;
//depthTextureArray.Create();
//colorTextureArray.dimension = UnityEngine.Rendering.TextureDimension.Tex2DArray;
//colorTextureArray.enableRandomWrite = true;
//colorTextureArray.volumeDepth = colorSlimes.Count; ?
//colorTextureArray.Create();
//Prepare buffers
//Initialize array of 25s
ComputeBuffer minimumDistanceValueBuffer = new ComputeBuffer(colorSlimes.Count, sizeof(float));
//Maybe the problem is due to the clash between SetData from c# and resetting
//This value from shader.
// List<float> minimumDistances = new List<float>();
// for(int i = 0; i < colorSlimes.Count; i++) {
// minimumDistances.Add(20);
// }
//minimumDistanceValueBuffer.SetData(minimumDistances);
ComputeBuffer minimumDistancePixelBuffer = new ComputeBuffer(colorSlimes.Count, sizeof(float)*4);
//Make a writable copy of src and dest
RenderTexture srcWritable = new RenderTexture(src.width, src.height, 32, RenderTextureFormat.ARGB32, RenderTextureReadWrite.Default){ useMipMap = false };
srcWritable.enableRandomWrite = true; // Set the UAV usage flag
srcWritable.Create();
RenderTexture destWritable = new RenderTexture(srcWritable);
destWritable.Create();
RenderTexture.active = src;
Graphics.Blit(src, srcWritable);
//Fill compute shader
metaballComputeShader.SetTexture(kernelHandle, "Src", srcWritable);
metaballComputeShader.SetTexture(kernelHandle, "Dest", destWritable);
metaballComputeShader.SetBuffer(kernelHandle, "MinimumDistancePixelBuffer", minimumDistancePixelBuffer);
metaballComputeShader.SetBuffer(kernelHandle, "MinimumDistanceValueBuffer", minimumDistanceValueBuffer);
metaballComputeShader.SetFloat("MaxWidth", src.width);
metaballComputeShader.SetFloat("GameObjectListCount", (float)colorSlimes.Count);
metaballComputeShader.SetFloat("MaxHeight", src.height);
metaballComputeShader.SetTexture(kernelHandle, "ColorTextureArray", colorTextureArray);
metaballComputeShader.SetTexture(kernelHandle, "DepthTextureArray", depthTextureArray);
metaballComputeShader.Dispatch(kernelHandle, threadGroupX, threadGroupY, 1);
//Write metaball effect to final dest
Graphics.SetRenderTarget(destWritable);
Graphics.Blit(destWritable, dest);
RenderTexture.active = null;
minimumDistancePixelBuffer.Release();
minimumDistanceValueBuffer.Release();
srcWritable.Release();
destWritable.Release();
Debug.Log("Metaball effect complete!");
}
}
MetaballComputeShader.compute
[numthreads(8,8,1)]
void CSMain (uint3 id : SV_DispatchThreadID)
{
//Dest[id.xy] = float4(1,1,1,1);
// Convolution:
// 1 0 -1
// 2 0 -2
// 1 0 -1
//25x25 matrix
//13th pixel == center x == center y
//For every pixel, after doing operations on itself, start from itself's
//id.xy - 13~self~+11
//First check whether this pixel is opaque
//Src[id.xy]
//DepthTextureArray.Load(uint3(id.xy, i))
//color = //either Src[id.xy] if first or DepthTextureArray.Load(uint3(id.xy, depthSearchCount++)) for the rest
//If Src is not opaque, rest assured.
if(int(id.x) - startX < 0 || int(id.x) + startX >= MaxWidth || int(id.y) - startY < 0 || int(id.y) + startY >= MaxHeight) {
//Overflow prevention.
//For now, ignore screen edges.
return;
}
//RWStructuredBuffer minimumDistancePixelBuffer;
float4 MinimumDistancePixelBufferTest[2];
for (int i = 0; i < (int)GameObjectListCount; i++) {
MinimumDistanceValueBuffer[i] = 20;
//Value here must be equal to value set in
MinimumDistancePixelBufferTest[i].rgba = float4(1, 1, 1, 1).rgba;
//minimumDistancePixelBuffer[i] = float4(0.5,0.5,0.5,0.5);
}
//Dest[id.xy] = Src[id.xy];
float4 srcPixel = Src[id.xy];
if(!all(srcPixel <= nullColor)) {
int numberOfVisibleObjectsAtCurrent = 0;
for(int i = 0; i < GameObjectListCount; i++) {
float4 colorAtDepth = ColorTextureArray.Load(uint3(id.xy, i));
if(!all(colorAtDepth <= nullColor)) {
if(all(colorAtDepth != srcPixel)) {
srcPixel += colorAtDepth;
}
numberOfVisibleObjectsAtCurrent++;
}
}
//Dest[id.xy] = srcPixel / numberOfVisibleObjectsAtCurrent;
//Dest[id.xy] = Dest[id.xy] / GameObjectListCount;
//Dest[id.xy] = Src[id.xy];
return;
}
else {
//Is Src[id.xy] at that location visible? If not, move on
//int2(-13, -13)
//Since we're now subtracting,
//Dest[id.xy] = Src[id.xy - int2(25, 0)];
//Dest[id.xy] = float4(1,1,1,1);
bool emptyMatrix = true;
//Something's wrong with the for loop
//[loop]
//Problem had to do with using startX and startY instead of 25(actual number)
//Maybe Unity compute shaders are too dumb to recognize initialized variables
//Don't necessarily have to unroll, but use numbers instead of variables when defining
//condition in for loop.
for(int i = -24; i <= 25; i++) {
for(int j = -24; j <= 25; j++) {
//int2 currentPosDelta = int2(r, g);
float4 srcPixel = Src[id.xy - int2(j, i)];
//float4 minimumDistancePixel = float4(0,0,0,0);
//minimumDistancePixel.rgba = srcPixel.rgba;
//Dest[id.xy] = Src[id.xy - int2(15, 0)];
if(!all(srcPixel.rgba <= nullColor.rgba)) {
//This means we have found an opaque pixel in the
//25x25 convolution matrix
//What I want to do:
//Update minimum distance for each game object based on
//
//emptyMatrix = false;
//Line below makes noise in game view
//No longer makes noise
float minimumDistance = CalcMinimumDistance(int2(id.xy - int2(j, i)), int2(id.xy));
//[unroll(2)]
for(int k = 0; k < (int)GameObjectListCount; k++) {
float4 minimumDistancePixel = ColorTextureArray.Load(uint3(id.xy - int2(j, i), k));
//minimumDistance < MinimumDistanceValueBuffer[k] &&
//minimumDistance < MinimumDistanceValueBuffer[k] &&
//&& !all(minimumDistancePixel.rgba <= nullColor.rgba)
if(any(minimumDistancePixel.rgba > nullColor.rgba) && minimumDistance < MinimumDistanceValueBuffer[k]) {
//No more noise
MinimumDistanceValueBuffer[k] = minimumDistance;
//The line below works fine.
//Setting MinimumDistancePixelBuffer anything other than 1,1,1,1 causes noise
//For some stupid reason, the value I set here must agree with the values I set in the beginning of the function
//
MinimumDistancePixelBufferTest[k].rgba = float4(0.5,0.5,0.5,0.5).rgba;
// Add a memory barrier here to ensure proper synchronization
GroupMemoryBarrierWithGroupSync();
//float4(1,1,1,1).rgba;
//emptyMatrix = false;
// we've checked that this works
emptyMatrix = false;
}
}
//Noise had to do with sizeof(float) in a compute shader slightly larger
//than the actual sizeof(float) in compute shader.
//float minimumDistance = CalcMinimumDistance(id.xy - int2(j, i), id.xy);
//Sample Depth Texture at that location: iterate through the z of that id.rg
//To see all the game objects that the pixel is part of.
//A minimum distance will be calculated for each game object.
//float minimumDistance = CalcMinimumDistance(id.xy - int2(j, i), id.xy);
// for(int b = 0; b < 2; b++) {
// srcPixel.rgba = ColorTextureArray.Load(uint3(id.xy - int2(j, i), b)).rgba;
// if(minimumDistance < MinimumDistanceValueBuffer[b] && !all(srcPixel <= nullColor)) {
// MinimumDistanceValueBuffer[b] = minimumDistance;
// MinimumDistancePixelBuffer[b].rgba = srcPixel.rgba;
// }
// minimumDistancePixel.rgba += srcPixel.rgba;
// }
//Dest[id.xy] = minimumDistancePixel / GameObjectListCount;
//emptyMatrix = false;
//emptyMatrix = false;
}
}
//Dest[id.xy] = float4(1,1,1,1);
}
//emptyMatrix = false;
//At this point, either emptyMatrix or we have all the information needed to apply to this pixel
if(!emptyMatrix) {
//Dest[id.xy] already stores average Src of the location with min dist
//Metaball calculation
//double not supported
float avgDistance = 0;
float4 totalColor = float4(1, 1, 1, 1);
float avgColorRed = 0;
float avgColorBlue = 0;
float avgColorGreen = 0;
float avgColorAlpha = 0;
//Maybe think about storing each rgba in a separate float
//Simple average
int averageCounter = 1;
for(int i = 0; i < (int)GameObjectListCount; i++) {
if(MinimumDistanceValueBuffer[i] < 20) {
avgDistance += MinimumDistanceValueBuffer[i];
//totalColor.rgba = MinimumDistancePixelBuffer[i].rgba;
//Dest[id.xy].rgba += MinimumDistancePixelBuffer[i].rgba;
//avgColorGreen += MinimumDistancePixelBuffer[i].g;
//avgColorBlue += MinimumDistancePixelBuffer[i].b;
//avgColorAlpha += MinimumDistancePixelBuffer[i].a;
//averageCounter++;
//totalColor = tmpColor
}
}
avgDistance /= averageCounter;
//float4 avgColor = round(totalColor / averageCounter);
// avgColorRed = avgColorRed / averageCounter;
// avgColorGreen = avgColorGreen / averageCounter;
// avgColorBlue = avgColorBlue / averageCounter;
// avgColorAlpha = avgColorAlpha / averageCounter;
//float4 avgColor = float4(avgColorRed, avgColorGreen, avgColorBlue, avgColorAlpha);
if(avgDistance < 10) {
//Dest[id.xy] = avgColor;
//Storing avgColor in Dest causes noise.
//What is noise?
//Dest[id.xy] is not fixed
//MinimumDistancePixelBuffer[0] is not fixed
//At the end of the nested for loop, MinimumDistancePixelBuffer[0] must be fixed to the
//minimum distance from game object 0 to a pixel in the convolution matrix.
//The problem with the algorithm right now:
//
Dest[id.xy] = MinimumDistancePixelBufferTest[1];
}
// for(int i = 0; i < (int)GameObjectListCount; i++) {
// MinimumDistanceValueBuffer[i] = 20;
// }
// if(totalWeight > 0) {
// Dest[id.xy] = accumulatedColor / totalWeight;
// } else {
// Dest[id.xy] = srcPixel;
// }
// float dynamicThreshold = count > 0 ? (totalDistance / (25.0f * count)) : 0;
// if(dynamicThreshold > 0.4) {
// Dest[id.xy] = weightedColor / accumulatedColor / totalWeight;;
// }
//game object0Color * distanceN + game object1Color * distanceN-1 ... / aggregate distance
//This order does not necessarily follow order of creation for game objects
//Dest[id.xy] = float4(1,1,1,1);
}
// else {
// Dest[id.xy] = Src[id.xy];
// //Dest[id.xy] = float4(1,1,1,1);
// }
//Pixel color of game object with min distance will be multiplied by max distance
//average = each game object's rgba * (minOfMin/)
//Dest[id.xy] = float4(1,1,1,1);
//Dest[id.xy] = Src[id.xy];
return;
}
}
And when I change the threadsPerGroup to a lower number like 1, there is much less noise, but now the game is ridiculously laggy. Furthermore, there is still some noise, which is not the behavior I want.
Chat gpt mentioned memory grouping as a potential solution, and I have no idea how to use them, and the documentation or tutorials for unity compute shader programming is practically null. Please help me to understand how to minimize noise while maximizing game performance at the same time, without relying on the threadsPerGroup, so that I can have many threads while still maintaining the integrity of the computations done in the compute shader.