From 225afd0f92b24960ff6f9c0d08a01946fa2e7cec Mon Sep 17 00:00:00 2001 From: Nehon Date: Thu, 17 Aug 2017 23:58:08 +0200 Subject: [PATCH] Compute the 4 more weighted bone index for skin animation when there are several Joint buffers for a mesh. --- .../jme3/scene/plugins/gltf/GltfLoader.java | 122 +++++++++++--- .../jme3/scene/plugins/gltf/GltfUtils.java | 157 ++++++++++++++---- 2 files changed, 228 insertions(+), 51 deletions(-) diff --git a/jme3-plugins/src/gltf/java/com/jme3/scene/plugins/gltf/GltfLoader.java b/jme3-plugins/src/gltf/java/com/jme3/scene/plugins/gltf/GltfLoader.java index ce181ec73..e3b605ec8 100644 --- a/jme3-plugins/src/gltf/java/com/jme3/scene/plugins/gltf/GltfLoader.java +++ b/jme3-plugins/src/gltf/java/com/jme3/scene/plugins/gltf/GltfLoader.java @@ -11,6 +11,8 @@ import com.jme3.renderer.queue.RenderQueue; import com.jme3.scene.*; import com.jme3.texture.Texture; import com.jme3.texture.Texture2D; +import com.jme3.util.BufferUtils; +import com.jme3.util.IntMap; import com.jme3.util.mikktspace.MikktspaceTangentGenerator; import java.io.*; @@ -50,13 +52,12 @@ public class GltfLoader implements AssetLoader { private FloatArrayPopulator floatArrayPopulator = new FloatArrayPopulator(); private Vector3fArrayPopulator vector3fArrayPopulator = new Vector3fArrayPopulator(); private QuaternionArrayPopulator quaternionArrayPopulator = new QuaternionArrayPopulator(); - private Matrix4fArrayPopulator matrix4fArrayPopulator = new Matrix4fArrayPopulator(); private static Map defaultMaterialAdapters = new HashMap<>(); private boolean useNormalsFlag = false; - private Transform tmpTransforms = new Transform(); private Quaternion tmpQuat = new Quaternion(); Map> skinnedSpatials = new HashMap<>(); + IntMap skinBuffers = new IntMap<>(); static { defaultMaterialAdapters.put("pbrMetallicRoughness", new PBRMaterialAdapter()); @@ -323,9 +324,29 @@ public class GltfLoader implements AssetLoader { } JsonObject attributes = meshObject.getAsJsonObject("attributes"); assertNotNull(attributes, "No attributes defined for mesh " + mesh); + + skinBuffers.clear(); + for (Map.Entry entry : attributes.entrySet()) { - mesh.setBuffer(readAccessorData(entry.getValue().getAsInt(), new VertexBufferPopulator(getVertexBufferType(entry.getKey())))); + //special case for joints and weights buffer. If there are more than 4 bones per vertex, there might be several of them + //we need to read them all and to keep only the 4 that have the most weight on the vertex. + String bufferType = entry.getKey(); + if (bufferType.startsWith("JOINTS")) { + SkinBuffers buffs = getSkinBuffers(bufferType); + SkinBuffers buffer = readAccessorData(entry.getValue().getAsInt(), new JointArrayPopulator()); + buffs.joints = buffer.joints; + buffs.componentSize = buffer.componentSize; + } else if (bufferType.startsWith("WEIGHTS")) { + SkinBuffers buffs = getSkinBuffers(bufferType); + buffs.weights = readAccessorData(entry.getValue().getAsInt(), new FloatArrayPopulator()); + } else { + VertexBuffer vb = readAccessorData(entry.getValue().getAsInt(), new VertexBufferPopulator(getVertexBufferType(bufferType))); + if (vb != null) { + mesh.setBuffer(vb); + } + } } + handleSkinningBuffers(mesh, skinBuffers); if (mesh.getBuffer(VertexBuffer.Type.BoneIndex) != null) { //the mesh has some skinning let's create needed buffers for HW skinning @@ -374,6 +395,28 @@ public class GltfLoader implements AssetLoader { return geomArray; } + public static class WeightData { + float value; + short index; + int componentSize; + + public WeightData(float value, short index, int componentSize) { + this.value = value; + this.index = index; + this.componentSize = componentSize; + } + } + + private SkinBuffers getSkinBuffers(String bufferType) { + int bufIndex = getIndex(bufferType); + SkinBuffers buffs = skinBuffers.get(bufIndex); + if (buffs == null) { + buffs = new SkinBuffers(); + skinBuffers.put(bufIndex, buffs); + } + return buffs; + } + private R readAccessorData(int accessorIndex, Populator populator) throws IOException { assertNotNull(accessors, "No accessor attribute in the gltf file"); @@ -399,8 +442,7 @@ public class GltfLoader implements AssetLoader { return populator.populate(bufferViewIndex, componentType, type, count, byteOffset); } - private void readBuffer(Integer bufferViewIndex, int byteOffset, int bufferSize, Object store, int numComponents) throws IOException { - + private void readBuffer(Integer bufferViewIndex, int byteOffset, int bufferSize, Object store, int numComponents, int componentSize) throws IOException { JsonObject bufferView = bufferViews.get(bufferViewIndex).getAsJsonObject(); Integer bufferIndex = getAsInteger(bufferView, "buffer"); @@ -415,7 +457,7 @@ public class GltfLoader implements AssetLoader { //int target = getAsInteger(bufferView, "target", 0); byte[] data = readData(bufferIndex); - populateBuffer(store, data, bufferSize, byteOffset + bvByteOffset, byteStride, numComponents); + populateBuffer(store, data, bufferSize, byteOffset + bvByteOffset, byteStride, numComponents, componentSize); //TODO extensions? //TODO extras? @@ -606,7 +648,15 @@ public class GltfLoader implements AssetLoader { //check if we are loading the same time array //TODO specs actually don't forbid this...maybe remove this check and handle it. if (animData.times != times) { - throw new AssetLoadException("Channel has different input accessors for samplers"); + logger.log(Level.WARNING, "Channel has different input accessors for samplers"); +// for (float time : animData.times) { +// System.err.print(time + ", "); +// } +// System.err.println(""); +// for (float time : times) { +// System.err.print(time + ", "); +// } +// System.err.println(""); } } if (animData.length == null) { @@ -623,7 +673,6 @@ public class GltfLoader implements AssetLoader { Quaternion[] rotations = readAccessorData(dataIndex, quaternionArrayPopulator); animData.rotations = rotations; } - } if (name == null) { @@ -911,6 +960,20 @@ public class GltfLoader implements AssetLoader { Transform armatureTransforms; } + public static class SkinBuffers { + short[] joints; + float[] weights; + int componentSize; + + public SkinBuffers(short[] joints, int componentSize) { + this.joints = joints; + this.componentSize = componentSize; + } + + public SkinBuffers() { + } + } + private interface Populator { T populate(Integer bufferViewIndex, int componentType, String type, int count, int byteOffset) throws IOException; } @@ -925,6 +988,11 @@ public class GltfLoader implements AssetLoader { @Override public VertexBuffer populate(Integer bufferViewIndex, int componentType, String type, int count, int byteOffset) throws IOException { + if (bufferType == null) { + logger.log(Level.WARNING, "could not assign data to any VertexBuffer type for buffer view " + bufferViewIndex); + return null; + } + VertexBuffer vb = new VertexBuffer(bufferType); VertexBuffer.Format format = getVertexBufferFormat(componentType); int numComponents = getNumberOfComponents(type); @@ -935,7 +1003,7 @@ public class GltfLoader implements AssetLoader { //no referenced buffer, specs says to pad the buffer with zeros. padBuffer(buff, bufferSize); } else { - readBuffer(bufferViewIndex, byteOffset, bufferSize, buff, numComponents); + readBuffer(bufferViewIndex, byteOffset, bufferSize, buff, numComponents, format.getComponentSize()); } if (bufferType == VertexBuffer.Type.Index) { @@ -955,13 +1023,13 @@ public class GltfLoader implements AssetLoader { int numComponents = getNumberOfComponents(type); int dataSize = numComponents * count; - float[] data = new float[count]; + float[] data = new float[dataSize]; if (bufferViewIndex == null) { //no referenced buffer, specs says to pad the data with zeros. padBuffer(data, dataSize); } else { - readBuffer(bufferViewIndex, byteOffset, dataSize, data, numComponents); + readBuffer(bufferViewIndex, byteOffset, dataSize, data, numComponents, 4); } return data; @@ -982,9 +1050,8 @@ public class GltfLoader implements AssetLoader { //no referenced buffer, specs says to pad the data with zeros. padBuffer(data, dataSize); } else { - readBuffer(bufferViewIndex, byteOffset, dataSize, data, numComponents); + readBuffer(bufferViewIndex, byteOffset, dataSize, data, numComponents, 4); } - return data; } } @@ -1002,30 +1069,47 @@ public class GltfLoader implements AssetLoader { //no referenced buffer, specs says to pad the data with zeros. padBuffer(data, dataSize); } else { - readBuffer(bufferViewIndex, byteOffset, dataSize, data, numComponents); + readBuffer(bufferViewIndex, byteOffset, dataSize, data, numComponents, 4); } return data; } } - private class Matrix4fArrayPopulator implements Populator { + private class JointData { + short[] joints; + int componentSize; + + public JointData(short[] joints, int componentSize) { + this.joints = joints; + this.componentSize = componentSize; + } + } + + private class JointArrayPopulator implements Populator { @Override - public Matrix4f[] populate(Integer bufferViewIndex, int componentType, String type, int count, int byteOffset) throws IOException { + public SkinBuffers populate(Integer bufferViewIndex, int componentType, String type, int count, int byteOffset) throws IOException { int numComponents = getNumberOfComponents(type); + + //can be bytes or shorts. + int componentSize = 1; + if (componentType == 5123) { + componentSize = 2; + } + int dataSize = numComponents * count; - Matrix4f[] data = new Matrix4f[count]; + short[] data = new short[dataSize]; if (bufferViewIndex == null) { //no referenced buffer, specs says to pad the data with zeros. padBuffer(data, dataSize); } else { - readBuffer(bufferViewIndex, byteOffset, dataSize, data, numComponents); + readBuffer(bufferViewIndex, byteOffset, dataSize, data, numComponents, componentSize); } - return data; + return new SkinBuffers(data, componentSize); } } } diff --git a/jme3-plugins/src/gltf/java/com/jme3/scene/plugins/gltf/GltfUtils.java b/jme3-plugins/src/gltf/java/com/jme3/scene/plugins/gltf/GltfUtils.java index 7b03f333b..36d2b2946 100644 --- a/jme3-plugins/src/gltf/java/com/jme3/scene/plugins/gltf/GltfUtils.java +++ b/jme3-plugins/src/gltf/java/com/jme3/scene/plugins/gltf/GltfUtils.java @@ -10,20 +10,21 @@ import com.jme3.scene.Mesh; import com.jme3.scene.Spatial; import com.jme3.scene.VertexBuffer; import com.jme3.texture.Texture; -import com.jme3.util.LittleEndien; +import com.jme3.util.*; import java.io.*; import java.nio.*; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; +import java.util.*; +import java.util.logging.Level; +import java.util.logging.Logger; /** * Created by Nehon on 07/08/2017. */ public class GltfUtils { + private static final Logger logger = Logger.getLogger(GltfUtils.class.getName()); + public static Mesh.Mode getMeshMode(Integer mode) { if (mode == null) { return Mesh.Mode.Triangles; @@ -120,11 +121,17 @@ public class GltfUtils { case "WEIGHTS_0": return VertexBuffer.Type.BoneWeight; default: - throw new AssetLoadException("Unsupported buffer attribute: " + attribute); + logger.log(Level.WARNING, "Unsupported Vertex Buffer type " + attribute); + return null; } } + public static int getIndex(String name) { + String num = name.substring(name.lastIndexOf("_") + 1); + return Integer.parseInt(num); + } + public static Texture.MagFilter getMagFilter(Integer value) { if (value == null) { return null; @@ -201,7 +208,12 @@ public class GltfUtils { } buffer.rewind(); } - if (store instanceof float[]) { + if (store instanceof short[]) { + short[] array = (short[]) store; + for (int i = 0; i < array.length; i++) { + array[i] = 0; + } + } else if (store instanceof float[]) { float[] array = (float[]) store; for (int i = 0; i < array.length; i++) { array[i] = 0; @@ -224,41 +236,43 @@ public class GltfUtils { } } - public static void populateBuffer(Object store, byte[] source, int length, int byteOffset, int byteStride, int numComponents) throws IOException { + public static void populateBuffer(Object store, byte[] source, int length, int byteOffset, int byteStride, int numComponents, int componentSize) throws IOException { if (store instanceof Buffer) { Buffer buffer = (Buffer) store; buffer.clear(); if (buffer instanceof ByteBuffer) { - populateByteBuffer((ByteBuffer) buffer, source, length, byteOffset, byteStride, numComponents); + populateByteBuffer((ByteBuffer) buffer, source, length, byteOffset, byteStride, numComponents, componentSize); return; } LittleEndien stream = getStream(source); if (buffer instanceof ShortBuffer) { - populateShortBuffer((ShortBuffer) buffer, stream, length, byteOffset, byteStride, numComponents); + populateShortBuffer((ShortBuffer) buffer, stream, length, byteOffset, byteStride, numComponents, componentSize); } else if (buffer instanceof IntBuffer) { - populateIntBuffer((IntBuffer) buffer, stream, length, byteOffset, byteStride, numComponents); + populateIntBuffer((IntBuffer) buffer, stream, length, byteOffset, byteStride, numComponents, componentSize); } else if (buffer instanceof FloatBuffer) { - populateFloatBuffer((FloatBuffer) buffer, stream, length, byteOffset, byteStride, numComponents); + populateFloatBuffer((FloatBuffer) buffer, stream, length, byteOffset, byteStride, numComponents, componentSize); } buffer.rewind(); return; } LittleEndien stream = getStream(source); + if (store instanceof short[]) { + populateShortArray((short[]) store, stream, length, byteOffset, byteStride, numComponents, componentSize); + } else if (store instanceof float[]) { - populateFloatArray((float[]) store, stream, length, byteOffset, byteStride, numComponents); + populateFloatArray((float[]) store, stream, length, byteOffset, byteStride, numComponents, componentSize); } else if (store instanceof Vector3f[]) { - populateVector3fArray((Vector3f[]) store, stream, length, byteOffset, byteStride, numComponents); + populateVector3fArray((Vector3f[]) store, stream, length, byteOffset, byteStride, numComponents, componentSize); } else if (store instanceof Quaternion[]) { - populateQuaternionArray((Quaternion[]) store, stream, length, byteOffset, byteStride, numComponents); + populateQuaternionArray((Quaternion[]) store, stream, length, byteOffset, byteStride, numComponents, componentSize); } else if (store instanceof Matrix4f[]) { - populateMatrix4fArray((Matrix4f[]) store, stream, length, byteOffset, byteStride, numComponents); + populateMatrix4fArray((Matrix4f[]) store, stream, length, byteOffset, byteStride, numComponents, componentSize); } } - private static void populateByteBuffer(ByteBuffer buffer, byte[] source, int length, int byteOffset, int byteStride, int numComponents) { + private static void populateByteBuffer(ByteBuffer buffer, byte[] source, int length, int byteOffset, int byteStride, int numComponents, int componentSize) { int index = byteOffset; - int componentSize = 1; while (index < length + byteOffset) { for (int i = 0; i < numComponents; i++) { buffer.put(source[index + i]); @@ -267,9 +281,8 @@ public class GltfUtils { } } - private static void populateShortBuffer(ShortBuffer buffer, LittleEndien stream, int length, int byteOffset, int byteStride, int numComponents) throws IOException { + private static void populateShortBuffer(ShortBuffer buffer, LittleEndien stream, int length, int byteOffset, int byteStride, int numComponents, int componentSize) throws IOException { int index = byteOffset; - int componentSize = 2; int end = length * componentSize + byteOffset; stream.skipBytes(byteOffset); while (index < end) { @@ -280,9 +293,8 @@ public class GltfUtils { } } - private static void populateIntBuffer(IntBuffer buffer, LittleEndien stream, int length, int byteOffset, int byteStride, int numComponents) throws IOException { + private static void populateIntBuffer(IntBuffer buffer, LittleEndien stream, int length, int byteOffset, int byteStride, int numComponents, int componentSize) throws IOException { int index = byteOffset; - int componentSize = 4; int end = length * componentSize + byteOffset; stream.skipBytes(byteOffset); while (index < end) { @@ -293,9 +305,8 @@ public class GltfUtils { } } - private static void populateFloatBuffer(FloatBuffer buffer, LittleEndien stream, int length, int byteOffset, int byteStride, int numComponents) throws IOException { + private static void populateFloatBuffer(FloatBuffer buffer, LittleEndien stream, int length, int byteOffset, int byteStride, int numComponents, int componentSize) throws IOException { int index = byteOffset; - int componentSize = 4; int end = length * componentSize + byteOffset; stream.skipBytes(byteOffset); while (index < end) { @@ -306,9 +317,94 @@ public class GltfUtils { } } - private static void populateFloatArray(float[] array, LittleEndien stream, int length, int byteOffset, int byteStride, int numComponents) throws IOException { + private static void populateShortArray(short[] array, LittleEndien stream, int length, int byteOffset, int byteStride, int numComponents, int componentSize) throws IOException { + int index = byteOffset; + int end = length * componentSize + byteOffset; + stream.skipBytes(byteOffset); + int arrayIndex = 0; + while (index < end) { + for (int i = 0; i < numComponents; i++) { + if (componentSize == 2) { + array[arrayIndex] = stream.readShort(); + } else { + array[arrayIndex] = stream.readByte(); + } + arrayIndex++; + } + + index += Math.max(componentSize * numComponents, byteStride); + } + } + + public static byte[] toByteArray(short[] shortArray) { + byte[] bytes = new byte[shortArray.length]; + for (int i = 0; i < shortArray.length; i++) { + bytes[i] = (byte) shortArray[i]; + } + return bytes; + } + + + public static void handleSkinningBuffers(Mesh mesh, IntMap skinBuffers) { + if (skinBuffers.size() > 0) { + if (skinBuffers.size() == 1) { + GltfLoader.SkinBuffers buffs = skinBuffers.get(0); + setSkinBuffers(mesh, buffs.joints, skinBuffers.get(0).weights, buffs.componentSize); + } else { + + int length = skinBuffers.get(0).joints.length; + short[] jointsArray = new short[length]; + float[] weightsArray = new float[length]; + List weightData = new ArrayList<>(); + int componentSize = 1; + + for (int i = 0; i < weightsArray.length; i += 4) { + weightData.clear(); + for (int j = 0; j < skinBuffers.size(); j++) { + GltfLoader.SkinBuffers buffs = skinBuffers.get(j); + for (int k = 0; k < 4; k++) { + weightData.add(new GltfLoader.WeightData(buffs.weights[i + k], buffs.joints[i + k], buffs.componentSize)); + } + + } + Collections.sort(weightData, new Comparator() { + @Override + public int compare(GltfLoader.WeightData o1, GltfLoader.WeightData o2) { + if (o1.value > o2.value) { + return -1; + } else if (o1.value < o2.value) { + return 1; + } else { + return 0; + } + } + }); + for (int j = 0; j < 4; j++) { + GltfLoader.WeightData data = weightData.get(j); + jointsArray[i + j] = data.index; + weightsArray[i + j] = data.value; + if (data.componentSize > componentSize) { + componentSize = data.componentSize; + } + } + } + setSkinBuffers(mesh, jointsArray, weightsArray, componentSize); + } + } + } + + + public static void setSkinBuffers(Mesh mesh, short[] jointsArray, float[] weightsArray, int componentSize) { + if (componentSize == 1) { + mesh.setBuffer(VertexBuffer.Type.BoneIndex, 4, BufferUtils.createByteBuffer(toByteArray(jointsArray))); + } else { + mesh.setBuffer(VertexBuffer.Type.BoneIndex, 4, BufferUtils.createShortBuffer(jointsArray)); + } + mesh.setBuffer(VertexBuffer.Type.BoneWeight, 4, BufferUtils.createFloatBuffer(weightsArray)); + } + + private static void populateFloatArray(float[] array, LittleEndien stream, int length, int byteOffset, int byteStride, int numComponents, int componentSize) throws IOException { int index = byteOffset; - int componentSize = 4; int end = length * componentSize + byteOffset; stream.skipBytes(byteOffset); int arrayIndex = 0; @@ -322,9 +418,8 @@ public class GltfUtils { } } - private static void populateVector3fArray(Vector3f[] array, LittleEndien stream, int length, int byteOffset, int byteStride, int numComponents) throws IOException { + private static void populateVector3fArray(Vector3f[] array, LittleEndien stream, int length, int byteOffset, int byteStride, int numComponents, int componentSize) throws IOException { int index = byteOffset; - int componentSize = 4; int end = length * componentSize + byteOffset; stream.skipBytes(byteOffset); int arrayIndex = 0; @@ -341,9 +436,8 @@ public class GltfUtils { } } - private static void populateQuaternionArray(Quaternion[] array, LittleEndien stream, int length, int byteOffset, int byteStride, int numComponents) throws IOException { + private static void populateQuaternionArray(Quaternion[] array, LittleEndien stream, int length, int byteOffset, int byteStride, int numComponents, int componentSize) throws IOException { int index = byteOffset; - int componentSize = 4; int end = length * componentSize + byteOffset; stream.skipBytes(byteOffset); int arrayIndex = 0; @@ -361,9 +455,8 @@ public class GltfUtils { } } - private static void populateMatrix4fArray(Matrix4f[] array, LittleEndien stream, int length, int byteOffset, int byteStride, int numComponents) throws IOException { + private static void populateMatrix4fArray(Matrix4f[] array, LittleEndien stream, int length, int byteOffset, int byteStride, int numComponents, int componentSize) throws IOException { int index = byteOffset; - int componentSize = 4; int end = length * componentSize + byteOffset; stream.skipBytes(byteOffset); int arrayIndex = 0;