Skip to content

Commit

Permalink
Added support for faces types in SceneTracker6DOF
Browse files Browse the repository at this point in the history 
Summary: Further, forwarded the ARKit's mesh face types.

Reviewed By: enpe

Differential Revision: D67681351

fbshipit-source-id: 68a8fabc481f1a876eef278b804ee3c0d10f4cf7
  • Loading branch information
@janherling @facebook-github-bot
janherling authored and facebook-github-bot committed Jan 6, 2025
1 parent bb4d72b commit d353a64
Show file tree
Hide file tree
Showing 3 changed files with 127 additions and 37 deletions.
70 changes: 46 additions & 24 deletions impl/ocean/devices/SceneTracker6DOF.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -355,28 +355,35 @@ class OCEAN_DEVICES_EXPORT SceneTracker6DOF : virtual public Tracker6DOF
public:

/**
* Definition of individual mesh types.
* Definition of individual face types.
*/
enum MeshType : uint32_t
enum FaceType : uint8_t
{
/// The mesh type is unknown.
MT_UNKNOWN = 0u,
/// The mesh is representing a ceiling.
MT_CEILING,
/// The mesh is representing a door.
MT_DOOR,
/// The mesh is representing a floor.
MT_FLOOR,
/// The mesh is representing a seat.
MT_SEAT,
/// The mesh is representing a table.
MT_TABLE,
/// The mesh is representing a wall.
MT_WALL,
/// The mesh is representing a window.
MT_WINDOW
/// The face type is unknown.
FT_UNKNOWN = 0u,
/// The face is representing a wall.
FT_WALL,
/// The face is representing a floor.
FT_FLOOR,
/// The face is representing a ceiling.
FT_CEILING,
/// The face is representing a table.
FT_TABLE,
/// The face is representing a seat.
FT_SEAT,
/// The face is representing a window.
FT_WINDOW,
/// The face is representing a door.
FT_DOOR,
/// The exlusive end value.
FT_END
};

/**
* Definition of a vector holding face types.
*/
using FaceTypes = std::vector<FaceType>;

public:

/**
Expand All @@ -386,8 +393,9 @@ class OCEAN_DEVICES_EXPORT SceneTracker6DOF : virtual public Tracker6DOF
* @param vertices The mesh's vertices, defined in the mesh's coordinate system, at least 3
* @param perVertexNormals The per-vertex normals of the mesh's faces, defined in the mesh's coordinate system, one for each vertex
* @param triangleIndices The indices of the vertices representing the mesh's surface triangles, three indices define one triangle, always a multiple of three
* @param faceTypes Optional face types of the mesh, one for each triangle, empty if unknown
*/
inline Mesh(const Index32 meshId, const HomogenousMatrix4& world_T_mesh, Vectors3&& vertices, Vectors3&& perVertexNormals, Indices32&& triangleIndices);
inline Mesh(const Index32 meshId, const HomogenousMatrix4& world_T_mesh, Vectors3&& vertices, Vectors3&& perVertexNormals, Indices32&& triangleIndices, FaceTypes&& faceTypes = FaceTypes());

/**
* Returns the unique id of the mesh.
Expand Down Expand Up @@ -419,14 +427,17 @@ class OCEAN_DEVICES_EXPORT SceneTracker6DOF : virtual public Tracker6DOF
*/
inline const Indices32& triangleIndices() const;

/**
* Returns the face types of the mesh, one for each triangle, empty if unknown.
* @return The mesh's face types, empty if unknown
*/
inline const FaceTypes& faceTypes() const;

protected:

/// The unique id of this mesh.
Index32 meshId_ = Index32(-1);

/// The type of the mesh.
MeshType meshType_ = MT_UNKNOWN;

/// The transformation between mesh and world.
HomogenousMatrix4 world_T_mesh_ = HomogenousMatrix4(false);

Expand All @@ -438,6 +449,9 @@ class OCEAN_DEVICES_EXPORT SceneTracker6DOF : virtual public Tracker6DOF

/// The indices of the vertices representing the mesh's surface triangles, three indices define one triangle, always a multiple of three.
Indices32 triangleIndices_;

/// The face types of the mesh, one for each triangle, empty if unknown.
FaceTypes faceTypes_;
};

/**
Expand Down Expand Up @@ -1128,15 +1142,18 @@ inline const SceneTracker6DOF::SceneElementPlanes::Planes& SceneTracker6DOF::Sce
return planes_;
}

inline SceneTracker6DOF::SceneElementMeshes::Mesh::Mesh(const Index32 meshId, const HomogenousMatrix4& world_T_mesh, Vectors3&& vertices, Vectors3&& perVertexNormals, Indices32&& triangleIndices) :
inline SceneTracker6DOF::SceneElementMeshes::Mesh::Mesh(const Index32 meshId, const HomogenousMatrix4& world_T_mesh, Vectors3&& vertices, Vectors3&& perVertexNormals, Indices32&& triangleIndices, FaceTypes&& faceTypes) :
meshId_(meshId),
world_T_mesh_(world_T_mesh),
vertices_(std::move(vertices)),
perVertexNormals_(std::move(perVertexNormals)),
triangleIndices_(std::move(triangleIndices))
triangleIndices_(std::move(triangleIndices)),
faceTypes_(std::move(faceTypes))
{
ocean_assert(triangleIndices_.size() % 3 == 0);
ocean_assert(vertices_.size() == perVertexNormals_.size());

ocean_assert(faceTypes_.empty() || faceTypes_.size() == triangleIndices_.size() / 3);
}

inline Index32 SceneTracker6DOF::SceneElementMeshes::Mesh::meshId() const
Expand Down Expand Up @@ -1164,6 +1181,11 @@ inline const Indices32& SceneTracker6DOF::SceneElementMeshes::Mesh::triangleIndi
return triangleIndices_;
}

inline const SceneTracker6DOF::SceneElementMeshes::Mesh::FaceTypes& SceneTracker6DOF::SceneElementMeshes::Mesh::faceTypes() const
{
return faceTypes_;
}

inline SceneTracker6DOF::SceneElementMeshes::SceneElementMeshes(SharedMeshes meshes) :
SceneElement(SET_MESHES),
meshes_(std::move(meshes))
Expand Down
13 changes: 11 additions & 2 deletions impl/ocean/devices/arkit/AKSceneTracker6DOF.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -150,7 +150,7 @@ class OCEAN_DEVICES_ARKIT_EXPORT AKSceneTracker6DOF :

/**
* Exracts the 3D vectors from a geometry source.
* @param geometrySource The geometry source from wich all vectors will be extracted
* @param geometrySource The geometry source from wich all vectors will be extracted, must be valid
* @param vectors The resulting 3D vectors
* @return True, if succeeded
*/
Expand All @@ -159,13 +159,22 @@ class OCEAN_DEVICES_ARKIT_EXPORT AKSceneTracker6DOF :

/**
* Extracts the indices from a geometry element.
* @param geometryElement The geometry elemnt from which the indices will be extracted
* @param geometryElement The geometry elemnt from which the indices will be extracted, must be valid
* @param indices The resulting indices
* @return True, if succeeded
*/
API_AVAILABLE(ios(13.4))
static bool extractIndices(ARGeometryElement* geometryElement, Indices32& indices);

/**
* Extracts the face classification from a geometry source.
* @param geometrySource The geometry source from which the face classification will be extracted, must be valid
* @param faceTypes The resulting face types
* @return True, if succeeded
*/
API_AVAILABLE(ios(13.4))
static bool extractFaceClassification(ARGeometrySource* geometrySource, SceneElementMeshes::Mesh::FaceTypes& faceTypes);

protected:

/// The unique id for the world object.
Expand Down
81 changes: 70 additions & 11 deletions impl/ocean/devices/arkit/AKSceneTracker6DOF.mm
View file
Original file line number Diff line number Diff line change
Expand Up @@ -197,8 +197,6 @@
{
SceneElementMeshes::SharedMeshes meshes;

// first, we check whether any mesh as changed, unfortuately ARKit does not provide some kind of mesh version, so that we use the number of triangles to decided whether the mesh has been updated

for (ARAnchor* anchor in arFrame.anchors)
{
if (![anchor isKindOfClass:[ARMeshAnchor class]])
Expand Down Expand Up @@ -252,29 +250,50 @@
continue;
}

SceneElementMeshes::Mesh::FaceTypes faceTypes;

if (meshGeometry.classification != nullptr)
{
const bool result = extractFaceClassification(meshGeometry.classification, faceTypes);
ocean_assert_and_suppress_unused(result, result);
}

ocean_assert(faceTypes.empty() || faceTypes.size() == triangleIndices.size() / 3);

// ARKit sometimes provides invalid per-vertex normals, we remove these triangles

for (size_t n = 0; n < triangleIndices.size(); /*noop*/)
size_t nIndex = 0;

for (size_t nFace = 0; nFace < triangleIndices.size() / 3; /*noop*/)
{
if (Numeric::isNan(perVertexNormals[triangleIndices[n + 0]].x())
|| Numeric::isNan(perVertexNormals[triangleIndices[n + 1]].x())
|| Numeric::isNan(perVertexNormals[triangleIndices[n + 2]].x()))
if (Numeric::isNan(perVertexNormals[triangleIndices[nIndex + 0]].x())
|| Numeric::isNan(perVertexNormals[triangleIndices[nIndex + 1]].x())
|| Numeric::isNan(perVertexNormals[triangleIndices[nIndex + 2]].x()))
{
triangleIndices[n + 0] = triangleIndices[triangleIndices.size() - 3];
triangleIndices[n + 1] = triangleIndices[triangleIndices.size() - 2];
triangleIndices[n + 2] = triangleIndices[triangleIndices.size() - 1];
triangleIndices[nIndex + 0] = triangleIndices[triangleIndices.size() - 3];
triangleIndices[nIndex + 1] = triangleIndices[triangleIndices.size() - 2];
triangleIndices[nIndex + 2] = triangleIndices[triangleIndices.size() - 1];

triangleIndices.resize(triangleIndices.size() - 3);

if (!faceTypes.empty())
{
faceTypes[nIndex] = faceTypes.back();
faceTypes.pop_back();
}
}
else
{
n += 3;
++nFace;
nIndex += 3;
}
}

ocean_assert(faceTypes.empty() || faceTypes.size() == triangleIndices.size() / 3);

if (!triangleIndices.empty())
{
meshes.emplace_back(std::make_shared<SceneElementMeshes::Mesh>(meshId, HomogenousMatrix4(world_T_mesh), std::move(vertices), std::move(perVertexNormals), std::move(triangleIndices)));
meshes.emplace_back(std::make_shared<SceneElementMeshes::Mesh>(meshId, HomogenousMatrix4(world_T_mesh), std::move(vertices), std::move(perVertexNormals), std::move(triangleIndices), std::move(faceTypes)));
}
}
}
Expand Down Expand Up @@ -443,6 +462,46 @@
return false;
}

bool AKSceneTracker6DOF::extractFaceClassification(ARGeometrySource* geometrySource, SceneElementMeshes::Mesh::FaceTypes& faceTypes)
{
static_assert(uint8_t(ARMeshClassificationNone) == SceneElementMeshes::Mesh::FT_UNKNOWN, "Invalid classification value");
static_assert(uint8_t(ARMeshClassificationWall) == SceneElementMeshes::Mesh::FT_WALL, "Invalid classification value");
static_assert(uint8_t(ARMeshClassificationFloor) == SceneElementMeshes::Mesh::FT_FLOOR, "Invalid classification value");
static_assert(uint8_t(ARMeshClassificationCeiling) == SceneElementMeshes::Mesh::FT_CEILING, "Invalid classification value");
static_assert(uint8_t(ARMeshClassificationTable) == SceneElementMeshes::Mesh::FT_TABLE, "Invalid classification value");
static_assert(uint8_t(ARMeshClassificationSeat) == SceneElementMeshes::Mesh::FT_SEAT, "Invalid classification value");
static_assert(uint8_t(ARMeshClassificationWindow) == SceneElementMeshes::Mesh::FT_WINDOW, "Invalid classification value");
static_assert(uint8_t(ARMeshClassificationDoor) == SceneElementMeshes::Mesh::FT_DOOR, "Invalid classification value");

ocean_assert(geometrySource != nullptr);

if (geometrySource == nullptr)
{
return false;
}

const size_t components = geometrySource.componentsPerVector;
ocean_assert(components == 1);

const MTLVertexFormat format = geometrySource.format;
ocean_assert(format == MTLVertexFormatUChar);

if (components != 1 || format != MTLVertexFormatUChar)
{
return false;
}

const size_t offsetBytes = geometrySource.offset;

const uint8_t* data = (const uint8_t*)(geometrySource.buffer.contents);

faceTypes.resize(geometrySource.count);

memcpy(faceTypes.data(), data + offsetBytes, faceTypes.size() * sizeof(uint8_t));

return true;
}

}

}
Expand Down

0 comments on commit d353a64

Please sign in to comment.