Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: 🚀 initial keypoint support for transformers added #1553

Open
wants to merge 4 commits into
base: develop
Choose a base branch
from
Open
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
54 changes: 54 additions & 0 deletions supervision/keypoint/core.py
Original file line number Diff line number Diff line change
Expand Up @@ -509,6 +509,60 @@ def from_detectron2(cls, detectron2_results: Any) -> KeyPoints:
else:
return cls.empty()

@classmethod
def from_transformers(cls, transformers_results: List) -> KeyPoints:
"""
Create a `sv.KeyPoints` object from the
[Transformers](https://huggingface.co/transformers/) inference result.

Args:
transformers_results (Any): The output of a
Hugging Face Transformers model containing instances with prediction data.

Returns:
A `sv.KeyPoints` object containing the keypoint coordinates, class IDs,
and class names, and confidences of each keypoint.

Example:
```python
import cv2
import torch
from PIL import Image
import supervision as sv
from transformers import AutoImageProcessor, SuperPointForKeypointDetection

processor = AutoImageProcessor.from_pretrained("magic-leap-community/superpoint")
model = SuperPointForKeypointDetection.from_pretrained("magic-leap-community/superpoint")

image = cv2.imread(<SOURCE_IMAGE_PATH>)
image_pil = Image.fromarray(image)
inputs = processor(images,return_tensors="pt").to(model.device, model.dtype)
outputs = model(**inputs)
keypoints = sv.KeyPoints.from_transformers(outputs)
```
""" # noqa: E501 // docs

keypoints_list = []
scores_list = []

for result in transformers_results:
if "keypoints" in result:
keypoints = result["keypoints"].detach().numpy()
scores = result["scores"].detach().numpy()

if keypoints.size > 0:
keypoints_list.append(keypoints)
scores_list.append(scores)

if not keypoints_list:
return cls.empty()

return cls(
xy=np.array(keypoints_list),
onuralpszr marked this conversation as resolved.
Show resolved Hide resolved
confidence=np.array(scores_list),
onuralpszr marked this conversation as resolved.
Show resolved Hide resolved
class_id=None,
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Do we need to explicitly set it to None?

)

def __getitem__(
self, index: Union[int, slice, List[int], np.ndarray, str]
) -> Union[KeyPoints, List, np.ndarray, None]:
Expand Down