import _engine_plugin_guard

import os
import shutil as sh

import numpy as np

# delete test project if the notebook was already run
proj_path = os.path.join(os.path.abspath('..'), 'test_project')
if os.path.isdir(proj_path):
    sh.rmtree(proj_path)

# formatting helpers
def bold(string: str):
    return '\033[1m' + string + '\033[0m'

Tutorial on the Python bindings for ImFusion Labels

# The labels module resides in the imfusion package
import imfusion
from imfusion import labels

Public beta build of ImFusion Python SDK. Not for commercial use.

# A lot of information can already be found in the module's docstring:
labels?

Setting up an ImFusion Labels project

Project is the central entity of the labels module. It holds the definitions of all annotations (Tags, Labelmap Layers, Keypoint Layers and BoundingBox Layers) and provides access to the project’s database in the form of Descriptor instances (we will get to those in the next section). The Project class can either create a new Labels project or load an existing one. All interactions with the annotations and data go through an instance of that class. Let’s create a new project for now:

# Create an empty project named 'TestProject' stored in a folder called 'test_project' in the notebooks parent directory
# Creating the project instance will also create the project folder on disk.
project = labels.Project(name='TestProject', project_path=os.path.join(os.path.abspath('..'), 'test_project'))

Annotation Layers

Labels has the concept of annotation layers for storing any type of annotations (Labelmaps, Landmarks, BoudingBoxes), meaning that each type of annotation can have multiple different data associated with it. An example would be having one labelmap layer per annotator in a multi-annotator dataset. The definitions for these layers can be added through the Project instance analagously to the tag definitions. The instances of these layers also have a very similar interface to the tags.

# Let's add two labelmap layers, a landmark layer and a bounding box layer
project.add_labelmap_layer(name='Segmentation')
project.add_landmark_layer(name='Points of Interest')
project.add_boundingbox_layer(name='Region of Interest')

print(f'Here are the defined labelmap layers:', f'{project.labelmap_layers=}', '', sep='\n')
print(f'Here are the defined landmark layers:', f'{project.landmark_layers=}', '', sep='\n')
print(f'Here are the defined bounding box layers:', f'{project.boundingbox_layers=}', '', sep='\n')

Here are the defined labelmap layers:
project.labelmap_layers=LabelMapsAccessor(
        LabelMapLayer(id=p6mWrFYpzc8m, name='Segmentation')
)

Here are the defined landmark layers:
project.landmark_layers=LandmarkLayersAccessor(
        LandmarkLayer(id=dICO9aWsoPsO, name='Points of Interest')
)

Here are the defined bounding box layers:
project.boundingbox_layers=BoundingBoxLayersAccessor(
        BoundingBoxLayer(id=QZcOWd9ZeKSw, name='Region of Interest')
)

# Each layer can store independent annotation definitions
segmentation_layer = project.labelmap_layers[0]
points_layer = project.landmark_layers[0]
roi_layer = project.boundingbox_layers[0]


#For now the layers are empty
print(bold('Background label is automatically added with a value of 0 when the layer is created.'))
print(f'{segmentation_layer.annotations=}')
print(f'{points_layer.annotations=}')
print(f'{roi_layer.annotations=}')

Background label is automatically added with a value of 0 when the layer is created.
segmentation_layer.annotations=LabelsAccessor(
        Label(name='Background', color=(0, 0, 0), value=0)
)
points_layer.annotations=LandmarksAccessor(

)
roi_layer.annotations=BoundingBoxAccessor(

)

# We can add new definitions to a layer using the add_annotation method
segmentation_layer.add_annotation(name='Circle', value=1, color=(128, 0, 0))
segmentation_layer.add_annotation(name='Core', value=2, color=(128, 128, 0))
points_layer.add_annotation(name='Center', color=(0, 128, 0))
roi_layer.add_annotation(name='Containing Box', color=(0, 0, 128))

print(f'{segmentation_layer.annotations=}')
print(f'{points_layer.annotations=}')
print(f'{roi_layer.annotations=}')

segmentation_layer.annotations=LabelsAccessor(
        Label(name='Background', color=(0, 0, 0), value=0),
        Label(name='Circle', color=(128, 0, 0), value=1),
        Label(name='Core', color=(128, 128, 0), value=2)
)
points_layer.annotations=LandmarksAccessor(
        Landmark(name='Center', color=(0, 128, 0))
)
roi_layer.annotations=BoundingBoxAccessor(
        BoundingBox(name='Containing Box', color=(0, 0, 128))
)

Tags

Tag definitions can be added using the add_tag method on a Project instance. This method will add the new tag to the project and return an instance of the Tag class that represents the newly created tag. This class bundles information about the tag definition with the actual tag value (if it is accessed through a Descriptor as we will see later). Tags, much like most of the other classes in the module that represent annotations in Labels, have a name, a color and a value. Additionally, they store a reference to their owning project and descriptor and know their index.

# New tags can be added
project.add_tag(name='MyBool', kind=labels.TagKind.Bool)
project.add_tag(name='MyNumber', kind=labels.TagKind.Float, color=(10, 20, 30))
enum_tag = project.add_tag(name='MyChoices', kind=labels.TagKind.Enum, options=['A', 'B'])
enum_tag.add_option('C')  # We can also add options to enum tags afterwards
print(f'Here are the defined tags:\n{project.tags=}')

Here are the defined tags:
project.tags=TagsAccessor(
        Tag(name='MyBool', id='1v0gMQm8VH5w', kind=<TagKind.Bool: 0>, color=(255, 255, 255)),
        Tag(name='MyNumber', id='Yk4jntz9YmwX', kind=<TagKind.Float: 2>, color=(10, 20, 30)),
        Tag(name='MyChoices', id='byOPycbPqM9o', kind=<TagKind.Enum: 1>, color=(255, 255, 255), options=['default', 'A', 'B', 'C'])
)

# You can retrieve tag definitions from the project using their name or index:
print(f'Using name:\t\t{project.tags["MyNumber"]=}\n'
      f'Using index:\t{project.tags[1]=}')

Using name:             project.tags["MyNumber"]=Tag(name='MyNumber', id='Yk4jntz9YmwX', kind=<TagKind.Float: 2>, color=(10, 20, 30))
Using index:    project.tags[1]=Tag(name='MyNumber', id='Yk4jntz9YmwX', kind=<TagKind.Float: 2>, color=(10, 20, 30))

# Using a Tag instance you can access the information in the tag definition and the parents of the tag
print(f'{enum_tag.name=}',
      f'{enum_tag.color=}',
      f'{enum_tag.options=}',
      '',
      bold('These are the parents of this tag:'),
      f'{enum_tag.project=}',
      f'{enum_tag.descriptor=}',
      '',
      bold('Index in the project`s tag definitions:'),
      f'{enum_tag.index=}',
      '',
      bold('The tag has no value, since it is accessed through the project and not a descriptor:'),
      f'{enum_tag.value=}',
      sep='\n')

enum_tag.name='MyChoices'
enum_tag.color=(255, 255, 255)
enum_tag.options=['default', 'A', 'B', 'C']

These are the parents of this tag:
enum_tag.project=<imfusion.labels._bindings.Project object at 0x7f172216e270>
enum_tag.descriptor=None

Index in the project`s tag definitions:
enum_tag.index=2

The tag has no value, since it is accessed through the project and not a descriptor:
enum_tag.value=None

Adding data

Data in Labels is handled as instances of the Descriptor class. It stores the meta information about a data samples, lets you load the underlying image and access the annotations on that particular sample. New data can be added to the project using the add_descriptor method on the Project class. It expects a SharedImageSet as the basis for creating a new descriptor. Optionally, the descriptor can be named using the name parameter. If name is not supplied it will use the name of the SharedImageSet. Additionally, we can tell Labels whether it should store a copy of the data in the project folder or not using the own_copy parameter. If the SharedImageSet was created on the fly (it will not have a DataSourceComponent) then this flag is ignored and Labels always saves a copy to the project folder.

# Let's create a circle and use that as a data sample
side_length = 128
radius = side_length // 4
circle = np.zeros((side_length,side_length))
Y, X = np.ogrid[:side_length, :side_length]
dist_from_center = np.sqrt((X - side_length//2)**2 + (Y-side_length//2)**2)
circle[dist_from_center <= radius] = 1

# Now we create a SharedImageSet with the circle and add it to the project
circle_sis = imfusion.SharedImageSet(circle[None, ..., None])  # We have to create a batch and a channel dimension
descriptor = project.add_descriptor(circle_sis, name='Circle')

# We can also load data from disk and add it to our project
# To demonstrate this, we will first save our circle to disk and then load it back in:
save_path = os.path.join(project.path, 'circle.imf')
imfusion.save([circle_sis], save_path)
circle_sis2, *_ = imfusion.load(save_path)
circle_sis2.name = 'Circle-from-disk'  # If we give the SIS a name we won't have to specify it when adding it as a descriptor
descriptor2 = project.add_descriptor(circle_sis2)

# Note that the two descriptors have different identifiers, since they are treated as separate entities
# Also note that for the descriptor which we added from a numpy array `own_copy` was automatically set to True, meaning that labels saved a copy of the data in the project folder
print(descriptor)
print(descriptor2)

[GUI.Animations] No global backend set: installing a dummy one.
Descriptor(
        boundingbox_layers=BoundingBoxLayersAccessor(
                BoundingBoxLayer(id=QZcOWd9ZeKSw, name='Region of Interest', value=None)
        )
        byte_size=65536
        comments=''
        grouping=[]
        has_data=False
        height=128
        identifier='oWsRLnHFRuXn'
        import_time=1726731256011
        is_locked=False
        labelmap_layers=LabelMapsAccessor(
                LabelMapLayer(id=p6mWrFYpzc8m, name='Segmentation', value=None)
        )
        landmark_layers=LandmarkLayersAccessor(
                LandmarkLayer(id=dICO9aWsoPsO, name='Points of Interest', value=None)
        )
        latest_edit_time=1726731256011
        load_path=('/data/imfusion/github/public-python-demos/test_project/data/oWsRLnHFRuXn.imf', '')
        modality=<Modality.NA: 0>
        n_channels=1
        n_images=1
        n_slices=1
        name='Circle'
        original_data_path=''
        own_copy=True
        patient_name=''
        project=<imfusion.labels._bindings.Project object at 0x7f172216e270>
        region_of_interest=(array([0, 0, 0], dtype=int32), array([0, 0, 0], dtype=int32))
        roi=(array([0, 0, 0], dtype=int32), array([0, 0, 0], dtype=int32))
        scale=1.0
        series_instance_uid=''
        shift=0.0
        spacing=array([1., 1., 1.])
        sub_file_id=''
        tags=TagsAccessor(
                Tag(name='MyBool', id='1v0gMQm8VH5w', kind=<TagKind.Bool: 0>, color=(255, 255, 255), value=False),
                Tag(name='MyNumber', id='Yk4jntz9YmwX', kind=<TagKind.Float: 2>, color=(10, 20, 30), value=0.0),
                Tag(name='MyChoices', id='byOPycbPqM9o', kind=<TagKind.Enum: 1>, color=(255, 255, 255), options=['default', 'A', 'B', 'C'], value='default')
        )
        thumbnail_path='/data/imfusion/github/public-python-demos/test_project/thumbnails/data/oWsRLnHFRuXn.png'
        top_down=True
        type=<PixelType.FLOAT: 5126>
        width=128)
Descriptor(
        boundingbox_layers=BoundingBoxLayersAccessor(
                BoundingBoxLayer(id=QZcOWd9ZeKSw, name='Region of Interest', value=None)
        )
        byte_size=65536
        comments=''
        grouping=[]
        has_data=False
        height=128
        identifier='ayEssZSFPlal'
        import_time=1726731256036
        is_locked=False
        labelmap_layers=LabelMapsAccessor(
                LabelMapLayer(id=p6mWrFYpzc8m, name='Segmentation', value=None)
        )
        landmark_layers=LandmarkLayersAccessor(
                LandmarkLayer(id=dICO9aWsoPsO, name='Points of Interest', value=None)
        )
        latest_edit_time=1726731256036
        load_path=('circle.imf', '')
        modality=<Modality.NA: 0>
        n_channels=1
        n_images=1
        n_slices=1
        name='Circle-from-disk'
        original_data_path='circle.imf'
        own_copy=False
        patient_name=''
        project=<imfusion.labels._bindings.Project object at 0x7f172216e270>
        region_of_interest=(array([0, 0, 0], dtype=int32), array([0, 0, 0], dtype=int32))
        roi=(array([0, 0, 0], dtype=int32), array([0, 0, 0], dtype=int32))
        scale=1.0
        series_instance_uid=''
        shift=0.0
        spacing=array([1., 1., 1.])
        sub_file_id=''
        tags=TagsAccessor(
                Tag(name='MyBool', id='1v0gMQm8VH5w', kind=<TagKind.Bool: 0>, color=(255, 255, 255), value=False),
                Tag(name='MyNumber', id='Yk4jntz9YmwX', kind=<TagKind.Float: 2>, color=(10, 20, 30), value=0.0),
                Tag(name='MyChoices', id='byOPycbPqM9o', kind=<TagKind.Enum: 1>, color=(255, 255, 255), options=['default', 'A', 'B', 'C'], value='default')
        )
        thumbnail_path='/data/imfusion/github/public-python-demos/test_project/thumbnails/data/ayEssZSFPlal.png'
        top_down=True
        type=<PixelType.FLOAT: 5126>
        width=128)

# We can now find the descriptors in the project using the `descriptor` attribute
print(project.descriptors)

[Descriptor(
        boundingbox_layers=BoundingBoxLayersAccessor(
                BoundingBoxLayer(id=QZcOWd9ZeKSw, name='Region of Interest', value=None)
        )
        byte_size=65536
        comments=''
        grouping=[]
        has_data=False
        height=128
        identifier='oWsRLnHFRuXn'
        import_time=1726731256011
        is_locked=False
        labelmap_layers=LabelMapsAccessor(
                LabelMapLayer(id=p6mWrFYpzc8m, name='Segmentation', value=None)
        )
        landmark_layers=LandmarkLayersAccessor(
                LandmarkLayer(id=dICO9aWsoPsO, name='Points of Interest', value=None)
        )
        latest_edit_time=1726731256011
        load_path=('/data/imfusion/github/public-python-demos/test_project/data/oWsRLnHFRuXn.imf', '')
        modality=<Modality.NA: 0>
        n_channels=1
        n_images=1
        n_slices=1
        name='Circle'
        original_data_path=''
        own_copy=True
        patient_name=''
        project=<imfusion.labels._bindings.Project object at 0x7f172216e270>
        region_of_interest=(array([0, 0, 0], dtype=int32), array([0, 0, 0], dtype=int32))
        roi=(array([0, 0, 0], dtype=int32), array([0, 0, 0], dtype=int32))
        scale=1.0
        series_instance_uid=''
        shift=0.0
        spacing=array([1., 1., 1.])
        sub_file_id=''
        tags=TagsAccessor(
                Tag(name='MyBool', id='1v0gMQm8VH5w', kind=<TagKind.Bool: 0>, color=(255, 255, 255), value=False),
                Tag(name='MyNumber', id='Yk4jntz9YmwX', kind=<TagKind.Float: 2>, color=(10, 20, 30), value=0.0),
                Tag(name='MyChoices', id='byOPycbPqM9o', kind=<TagKind.Enum: 1>, color=(255, 255, 255), options=['default', 'A', 'B', 'C'], value='default')
        )
        thumbnail_path='/data/imfusion/github/public-python-demos/test_project/thumbnails/data/oWsRLnHFRuXn.png'
        top_down=True
        type=<PixelType.FLOAT: 5126>
        width=128), Descriptor(
        boundingbox_layers=BoundingBoxLayersAccessor(
                BoundingBoxLayer(id=QZcOWd9ZeKSw, name='Region of Interest', value=None)
        )
        byte_size=65536
        comments=''
        grouping=[]
        has_data=False
        height=128
        identifier='ayEssZSFPlal'
        import_time=1726731256036
        is_locked=False
        labelmap_layers=LabelMapsAccessor(
                LabelMapLayer(id=p6mWrFYpzc8m, name='Segmentation', value=None)
        )
        landmark_layers=LandmarkLayersAccessor(
                LandmarkLayer(id=dICO9aWsoPsO, name='Points of Interest', value=None)
        )
        latest_edit_time=1726731256036
        load_path=('circle.imf', '')
        modality=<Modality.NA: 0>
        n_channels=1
        n_images=1
        n_slices=1
        name='Circle-from-disk'
        original_data_path='circle.imf'
        own_copy=False
        patient_name=''
        project=<imfusion.labels._bindings.Project object at 0x7f172216e270>
        region_of_interest=(array([0, 0, 0], dtype=int32), array([0, 0, 0], dtype=int32))
        roi=(array([0, 0, 0], dtype=int32), array([0, 0, 0], dtype=int32))
        scale=1.0
        series_instance_uid=''
        shift=0.0
        spacing=array([1., 1., 1.])
        sub_file_id=''
        tags=TagsAccessor(
                Tag(name='MyBool', id='1v0gMQm8VH5w', kind=<TagKind.Bool: 0>, color=(255, 255, 255), value=False),
                Tag(name='MyNumber', id='Yk4jntz9YmwX', kind=<TagKind.Float: 2>, color=(10, 20, 30), value=0.0),
                Tag(name='MyChoices', id='byOPycbPqM9o', kind=<TagKind.Enum: 1>, color=(255, 255, 255), options=['default', 'A', 'B', 'C'], value='default')
        )
        thumbnail_path='/data/imfusion/github/public-python-demos/test_project/thumbnails/data/ayEssZSFPlal.png'
        top_down=True
        type=<PixelType.FLOAT: 5126>
        width=128)]

# We can also retrieve the image from the descriptor
circle_from_descriptor = descriptor.load_image(crop_to_roi=False).numpy()  # We don't have a roi on this descriptor but this argument can be used to either get the full or the cropped image
assert np.allclose(circle, circle_from_descriptor.squeeze())

Adding annotations to data

While the project holds the definitions of tags and annotations, their values can only be changed when accessing them through a Descriptor instance.

Tags

# Given our descriptor, we can set the values of its tags
descriptor.tags['MyBool'] = True
descriptor.tags['MyNumber'] = 9001
descriptor.tags['MyChoices'] = 'B'

# Since the tags now have non-default values they implicitly evaluate to True
for tag in descriptor.tags:
    print(tag)
    assert tag

# We can reset the tags by setting them to the default values of their respective tag type
descriptor.tags['MyBool'] = False
descriptor.tags['MyNumber'] = 0
descriptor.tags['MyChoices'] = 'default'

# Now the tags will evaluate to False again
for tag in descriptor.tags:
    print(tag)
    assert not tag

Tag(name='MyBool', id='1v0gMQm8VH5w', kind=<TagKind.Bool: 0>, color=(255, 255, 255), value=True)
Tag(name='MyNumber', id='Yk4jntz9YmwX', kind=<TagKind.Float: 2>, color=(10, 20, 30), value=9001.0)
Tag(name='MyChoices', id='byOPycbPqM9o', kind=<TagKind.Enum: 1>, color=(255, 255, 255), options=['default', 'A', 'B', 'C'], value='B')
Tag(name='MyBool', id='1v0gMQm8VH5w', kind=<TagKind.Bool: 0>, color=(255, 255, 255), value=False)
Tag(name='MyNumber', id='Yk4jntz9YmwX', kind=<TagKind.Float: 2>, color=(10, 20, 30), value=0.0)
Tag(name='MyChoices', id='byOPycbPqM9o', kind=<TagKind.Enum: 1>, color=(255, 255, 255), options=['default', 'A', 'B', 'C'], value='default')

Annotations

# Now we can add some annotation to our circle

# First let's add a segmentation map of the pixels that belong to the circle
# Note that this labelmap will be saved to disk once we call `save_new_data` (that's just how Labels currently works)
labelmap = imfusion.SharedImageSet(circle.astype(np.uint8)[None, ..., None])
descriptor.labelmap_layers['Segmentation'].save_new_data(labelmap)

# And we can verify that it is indeed stored for this descriptor
print(descriptor.labelmap_layers['Segmentation'].load())

imfusion.SharedImageSet(size: 1, [imfusion.SharedImage(UBYTE width: 128 height: 128)])

# We can also add a landmark at the center of the circle
point_set = labels.LandmarkSet.from_descriptor(descriptor=descriptor, layer_name='Points of Interest')
point_set.add(type='Center', frame=0, world=(0, 0, 0))

descriptor.landmark_layers['Points of Interest'].save_new_data(point_set)

print(descriptor.landmark_layers['Points of Interest'].load().asdict())

{'Center': {0: [(0.0, 0.0, 0.0)]}}

# Finally let's add a box around the circle
# It works analogous to adding a landmark
box_set = labels.BoxSet.from_descriptor(descriptor=descriptor, layer_name='Region of Interest')
box_set.add(type='Containing Box', frame=0, top_left=(-32, -32, 1), lower_right=(32, 32, 1))

descriptor.boundingbox_layers['Region of Interest'].save_new_data(box_set)

print(descriptor.boundingbox_layers['Region of Interest'].load().asdict())

{'Containing Box': {0: [((-32.0, -32.0, 1.0), (32.0, 32.0, 1.0))]}}

Saving the project

Almost all actions we perform on the Project instance are held in memory. The only exception is when we add data to the annotation layers of a descriptor as it is written to disk immediately. To save the modifications on the project we have to call its save method

# Here we save all the changes we made to the project to disk
project.save()

# You can now open the project through the Labels GUI and verify that everything is as you expect it to be

Loading the project

# Let's load the project back in and verify that everything is still there
loaded_project = labels.Project.load(project.path)

for loaded, expected in zip(loaded_project.descriptors, project.descriptors):
    assert loaded.name == expected.name
    assert loaded.identifier == expected.identifier

for loaded, expected in zip(loaded_project.tags, project.tags):
    assert loaded.name == expected.name
    assert loaded.kind == expected.kind

# We can modify the loaded project and then save it back.
# For example let's add a new label and modify the labelmap of one of the circle descriptors

# Instead of calling `Project.save()` at the end, you can also use a context manager to automatically save the project when exiting the context
with loaded_project:

    # We can also modify the annotation definition from the descriptor
    descriptor = loaded_project.descriptors[0]
    labelmap_layer = descriptor.labelmap_layers['Segmentation']
    labelmap_layer.add_annotation(name='Core', value=2)

    # Let's load the label map and modify it
    labelmap = labelmap_layer.load().numpy().squeeze()
    labelmap[dist_from_center < radius//2] = 2

    # Now we can write the labelmap back to disk
    labelmap_layer.save_new_data(imfusion.SharedImageSet(labelmap[None, ..., None]))