Import RLE segmentation masks as outlines

client/platform/desktop/backend/native/common.spec.ts

@@ -1209,8 +1209,8 @@ describe('native.common', () => {
 
     expect(result.processedFiles).toEqual([first, empty, third]);
     expect(result.warnings).toEqual([
-      'The COCO file included run-length encoded segmentation masks that are not supported. '
-        + 'Bounding boxes and other annotation data were imported, but masks were skipped.',
+      'The COCO file included run-length encoded segmentation masks that could not be decoded. '
+        + 'Bounding boxes and other annotation data were imported, but those masks were skipped.',
       'Ignored dataset_info entry: expected a JSON object but got number',
     ]);
   });

client/platform/desktop/backend/serializers/coco.spec.ts

@@ -18,6 +18,52 @@ import {
   speciesListFromCategories,
   typeHierarchyFromCategories,
 } from 'platform/desktop/backend/serializers/coco';
+import { decodeRleCounts } from 'platform/desktop/backend/serializers/cocoRle';
+
+/** pycocotools rleToString, so the decoder is tested against real output. */
+function rleToString(cnts: number[]): string {
+  const out: string[] = [];
+  cnts.forEach((count, i) => {
+    let x = count;
+    if (i > 2) {
+      x -= cnts[i - 2];
+    }
+    let more = true;
+    while (more) {
+      /* eslint-disable no-bitwise -- LEB128 digit packing from pycocotools */
+      let chunk = x & 0x1f;
+      x >>= 5;
+      more = (chunk & 0x10) ? (x !== -1) : (x !== 0);
+      if (more) {
+        chunk |= 0x20;
+      }
+      /* eslint-enable no-bitwise */
+      out.push(String.fromCharCode(chunk + 48));
+    }
+  });
+  return out.join('');
+}
+
+/** Column-major run lengths for a 6x6 square at (3, 2) in a 10x10 mask. */
+function squareMaskRuns(): number[] {
+  const runs: number[] = [];
+  let current = 0;
+  let length = 0;
+  for (let column = 0; column < 10; column += 1) {
+    for (let row = 0; row < 10; row += 1) {
+      const value = (row >= 2 && row < 8 && column >= 3 && column < 9) ? 1 : 0;
+      if (value === current) {
+        length += 1;
+      } else {
+        runs.push(length);
+        current = value;
+        length = 1;
+      }
+    }
+  }
+  runs.push(length);
+  return runs;
+}
 
 const kwcocoProfile = fs.readJSONSync('../testutils/kwcoco/import-profile.json');
 
@@ -172,7 +218,7 @@ describe('COCO serializer', () => {
       },
     });
     await expect(parseFile('/input/coco_no_bbox.json')).rejects.toThrow(/no bbox and no usable polygon/);
-    await expect(parseFile('/input/coco_no_bbox.json')).rejects.toThrow(/RLE segmentation masks still require a bbox/);
+    await expect(parseFile('/input/coco_no_bbox.json')).rejects.toThrow(/An RLE mask supplies bounds only when it can be decoded/);
   });
 
   it('derives bbox from polygon when bbox is omitted', async () => {
@@ -197,7 +243,7 @@ describe('COCO serializer', () => {
     expect(warnings).toEqual([]);
   });
 
-  it('imports polygon segmentations and warns on RLE in the same file', async () => {
+  it('imports polygon segmentations and warns on undecodable RLE in the same file', async () => {
     mockfs({
       '/input': {
         'coco_mixed.json': JSON.stringify({
@@ -229,9 +275,10 @@ describe('COCO serializer', () => {
     expect(parsed.tracks[301].features[0].geometry?.features.length).toBe(1);
     expect(parsed.tracks[302].features[0].geometry).toBeUndefined();
     expect(warnings).toHaveLength(1);
+    expect(warnings[0]).toContain('could not be decoded');
   });
 
-  it('imports bbox when RLE masks are present and returns a warning', async () => {
+  it('imports bbox when RLE masks cannot be decoded and returns a warning', async () => {
     mockfs({
       '/input': {
         'coco_rle.json': JSON.stringify({
@@ -253,7 +300,48 @@ describe('COCO serializer', () => {
     expect(parsed.tracks[8].features[0].bounds).toEqual([10, 20, 40, 60]);
     expect(parsed.tracks[8].features[0].geometry).toBeUndefined();
     expect(warnings).toHaveLength(1);
-    expect(warnings[0]).toContain('segmentation masks');
+    expect(warnings[0]).toContain('could not be decoded');
+  });
+
+  it.each([
+    ['uncompressed list', false],
+    ['pycocotools string', true],
+  ])('imports RLE masks as outlines (%s)', async (_label, asString) => {
+    const runs = squareMaskRuns();
+    const counts = asString ? rleToString(runs) : runs;
+    mockfs({
+      '/input': {
+        'coco_rle_ok.json': JSON.stringify({
+          images: [{ id: 1, file_name: 'frame_000000.png', frame_index: 0 }],
+          annotations: [{
+            id: 1,
+            image_id: 1,
+            category_id: 1,
+            track_id: 1,
+            iscrowd: 1,
+            segmentation: { counts, size: [10, 10] },
+          }],
+          categories: [{ id: 1, name: 'fish' }],
+        }),
+      },
+    });
+    const [parsed, , warnings] = await parseFile('/input/coco_rle_ok.json');
+    const feature = parsed.tracks[1].features[0];
+    const polygon = feature.geometry?.features.find((g) => g.geometry.type === 'Polygon');
+    expect(polygon).toBeTruthy();
+    const coords = (polygon?.geometry as GeoJSON.Polygon).coordinates[0];
+    const xs = coords.map(([x]) => x);
+    const ys = coords.map(([, y]) => y);
+    expect([Math.min(...xs), Math.max(...xs), Math.min(...ys), Math.max(...ys)]).toEqual([3, 8, 2, 7]);
+    expect(feature.bounds).toEqual([3, 2, 8, 7]);
+    expect(warnings).toEqual([]);
+  });
+
+  it('rejects junk RLE counts', () => {
+    expect(decodeRleCounts([1, -2])).toBeNull();
+    expect(decodeRleCounts([1, 'x'])).toBeNull();
+    expect(decodeRleCounts(null)).toBeNull();
+    expect(decodeRleCounts(rleToString([4, 2, 4]))).toEqual([4, 2, 4]);
   });
 
   it('serializes COCO with DIVE extension attributes', async () => {

client/platform/desktop/backend/serializers/coco.ts

@@ -6,6 +6,7 @@ import { JsonConfig } from 'platform/desktop/constants';
 import processTrackAttributes from 'platform/desktop/backend/native/attributeProcessor';
 import { strNumericCompare } from 'platform/desktop/sharedUtils';
 import { TrackSupportedFeature } from 'vue-media-annotator/track';
+import { rlePolygonCoords } from 'platform/desktop/backend/serializers/cocoRle';
 
 type CocoImage = {
   id: number;
@@ -24,8 +25,8 @@ type CocoCategory = {
 };
 
 const RLE_SEGMENTATION_WARNING = (
-  'The COCO file included run-length encoded segmentation masks that are not supported. '
-  + 'Bounding boxes and other annotation data were imported, but masks were skipped.'
+  'The COCO file included run-length encoded segmentation masks that could not be decoded. '
+  + 'Bounding boxes and other annotation data were imported, but those masks were skipped.'
 );
 
 const PROB_TOP_K = 25;
@@ -165,14 +166,29 @@ function bboxFromPoints(points: [number, number][]): [number, number, number, nu
   return [xMin, yMin, Math.max(...xs) - xMin, Math.max(...ys) - yMin];
 }
 
-function annotationHasImportableBounds(annotation: CocoAnnotation): boolean {
-  if (hasValidBbox(annotation)) {
-    return true;
+/** Decode segmentation once; reuse for bounds check, bbox, and geometry. */
+function segmentationCoordLists(
+  annotation: CocoAnnotation,
+): { coordLists: [number, number][][]; rleSkipped: boolean } {
+  if (!annotation.segmentation) {
+    return { coordLists: [], rleSkipped: false };
   }
   if (hasRleSegmentation(annotation)) {
-    return false;
+    const coordLists = rlePolygonCoords(annotation.segmentation);
+    return { coordLists, rleSkipped: !coordLists.length };
   }
-  return extractPolygonCoordsLists(annotation.segmentation).length > 0;
+  return { coordLists: extractPolygonCoordsLists(annotation.segmentation), rleSkipped: false };
+}
+
+function annotationHasImportableBounds(
+  annotation: CocoAnnotation,
+  coordLists?: [number, number][][],
+): boolean {
+  if (hasValidBbox(annotation)) {
+    return true;
+  }
+  const lists = coordLists ?? segmentationCoordLists(annotation).coordLists;
+  return lists.length > 0;
 }
 
 function missingBoundsError(annotationIds: Array<number | string>): string {
@@ -182,24 +198,34 @@ function missingBoundsError(annotationIds: Array<number | string>): string {
     `${annotationIds.length} COCO annotation(s) cannot be imported because they have no bbox and `
     + `no usable polygon segmentation (ids: ${shown}${extra}). `
     + 'Provide bbox [x, y, width, height] or polygon segmentation as [[x1, y1, ...]]. '
-    + 'Annotations with only RLE segmentation masks still require a bbox.'
+    + 'An RLE mask supplies bounds only when it can be decoded.'
   );
 }
 
-function resolveCocoBbox(annotation: CocoAnnotation): [number, number, number, number] {
+function resolveCocoBbox(
+  annotation: CocoAnnotation,
+  coordLists?: [number, number][][],
+): [number, number, number, number] {
   if (hasValidBbox(annotation)) {
     return annotation.bbox as [number, number, number, number];
   }
-  const allPoints = extractPolygonCoordsLists(annotation.segmentation).flat();
+  const lists = coordLists ?? segmentationCoordLists(annotation).coordLists;
+  const allPoints = lists.flat();
   if (allPoints.length) {
     return bboxFromPoints(allPoints);
   }
   throw new Error(missingBoundsError([annotation.id]));
 }
 
-function validateAnnotationBounds(annotations: CocoAnnotation[]): void {
+function validateAnnotationBounds(
+  annotations: CocoAnnotation[],
+  coordListsByAnnotation?: WeakMap<CocoAnnotation, [number, number][][]>,
+): void {
   const missingIds = annotations
-    .filter((annotation) => !annotationHasImportableBounds(annotation))
+    .filter((annotation) => !annotationHasImportableBounds(
+      annotation,
+      coordListsByAnnotation?.get(annotation),
+    ))
     .map((annotation) => annotation.id);
   if (missingIds.length) {
     throw new Error(missingBoundsError(missingIds));
@@ -219,8 +245,8 @@ type CocoAnnotation = {
    * COCO `iscrowd` flag (0 or 1). In the COCO spec, 0 means a single instance with
    * polygon `segmentation` ([[x1, y1, ...]]); 1 means a crowd region whose
    * `segmentation` is run-length encoded (RLE) as an object (e.g. { counts, size }).
-   * DIVE does not import RLE masks: when `iscrowd` is truthy, or `segmentation` is
-   * a dict, polygon/mask geometry is skipped (bbox and other fields still import).
+   * Decodable RLE is traced to an outline polygon; undecodable masks keep bbox
+   * (when present) and skip geometry.
    */
   iscrowd?: number;
   keypoints?: number[] | { xy: number[]; keypoint_category?: string; keypoint_category_id?: number; visible?: number }[];
@@ -270,7 +296,7 @@ function frameRateFromDocument(document: CocoDocument): number | undefined {
   return undefined;
 }
 
-/** True when segmentation is COCO RLE (crowd / `iscrowd: 1`), which DIVE does not decode. */
+/** True when segmentation is COCO RLE / crowd (`iscrowd: 1` or a counts dict). */
 function hasRleSegmentation(annotation: CocoAnnotation): boolean {
   if (annotation.iscrowd) {
     return true;
@@ -282,12 +308,17 @@ function hasRleSegmentation(annotation: CocoAnnotation): boolean {
 function buildFeatureGeometry(
   annotation: CocoAnnotation,
   category?: CocoCategory,
+  precomputed?: { coordLists: [number, number][][]; rleSkipped: boolean },
   keypointCategories: { id: number; name: string }[] = [],
 ): { geometry?: GeoJSON.FeatureCollection<TrackSupportedFeature, GeoJSON.GeoJsonProperties>; rleSkipped: boolean } {
-  const rleSkipped = hasRleSegmentation(annotation);
   const geometryFeatures:
     GeoJSON.Feature<TrackSupportedFeature, GeoJSON.GeoJsonProperties>[] = [];
-  const coordLists = rleSkipped ? [] : extractPolygonCoordsLists(annotation.segmentation);
+  const resolved = precomputed ?? segmentationCoordLists(annotation);
+  const { rleSkipped } = resolved;
+  // Largest outline only for RLE (server parity); holes / extra components are dropped.
+  const coordLists = hasRleSegmentation(annotation) && resolved.coordLists.length
+    ? [resolved.coordLists[0]]
+    : resolved.coordLists;
   coordLists.forEach((coords) => {
     geometryFeatures.push({
       type: 'Feature',
@@ -457,12 +488,25 @@ async function parseFile(path: string): Promise<[AnnotationSchema, Record<string
   let probIgnoredForDuplicates = false;
   let diveConfidencePairsInvalid = false;
 
-  validateAnnotationBounds(parsed.annotations);
+  // Decode each segmentation once; reuse for bounds check, bbox, and geometry.
+  const segByAnnotation = new WeakMap<CocoAnnotation, {
+    coordLists: [number, number][][];
+    rleSkipped: boolean;
+  }>();
+  const coordListsByAnnotation = new WeakMap<CocoAnnotation, [number, number][][]>();
+  parsed.annotations.forEach((annotation) => {
+    const resolved = segmentationCoordLists(annotation);
+    segByAnnotation.set(annotation, resolved);
+    coordListsByAnnotation.set(annotation, resolved.coordLists);
+  });
+  validateAnnotationBounds(parsed.annotations, coordListsByAnnotation);
 
   parsed.annotations.forEach((annotation) => {
     const frame = frameByImageId[annotation.image_id];
     if (frame === undefined) return;
-    const [x, y, w, h] = resolveCocoBbox(annotation);
+    const precomputed = segByAnnotation.get(annotation)
+      ?? segmentationCoordLists(annotation);
+    const [x, y, w, h] = resolveCocoBbox(annotation, precomputed.coordLists);
     const bounds: [number, number, number, number] = [x, y, x + w, y + h];
     const trackId = annotation.track_id ?? annotation.id;
     const category = categoriesById[annotation.category_id];
@@ -528,7 +572,12 @@ async function parseFile(path: string): Promise<[AnnotationSchema, Record<string
     } else if (typeof noteField === 'string' && noteField.trim()) {
       feature.notes = [noteField.trim()];
     }
-    const { geometry, rleSkipped } = buildFeatureGeometry(annotation, category, parsed.keypoint_categories);
+    const { geometry, rleSkipped } = buildFeatureGeometry(
+      annotation,
+      category,
+      precomputed,
+      parsed.keypoint_categories,
+    );
     if (rleSkipped) {
       skippedRleMasks = true;
     }
@@ -712,6 +761,7 @@ export {
   PROB_DUPLICATE_CATEGORY_WARNING,
   PROB_LENGTH_MISMATCH_WARNING,
   PROB_TOP_K,
+  RLE_SEGMENTATION_WARNING,
   SUPERCATEGORY_DUPLICATE_CATEGORY_WARNING,
   SUPERCATEGORY_MULTI_PARENT_WARNING,
   invalidCocoHierarchyMessage,