diff --git a/client/dive-common/components/TrackSettingsPanel.vue b/client/dive-common/components/TrackSettingsPanel.vue index 03111b4df..45467923b 100644 --- a/client/dive-common/components/TrackSettingsPanel.vue +++ b/client/dive-common/components/TrackSettingsPanel.vue @@ -58,7 +58,7 @@ export default defineComponent({ help, modes, typeList, - stereoMatchMethods: stereoMatchMethodsFor(isDesktopRuntime()), + stereoMatchMethods: computed(() => stereoMatchMethodsFor(isDesktopRuntime())), }; }, }); diff --git a/client/dive-common/use/stereo/StereoServerMatcher.spec.ts b/client/dive-common/use/stereo/StereoServerMatcher.spec.ts new file mode 100644 index 000000000..fc7c8f012 --- /dev/null +++ b/client/dive-common/use/stereo/StereoServerMatcher.spec.ts @@ -0,0 +1,37 @@ +import { + describe, expect, it, vi, +} from 'vitest'; +import StereoServerMatcher, { parseFrameKey } from './StereoServerMatcher'; + +const RANGE = { min: 0, max: 0 } as never; + +describe('StereoServerMatcher', () => { + it('reads the frame pair out of the transfer key', () => { + expect(parseFrameKey('left>right@12')).toEqual({ source: 'left', target: 'right', frame: 12 }); + expect(() => parseFrameKey(undefined)).toThrow('which frame'); + }); + + it('asks the server for the points of that frame and marks the misses', async () => { + const api = { + transferPoints: vi.fn(async () => ({ + success: true, transferredPoints: [[10, 11], [NaN, NaN]] as [number, number][], validMatches: [true, false], + })), + setFrame: vi.fn(async () => ({})), + }; + const matcher = new StereoServerMatcher(api, 'foundation'); + const results = await matcher.warpPoints([[1, 2], [3, 4]], null, null, null, { range: RANGE, frameKey: 'left>right@3' }); + expect(api.transferPoints).toHaveBeenCalledWith({ + frame: 3, sourceCamera: 'left', points: [[1, 2], [3, 4]], method: 'foundation', + }); + expect(results.map((r) => r.accepted)).toEqual([true, false]); + expect([results[0].x, results[0].y]).toEqual([10, 11]); + await matcher.prepare('right>left@7', null, null, null); + expect(api.setFrame).toHaveBeenCalledWith(7, 'foundation'); + }); + + it('surfaces the server reason when the warp fails', async () => { + const api = { transferPoints: vi.fn(async () => ({ success: false, error: 'no calibration' })), setFrame: vi.fn() }; + const matcher = new StereoServerMatcher(api, 'ncc'); + await expect(matcher.warpPoints([[1, 2]], null, null, null, { range: RANGE, frameKey: 'a>b@1' })).rejects.toThrow('no calibration'); + }); +}); diff --git a/client/dive-common/use/stereo/StereoServerMatcher.ts b/client/dive-common/use/stereo/StereoServerMatcher.ts new file mode 100644 index 000000000..6a7685cd5 --- /dev/null +++ b/client/dive-common/use/stereo/StereoServerMatcher.ts @@ -0,0 +1,77 @@ +import type { StereoMatcher, StereoMatchMethod } from './stereoMatcher'; +import type { WarpOptions, WarpResult } from './StereoOnnxMatcher'; + +/** Identity of the frame pair a warp belongs to, from the transfer's frameKey. */ +export interface FramePair { + source: string; + target: string; + frame: number; +} + +/** The transfer keys frames as `source>target@frame`. */ +export function parseFrameKey(key: string | undefined): FramePair { + const match = key?.match(/^(.+?)>(.+?)@(\d+)$/); + if (!match) throw new Error('A server-side warp needs to know which frame it is for.'); + return { source: match[1], target: match[2], frame: Number(match[3]) }; +} + +export interface ServerStereoApi { + transferPoints(request: { + frame: number; sourceCamera: string; points: [number, number][]; method: StereoMatchMethod; + }): Promise<{ + success: boolean; error?: string; transferredPoints?: [number, number][]; validMatches?: boolean[]; + }>; + setFrame(frame: number, method: StereoMatchMethod): Promise; +} + +/** + * A matcher whose correspondence search runs in VIAME's interactive service + * on the server. The frame pixels and rig the transfer hands over are unused: + * the server reads the same frames from its own copy of the dataset. + */ +export default class StereoServerMatcher implements StereoMatcher { + private api: ServerStereoApi; + + private method: StereoMatchMethod; + + constructor(api: ServerStereoApi, method: StereoMatchMethod) { + this.api = api; + this.method = method; + } + + async warpPoints( + points: [number, number][], + _source: unknown, + _target: unknown, + _rig: unknown, + opts: WarpOptions, + ): Promise { + if (!points.length) return []; + const pair = parseFrameKey(opts.frameKey); + const response = await this.api.transferPoints({ + frame: pair.frame, sourceCamera: pair.source, points, method: this.method, + }); + if (!response.success) { + throw new Error(response.error || 'The server could not map the points to the other camera.'); + } + const mapped = response.transferredPoints ?? []; + return points.map((_, i) => { + const point = mapped[i]; + const valid = response.validMatches?.[i] !== false + && !!point && Number.isFinite(point[0]) && Number.isFinite(point[1]); + return { + x: valid ? point[0] : NaN, + y: valid ? point[1] : NaN, + score: valid ? 1 : 0, + secondScore: 0, + accepted: valid, + }; + }); + } + + async prepare(frameKey: string, _s: unknown, _t: unknown, _r: unknown, stillWanted?: () => boolean) { + const pair = parseFrameKey(frameKey); + if (stillWanted && !stillWanted()) return; + await this.api.setFrame(pair.frame, this.method); + } +} diff --git a/client/dive-common/use/stereo/stereoMatcher.ts b/client/dive-common/use/stereo/stereoMatcher.ts index 1a0881eb5..cf4df6b47 100644 --- a/client/dive-common/use/stereo/stereoMatcher.ts +++ b/client/dive-common/use/stereo/stereoMatcher.ts @@ -1,3 +1,4 @@ +import { ref } from 'vue'; /** * The contract both correspondence methods satisfy, so the transfer composable * and its callers never branch on which one is selected. @@ -45,8 +46,13 @@ export const STEREO_MATCH_METHODS: { value: StereoMatchMethod; text: string; des { value: 'ncc', text: 'Lower Quality, Faster' }, ]; +/** Methods a server-side interactive service can run for the web client. */ +export const serverStereoMethods = ref([]); + export function stereoMatchMethodsFor(desktop: boolean) { - return STEREO_MATCH_METHODS.filter((m) => desktop || !m.desktopOnly); + return STEREO_MATCH_METHODS.filter( + (m) => desktop || !m.desktopOnly || serverStereoMethods.value.includes(m.value), + ); } export function isStereoMatchMethod(value: unknown, desktop: boolean): value is StereoMatchMethod { diff --git a/client/dive-common/use/stereo/useStereoOnnxTransfer.ts b/client/dive-common/use/stereo/useStereoOnnxTransfer.ts index c142fa735..37287e262 100644 --- a/client/dive-common/use/stereo/useStereoOnnxTransfer.ts +++ b/client/dive-common/use/stereo/useStereoOnnxTransfer.ts @@ -603,6 +603,7 @@ export default function useStereoOnnxTransfer(config: StereoOnnxTransferConfig) warpAllFromCamera, warpPoint, measureAtFrame, + measureAndReport, precomputeFrame, }; } diff --git a/client/platform/web-girder/api/configuration.service.ts b/client/platform/web-girder/api/configuration.service.ts index 78cef19c5..1224511f7 100644 --- a/client/platform/web-girder/api/configuration.service.ts +++ b/client/platform/web-girder/api/configuration.service.ts @@ -65,6 +65,11 @@ export interface DiveConfiguration { trainingEnabled?: boolean; jobsDisabled?: boolean; jobsDisabledMessage?: string; + /** Server-side interactive segmentation and stereo (the `interactive` service). */ + interactiveEnabled?: boolean; + interactiveMessage?: string; + interactiveStereoMethods?: string[]; + interactiveTextQuery?: boolean; } export interface JobsDisabledConfig { diff --git a/client/platform/web-girder/api/index.ts b/client/platform/web-girder/api/index.ts index 9bcca84e4..7118469aa 100644 --- a/client/platform/web-girder/api/index.ts +++ b/client/platform/web-girder/api/index.ts @@ -10,6 +10,7 @@ export * from './configuration.service'; export * from './dataset.service'; export { default as loadFrameMetadata } from './frameMetadata.service'; export * from './girder.service'; +export * from './interactive.service'; export * from './multicamResolve'; export * from './rpc.service'; export * from './scoring.service'; diff --git a/client/platform/web-girder/api/interactive.service.ts b/client/platform/web-girder/api/interactive.service.ts new file mode 100644 index 000000000..af9631db8 --- /dev/null +++ b/client/platform/web-girder/api/interactive.service.ts @@ -0,0 +1,174 @@ +/** + * Server-side interactive segmentation and stereo: the GPU host runs VIAME's + * interactive service and girder resolves dataset frames to its inputs. + */ +import type { + SegmentationPolygon, SegmentationPolygonKeypointsResponse, SegmentationPredictResponse, + SegmentationStereoSegmentResponse, TextQueryResponse, +} from 'dive-common/apispec'; +import type { StereoMatchMethod } from 'dive-common/use/stereo/stereoMatcher'; +import girderRest from 'platform/web-girder/plugins/girder'; + +export interface InteractiveStatus { + interactiveEnabled: boolean; + interactiveMessage: string; + interactiveStereoMethods?: StereoMatchMethod[]; + interactiveTextQuery?: boolean; +} + +interface FrameRequest { + datasetId: string; + /** Camera name of a multi-camera dataset; ignored for single-camera ones. */ + camera?: string; + frame: number; +} + +export interface InteractivePredictRequest extends FrameRequest { + points: [number, number][]; + pointLabels: number[]; + box?: [number, number, number, number]; + line?: [number, number][]; + multimaskOutput?: boolean; +} + +export interface InteractiveStereoSegmentRequest { + datasetId: string; + frame: number; + sourceCamera: string; + points: [number, number][]; + pointLabels: number[]; + polygon?: [number, number][]; + polygons?: SegmentationPolygon[]; + method?: StereoMatchMethod; +} + +export interface InteractiveTransferPointsRequest { + datasetId: string; + frame: number; + sourceCamera: string; + points: [number, number][]; + strict?: boolean; + method?: StereoMatchMethod; +} + +export interface InteractiveTransferPointsResponse { + success: boolean; + error?: string; + transferredPoints?: [number, number][]; + validMatches?: boolean[]; + disparityValues?: number[]; +} + +export interface InteractiveMeasurement { + length: number; + midpoint_x: number; + midpoint_y: number; + midpoint_z: number; + midpoint_range: number; + stereo_rms: number; +} + +export interface InteractiveTransferLineResponse { + success: boolean; + error?: string; + transferredLine?: [number, number][]; + length?: number; + measurement?: InteractiveMeasurement; +} + +const BASE = 'dive_interactive'; + +async function post(path: string, body: Record): Promise { + return (await girderRest.post(`${BASE}/${path}`, body)).data; +} + +export async function getInteractiveStatus(): Promise { + return (await girderRest.get(`${BASE}/status`)).data; +} + +export function interactiveSegmentationPredict( + request: InteractivePredictRequest, +): Promise { + return post('segmentation/predict', { ...request }); +} + +export function interactiveSegmentationKeypoints(datasetId: string, polygon: [number, number][], polygons?: SegmentationPolygon[]): Promise { + return post('segmentation/keypoints', { datasetId, polygon, polygons }); +} + +export async function interactiveStereoSegment( + request: InteractiveStereoSegmentRequest, +): Promise { + const raw = await post>('segmentation/stereo_segment', { ...request }); + return { + id: '', + success: !!raw.success, + error: raw.error as string | undefined, + polygon: raw.polygon as SegmentationStereoSegmentResponse['polygon'], + polygons: raw.polygons as SegmentationPolygon[] | undefined, + bounds: raw.bounds as SegmentationStereoSegmentResponse['bounds'], + score: raw.score as number | undefined, + seedPoints: raw.seed_points as [number, number][] | undefined, + seedLabels: raw.seed_labels as number[] | undefined, + generateLine: raw.generate_line as boolean | undefined, + lineSource: raw.line_source as SegmentationStereoSegmentResponse['lineSource'], + lineOther: raw.line_other as SegmentationStereoSegmentResponse['lineOther'], + measurement: raw.measurement as InteractiveMeasurement | undefined, + }; +} + +export function interactiveTextQuery( + request: FrameRequest & { text: string }, +): Promise { + return post('text_query', { ...request }); +} + +export function interactiveStereoSetFrame(datasetId: string, frame: number, method?: StereoMatchMethod): Promise<{ success: boolean; error?: string; disparity_ready?: boolean }> { + return post('stereo/set_frame', { datasetId, frame, method }); +} + +export async function interactiveStereoTransferPoints( + request: InteractiveTransferPointsRequest, +): Promise { + const raw = await post>('stereo/transfer_points', { ...request }); + return { + success: !!raw.success, + error: raw.error as string | undefined, + transferredPoints: raw.transferred_points as [number, number][] | undefined, + validMatches: raw.valid_matches as boolean[] | undefined, + disparityValues: raw.disparity_values as number[] | undefined, + }; +} + +export async function interactiveStereoTransferLine(datasetId: string, frame: number, line: [number, number][], method?: StereoMatchMethod): Promise { + const raw = await post>('stereo/transfer_line', { + datasetId, frame, line, method, + }); + return { + success: !!raw.success, + error: raw.error as string | undefined, + transferredLine: raw.transferred_line as [number, number][] | undefined, + length: raw.length as number | undefined, + measurement: raw.measurement as InteractiveMeasurement | undefined, + }; +} + +export function interactiveStereoMeasureLine( + datasetId: string, + frame: number, + leftLine: [number, number][], + rightLine: [number, number][], + method?: StereoMatchMethod, +): Promise<{ success: boolean; error?: string; length?: number; measurement?: InteractiveMeasurement }> { + return post('stereo/measure_line', { + datasetId, frame, leftLine, rightLine, method, + }); +} + +export function interactiveStereoAggregateLengths(datasetId: string, lengths: number[], method?: string): Promise<{ success: boolean; error?: string; avg_length?: number }> { + return post('stereo/aggregate_lengths', { datasetId, lengths, method }); +} + +export async function endInteractiveSession(datasetId: string): Promise { + await girderRest.delete(`${BASE}/session`, { params: { datasetId } }); +} diff --git a/client/platform/web-girder/store/useConfig.ts b/client/platform/web-girder/store/useConfig.ts index c80ee0135..5a69064b8 100644 --- a/client/platform/web-girder/store/useConfig.ts +++ b/client/platform/web-girder/store/useConfig.ts @@ -4,6 +4,8 @@ import { DEFAULT_JOBS_DISABLED_MESSAGE, getConfig, } from 'platform/web-girder/api/configuration.service'; +import { getInteractiveStatus } from 'platform/web-girder/api/interactive.service'; +import { serverStereoMethods, type StereoMatchMethod } from 'dive-common/use/stereo/stereoMatcher'; export interface ConfigState { distributedWorkerEnabled: boolean; @@ -11,6 +13,10 @@ export interface ConfigState { trainingEnabled: boolean; jobsDisabled: boolean; jobsDisabledMessage: string; + interactiveEnabled: boolean; + interactiveMessage: string; + interactiveStereoMethods: StereoMatchMethod[]; + interactiveTextQuery: boolean; } const distributedWorkerEnabled = ref(false); @@ -18,6 +24,9 @@ const pipelinesEnabled = ref(false); const trainingEnabled = ref(false); const jobsDisabled = ref(false); const jobsDisabledMessage = ref(DEFAULT_JOBS_DISABLED_MESSAGE); +const interactiveEnabled = ref(false); +const interactiveMessage = ref(''); +const interactiveTextQuery = ref(false); export function useConfig() { function getDistributedWorkerEnabled(): boolean { @@ -60,12 +69,27 @@ export function useConfig() { jobsDisabledMessage.value = value || DEFAULT_JOBS_DISABLED_MESSAGE; } + function setInteractive(payload: { + interactiveEnabled?: boolean; + interactiveMessage?: string; + interactiveStereoMethods?: string[]; + interactiveTextQuery?: boolean; + }): void { + interactiveEnabled.value = !!payload.interactiveEnabled; + interactiveMessage.value = payload.interactiveMessage || ''; + interactiveTextQuery.value = !!payload.interactiveTextQuery; + // Methods the server can run are offered in the settings even when the + // browser could not run them itself. + serverStereoMethods.value = (payload.interactiveStereoMethods || []) as StereoMatchMethod[]; + } + function setCapabilities(payload: Partial): void { distributedWorkerEnabled.value = payload.distributedWorkerEnabled ?? false; pipelinesEnabled.value = payload.pipelinesEnabled ?? false; trainingEnabled.value = payload.trainingEnabled ?? false; jobsDisabled.value = payload.jobsDisabled ?? false; jobsDisabledMessage.value = payload.jobsDisabledMessage || DEFAULT_JOBS_DISABLED_MESSAGE; + setInteractive(payload); } async function loadConfig(): Promise { @@ -76,9 +100,23 @@ export function useConfig() { trainingEnabled: !!data.trainingEnabled, jobsDisabled: !!data.jobsDisabled, jobsDisabledMessage: data.jobsDisabledMessage || DEFAULT_JOBS_DISABLED_MESSAGE, + interactiveEnabled: !!data.interactiveEnabled, + interactiveMessage: data.interactiveMessage || '', + interactiveStereoMethods: (data.interactiveStereoMethods || []) as StereoMatchMethod[], + interactiveTextQuery: !!data.interactiveTextQuery, }); } + /** Re-check the interactive service, whose availability follows the job queue. */ + async function refreshInteractive(): Promise { + try { + setInteractive(await getInteractiveStatus()); + } catch { + setInteractive({ interactiveEnabled: false, interactiveMessage: 'The interactive service could not be reached.' }); + } + return interactiveEnabled.value; + } + return { distributedWorkerEnabled, pipelinesEnabled, @@ -97,5 +135,9 @@ export function useConfig() { setJobsDisabledMessage, setCapabilities, loadConfig, + interactiveEnabled, + interactiveMessage, + interactiveTextQuery, + refreshInteractive, }; } diff --git a/client/platform/web-girder/useServerSegmentation.ts b/client/platform/web-girder/useServerSegmentation.ts new file mode 100644 index 000000000..b3d8e7d0f --- /dev/null +++ b/client/platform/web-girder/useServerSegmentation.ts @@ -0,0 +1,136 @@ +/** + * Web point segmentation backed by the server's interactive service: the + * recipe's clicks become `dive_interactive` requests, and a finished mask is + * carried to the other stereo camera by the service's own `stereo_segment`. + */ +import type { SegmentationPolygon, SegmentationPredictRequest } from 'dive-common/apispec'; +import { SegmentationPolygonKey } from 'dive-common/recipes/segmentationpointclick'; +import { + componentsBounds, isSegmentationPolygonKey, segmentationPolygonFeatures, +} from 'dive-common/recipes/segmentationPolygons'; +import { clientSettings } from 'dive-common/store/settings'; +import type { StereoAnnotationCompleteParams } from 'dive-common/use/useModeManager'; +import { headTailFeatures } from 'vue-media-annotator/headTail'; +import type Track from 'vue-media-annotator/track'; +import { + interactiveSegmentationPredict, interactiveStereoSegment, +} from 'platform/web-girder/api/interactive.service'; +import { fromViewer } from 'platform/web-girder/useStereoOnnxWeb'; + +interface ServerSegmentationOptions { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + getViewer: () => any; + getDatasetId: () => string; + /** Whether the server can take requests right now, refreshed by the host. */ + isEnabled: () => boolean; + /** Why it cannot, shown when the user reaches for the tool anyway. */ + disabledMessage: () => string; + refreshMeasurement: (trackId: number, frameNum: number) => Promise; + onError: (message: string) => void; +} + +export default function useServerSegmentation(opts: ServerSegmentationOptions) { + function trackAt(camera: string, trackId: number): Track | undefined { + return opts.getViewer()?.cameraStore?.getPossibleTrack(trackId, camera); + } + + function polygonsAt(camera: string, trackId: number, frameNum: number): SegmentationPolygon[] { + return (trackAt(camera, trackId)?.getFeature(frameNum)[0]?.geometry?.features ?? []) + .filter((f): f is GeoJSON.Feature => f.geometry.type === 'Polygon') + .map((f) => ({ + exterior: f.geometry.coordinates[0] as [number, number][], + holes: f.geometry.coordinates.slice(1) as [number, number][][], + })); + } + + /** Hook the shared recipe up once the Viewer is mounted. */ + function initialize(): boolean { + const viewer = opts.getViewer(); + if (!viewer?.segmentationRecipe) return false; + viewer.segmentationRecipe.initialize({ + predictFn: (request: SegmentationPredictRequest, frameNum: number) => interactiveSegmentationPredict({ + datasetId: opts.getDatasetId(), + // The recipe passes getImagePath's value back: the camera name here. + camera: request.imagePath || undefined, + frame: frameNum, + points: request.points, + pointLabels: request.pointLabels, + box: request.box, + line: request.line, + multimaskOutput: request.multimaskOutput, + }), + getImagePath: () => fromViewer(opts.getViewer()?.selectedCamera) ?? '', + initializeServiceFn: async () => { + if (!opts.isEnabled()) { + throw new Error(opts.disabledMessage() || 'Interactive segmentation is not available right now.'); + } + }, + }); + return true; + } + + /** + * A click-segmented mask on one camera, carried to the other by the + * service. Returns false when the event is not one this handles, so the + * host can pass it to the geometry transfer instead. + */ + async function handleStereoSegmentation(params: StereoAnnotationCompleteParams): Promise { + if (params.type !== 'segmentation') return false; + const viewer = opts.getViewer(); + const cameras: string[] = fromViewer(viewer?.multiCamList) ?? []; + const otherCamera = cameras.find((c) => c !== params.camera); + if (!otherCamera || !clientSettings.stereoSettings.autoComputeOtherCamera) return true; + if (!opts.isEnabled()) return true; + const polygons = polygonsAt(params.camera, params.trackId, params.frameNum); + if (!polygons.length) return true; + try { + const response = await interactiveStereoSegment({ + datasetId: opts.getDatasetId(), + frame: params.frameNum, + sourceCamera: params.camera, + points: params.points, + pointLabels: params.labels, + polygon: polygons[0].exterior, + polygons, + method: clientSettings.stereoSettings.matchMethod, + }); + if (!response.success) throw new Error(response.error || 'No mask found on the other camera.'); + let mask = response.polygons ?? []; + if (!mask.length && response.polygon) mask = [{ exterior: response.polygon, holes: [] }]; + if (!mask.length) throw new Error('No mask found on the other camera.'); + if (viewer !== opts.getViewer()) return true; + let track = trackAt(otherCamera, params.trackId); + const source = trackAt(params.camera, params.trackId); + if (!track) { + track = viewer.cameraStore.camMap.value.get(otherCamera)?.trackStore + .add(params.frameNum, source?.confidencePairs?.[0]?.[0] || 'unknown', undefined, params.trackId); + if (track && source) track.set = source.set; + } + if (!track) throw new Error('Could not create the other-camera detection.'); + const bounds = response.bounds ?? componentsBounds(mask); + track.setFeature({ + frame: params.frameNum, keyframe: true, interpolate: false, bounds, + }); + const geometries = segmentationPolygonFeatures(mask, SegmentationPolygonKey); + const keys = new Set(geometries.map((g) => g.properties?.key)); + track.getPolygonFeatures(params.frameNum).forEach((p) => { + if (isSegmentationPolygonKey(p.key, SegmentationPolygonKey) && !keys.has(p.key)) { + track!.removeFeatureGeometry(params.frameNum, { key: p.key, type: 'Polygon' }); + } + }); + track.setFeature({ frame: params.frameNum, keyframe: true }, geometries); + if (response.generateLine && response.lineOther) { + track.setFeature({ frame: params.frameNum, keyframe: true, bounds }, headTailFeatures(response.lineOther)); + if (response.lineSource && source) { + source.setFeature({ frame: params.frameNum, keyframe: true }, headTailFeatures(response.lineSource)); + } + } + await opts.refreshMeasurement(params.trackId, params.frameNum); + } catch (err) { + opts.onError(`Stereo segmentation: ${(err as Error).message}`); + } + return true; + } + + return { initialize, handleStereoSegmentation }; +} diff --git a/client/platform/web-girder/useStereoOnnxWeb.ts b/client/platform/web-girder/useStereoOnnxWeb.ts index 92a598eb6..f024576f4 100644 --- a/client/platform/web-girder/useStereoOnnxWeb.ts +++ b/client/platform/web-girder/useStereoOnnxWeb.ts @@ -72,6 +72,11 @@ export interface StereoOnnxWebOptions { foundationModelSpec?: FoundationModelSpec; /** Overrides the user's dropdown choice; mainly for tests. */ getMatchMethod?: () => StereoMatchMethod; + /** + * A matcher run by the server's interactive service, used for every method + * while `enabled()` holds; the browser models are the fallback. + */ + server?: { enabled: () => boolean; matcher: (method: StereoMatchMethod) => StereoMatcher }; range?: SearchRange; /** * Progress message; null clears it. `progress` accompanies the messages whose @@ -213,6 +218,11 @@ export default function useStereoOnnxWeb(opts: StereoOnnxWebOptions) { function getMatcher(imagery?: ImagerySize): Promise { const method = currentMethod(); + if (opts.server?.enabled()) { + const serverKey = `server:${method}`; + if (!matchers[serverKey]) matchers[serverKey] = Promise.resolve(opts.server.matcher(method)); + return matchers[serverKey]; + } const key = method === 'foundation' && imagery ? `${method}:${imagery.width}x${imagery.height}` : method; @@ -438,5 +448,8 @@ export default function useStereoOnnxWeb(opts: StereoOnnxWebOptions) { stereoViewLink, precomputeCurrentFrame, invalidateCalibration, + refreshMeasurement: async (trackId: number, frameNum: number) => ( + getTransfer()?.measureAndReport(trackId, frameNum) + ), }; } diff --git a/client/platform/web-girder/views/ViewerLoader.vue b/client/platform/web-girder/views/ViewerLoader.vue index eedb65de1..cb6d4b1fa 100644 --- a/client/platform/web-girder/views/ViewerLoader.vue +++ b/client/platform/web-girder/views/ViewerLoader.vue @@ -1,6 +1,6 @@