From d42176d2e8d493f3c415d1e1919ca16d05c84625 Mon Sep 17 00:00:00 2001 From: Bryon Lewis Date: Mon, 11 May 2026 16:04:03 -0400 Subject: [PATCH 01/12] visual mask configuration --- client/dive-common/components/EditorMenu.vue | 40 +- .../dive-common/components/SidebarContext.vue | 3 + client/dive-common/components/Viewer.vue | 76 ++- .../components/VisualMaskSidebar.vue | 444 +++++++++++++++++ client/dive-common/use/useModeManager.ts | 2 +- .../web-girder/views/ViewerLoader.vue | 2 + client/src/ConfigurationManager.ts | 15 + client/src/components/LayerManager.vue | 147 +++++- client/src/layers/index.ts | 2 +- client/src/provides.ts | 27 +- client/src/visualMasks.spec.ts | 65 +++ client/src/visualMasks.ts | 460 ++++++++++++++++++ server/dive_utils/models.py | 10 + 13 files changed, 1274 insertions(+), 19 deletions(-) create mode 100644 client/dive-common/components/VisualMaskSidebar.vue create mode 100644 client/src/visualMasks.spec.ts create mode 100644 client/src/visualMasks.ts diff --git a/client/dive-common/components/EditorMenu.vue b/client/dive-common/components/EditorMenu.vue index caa6fe08..0c267ba5 100644 --- a/client/dive-common/components/EditorMenu.vue +++ b/client/dive-common/components/EditorMenu.vue @@ -7,7 +7,8 @@ import { Mousetrap, OverlayPreferences } from 'vue-media-annotator/types'; import { EditAnnotationTypes, VisibleAnnotationTypes } from 'vue-media-annotator/layers'; import Recipe from 'vue-media-annotator/recipe'; import { hexToRgb } from 'vue-media-annotator/utils'; -import { useMasks } from 'vue-media-annotator/provides'; +import { useMasks, useConfiguration } from 'vue-media-annotator/provides'; +import { useStore } from 'platform/web-girder/store/types'; import MaskTracking from 'dive-common/components/MaskTracking.vue'; interface ButtonData { @@ -79,6 +80,29 @@ export default defineComponent({ const toolTipForce = ref(false); let toolTimeTimeout: number | undefined; const { editorOptions, editorFunctions } = useMasks(); + const configMan = useConfiguration(); + const store = useStore(); + const isOwnerAdmin = computed(() => { + const currentUser = store.state.User.user as ({ + admin?: boolean; + _id?: string; + groups?: string[]; + } | null); + if (!currentUser) { + return false; + } + let ownerAdmin = !!currentUser.admin; + if (configMan.configOwners.value.users + .findIndex((item) => item.id === currentUser._id) !== -1) { + ownerAdmin = true; + } + (currentUser.groups || []).forEach((group: string) => { + if (configMan.configOwners.value.groups.findIndex((item) => item.id === group) !== -1) { + ownerAdmin = true; + } + }); + return ownerAdmin; + }); const modeToolTips = { Creating: { rectangle: 'Drag to draw rectangle. Press ESC to exit.', @@ -217,6 +241,14 @@ export default defineComponent({ tooltip: 'Tooltip Information about Hovered over annotations', click: () => toggleVisible('tooltip'), }, + { + id: 'VisualMask', + type: 'VisualMask', + active: isVisible('VisualMask'), + icon: 'mdi-image-filter-center-focus-strong', + tooltip: 'Configuration visual masks', + click: () => toggleVisible('VisualMask'), + }, ]; if (props.attributeKey) { buttons.push({ @@ -228,12 +260,18 @@ export default defineComponent({ click: () => toggleVisible('attributeKey'), }); } + if (!isOwnerAdmin.value) { + return buttons.filter((button) => button.id !== 'VisualMask'); + } return buttons; }); const isVisible = (mode: VisibleAnnotationTypes) => props.visibleModes.includes(mode); const toggleVisible = (mode: VisibleAnnotationTypes) => { + if (mode === 'VisualMask' && !isOwnerAdmin.value) { + return; + } if (isVisible(mode)) { emit('set-annotation-state', { visible: props.visibleModes.filter((m) => m !== mode), diff --git a/client/dive-common/components/SidebarContext.vue b/client/dive-common/components/SidebarContext.vue index 42bb5e13..36259c72 100644 --- a/client/dive-common/components/SidebarContext.vue +++ b/client/dive-common/components/SidebarContext.vue @@ -86,5 +86,8 @@ export default defineComponent({ overflow-y: hidden; } .sidebar-content { + flex: 1 1 auto; + min-height: 0; + overflow: hidden; } diff --git a/client/dive-common/components/Viewer.vue b/client/dive-common/components/Viewer.vue index 3e775fa4..c1ba79c4 100644 --- a/client/dive-common/components/Viewer.vue +++ b/client/dive-common/components/Viewer.vue @@ -22,6 +22,7 @@ import { StyleManager, TrackFilterControls, GroupFilterControls, ConfigurationManager, } from 'vue-media-annotator/index'; +import VisualMaskManager from 'vue-media-annotator/visualMasks'; import { provideAnnotator } from 'vue-media-annotator/provides'; import { @@ -66,6 +67,7 @@ import { useRoute } from 'vue-router/composables'; import AttributeShortcutToggle from './Attributes/AttributeShortcutToggle.vue'; import GroupSidebarVue from './GroupSidebar.vue'; import MultiCamToolsVue from './MultiCamTools.vue'; +import VisualMaskSidebarVue from './VisualMaskSidebar.vue'; import PrevNext from './PrevNext.vue'; import AttributesSideBarVue from './Attributes/AttributesSideBar.vue'; import TypeThresholdVue from './TypeThreshold.vue'; @@ -172,6 +174,27 @@ export default defineComponent({ }); const store = useStore(); + const isConfigOwnerAdmin = computed(() => { + const currentUser = store.state.User.user as ({ + admin?: boolean; + _id?: string; + groups?: string[]; + } | null); + if (!currentUser) { + return false; + } + let ownerAdmin = !!currentUser.admin; + if (configurationManager.configOwners.value.users + .findIndex((item) => item.id === currentUser._id) !== -1) { + ownerAdmin = true; + } + (currentUser.groups || []).forEach((group: string) => { + if (configurationManager.configOwners.value.groups.findIndex((item) => item.id === group) !== -1) { + ownerAdmin = true; + } + }); + return ownerAdmin; + }); const { save: saveToServer, @@ -208,12 +231,32 @@ export default defineComponent({ const vuetify = inject('vuetify') as Vuetify; const trackStyleManager = new StyleManager({ markChangesPending, vuetify }); const groupStyleManager = new StyleManager({ markChangesPending, vuetify }); + const visualMaskStyleManager = new StyleManager({ markChangesPending, vuetify }); const cameraStore = new CameraStore({ markChangesPending }); // eslint-disable-next-line max-len const configurationManager = new ConfigurationManager({ configurationId, setConfigurationId, saveConfiguration, transferConfiguration, }); + const visualMaskManager = new VisualMaskManager({ + markChangesPending, + styleManager: visualMaskStyleManager, + syncConfiguration: (visualMasks) => { + if (!configurationManager.configuration.value) { + if (!Object.keys(visualMasks).length) { + return; + } + configurationManager.setConfiguration({}); + } + if (configurationManager.configuration.value) { + if (Object.keys(visualMasks).length) { + configurationManager.configuration.value.visualMasks = visualMasks; + } else { + delete configurationManager.configuration.value.visualMasks; + } + } + }, + }); // This context for removal const removeGroups = (id: AnnotationId) => { @@ -507,6 +550,10 @@ export default defineComponent({ handler.trackSelect(selectedTrackId.value, false); } try { + await configurationManager.saveConfiguration( + configurationId.value, + { visualMasks: visualMaskManager.serialize() }, + ); await saveToServer({ customTypeStyling: trackStyleManager.getTypeStyles(trackFilters.allTypes), customGroupStyling: groupStyleManager.getTypeStyles(groupFilters.allTypes), @@ -526,7 +573,8 @@ export default defineComponent({ } } catch (err) { let text = 'Unable to Save Data'; - if (err.response && err.response.status === 403) { + const errorResponse = err as { response?: { status?: number } }; + if (errorResponse.response && errorResponse.response.status === 403) { text = 'You do not have permission to Save Data to this Folder.'; } await prompt({ @@ -635,6 +683,7 @@ export default defineComponent({ configurationManager.setConfiguration( config.diveConfig.metadata.configuration, ); + visualMaskManager.load(config.diveConfig.metadata.configuration.visualMasks); if (config.diveConfig.metadata.configuration.general?.baseConfiguration) { configurationManager.setConfigurationId( @@ -644,6 +693,8 @@ export default defineComponent({ if (config.diveConfig.metadata.configuration.filterTimelines) { useTimelineFilters.loadFilterTimelines(config.diveConfig.metadata.configuration.filterTimelines); } + } else { + visualMaskManager.load(); } const flatUIMap = configurationManager.getFlatUISettingMap(); ctx.emit('get-ui-settings', flatUIMap); @@ -836,6 +887,18 @@ export default defineComponent({ component: DatasetInfo, }); } + if (!configurationManager.getUISetting('UIVisualMasks') || !isConfigOwnerAdmin.value) { + context.unregister({ + description: 'Visual Masks', + component: VisualMaskSidebarVue, + }); + } else { + context.register({ + description: 'Visual Masks', + component: VisualMaskSidebarVue, + width: 340, + }); + } if (!configurationManager.getUISetting('UIThresholdControls')) { context.unregister({ @@ -861,7 +924,7 @@ export default defineComponent({ progress.loaded = false; console.error(err); const errorEl = document.createElement('div'); - errorEl.innerHTML = getResponseError(err); + errorEl.innerHTML = getResponseError(err as never); loadError.value = errorEl.innerText .concat(". If you don't know how to resolve this, please contact the server administrator."); throw err; @@ -979,6 +1042,8 @@ export default defineComponent({ editingMode, groupFilters, groupStyleManager, + visualMaskManager, + visualMaskStyleManager, multiSelectList, pendingSaveCount, progress, @@ -1063,6 +1128,7 @@ export default defineComponent({ selectedTrackId, editingGroupId, selectedKey, + visualMaskManager, trackFilters, videoUrl, overlays, @@ -1168,7 +1234,7 @@ export default defineComponent({ recipes, multiSelectActive, editingDetails, - overlays, + overlays: overlays ? [...overlays] : [], groupEditActive: editingGroupId !== null, }" :get-u-i-setting="getUISetting" @@ -1303,7 +1369,7 @@ export default defineComponent({ v-mousetrap="[ { bind: 'n', handler: () => !readonlyState && handler.trackAdd() }, { bind: 'r', handler: () => aggregateController.resetZoom() }, - { bind: 'esc', handler: () => getUISetting('UISelection') && handler.trackAbort() }, + { bind: 'esc', handler: () => (visualMaskManager.editingMaskId !== null ? visualMaskManager.stopEditing() : (getUISetting('UISelection') && handler.trackAbort())) }, ]" class="d-flex flex-column grow" > @@ -1337,7 +1403,7 @@ export default defineComponent({ diff --git a/client/dive-common/components/VisualMaskSidebar.vue b/client/dive-common/components/VisualMaskSidebar.vue new file mode 100644 index 00000000..c6478b16 --- /dev/null +++ b/client/dive-common/components/VisualMaskSidebar.vue @@ -0,0 +1,444 @@ + + + + + diff --git a/client/dive-common/use/useModeManager.ts b/client/dive-common/use/useModeManager.ts index 38590835..259bc617 100644 --- a/client/dive-common/use/useModeManager.ts +++ b/client/dive-common/use/useModeManager.ts @@ -84,7 +84,7 @@ export default function useModeManager({ let creating = false; const { prompt, inputValue } = usePrompt(); const annotationModes = reactive({ - visible: ['rectangle', 'Polygon', 'LineString', 'text', 'Time'] as VisibleAnnotationTypes[], + visible: ['rectangle', 'Polygon', 'LineString', 'VisualMask', 'text', 'Time'] as VisibleAnnotationTypes[], editing: 'rectangle' as EditAnnotationTypes, }); const trackSettings = toRef(clientSettings, 'trackSettings'); diff --git a/client/platform/web-girder/views/ViewerLoader.vue b/client/platform/web-girder/views/ViewerLoader.vue index dd288f43..17d7f541 100644 --- a/client/platform/web-girder/views/ViewerLoader.vue +++ b/client/platform/web-girder/views/ViewerLoader.vue @@ -12,6 +12,7 @@ import context from 'dive-common/store/context'; import { useStore } from 'platform/web-girder/store/types'; import { usePrompt } from 'dive-common/vue-utilities/prompt-service'; import ConfigurationEditor from 'dive-common/components/ConfigurationEditor.vue'; +import VisualMaskSidebar from 'dive-common/components/VisualMaskSidebar.vue'; import { UISettingsKey } from 'vue-media-annotator/ConfigurationManager'; import { useRouter } from 'vue-router/composables'; @@ -65,6 +66,7 @@ export default defineComponent({ ViewerAlert, DIVETools, ConfigurationEditor, + VisualMaskSidebar, AnnotationDataBrowser, ...context.getComponents(), }, diff --git a/client/src/ConfigurationManager.ts b/client/src/ConfigurationManager.ts index 7c8024d2..55c4f1e4 100644 --- a/client/src/ConfigurationManager.ts +++ b/client/src/ConfigurationManager.ts @@ -2,6 +2,8 @@ import { ref, Ref } from 'vue'; import { DIVEAction, DIVEActionShortcut } from 'dive-common/use/useActions'; import { isArray } from 'lodash'; import type { FilterTimeline } from './use/useTimelineFilters'; +import type { CustomStyle } from './StyleManager'; +import type { Feature } from './track'; export interface DiveConfiguration { prevNext?: { @@ -68,6 +70,7 @@ interface UIContextBar { UITrackList? : boolean; UIDatasetInfo?: boolean; UIAttributeUserReview?: boolean; + UIVisualMasks?: boolean; } interface UITrackDetails { @@ -139,6 +142,17 @@ export interface CustomUISettings { width? : number; } +export type VisualMaskGeometryType = 'rectangle' | 'Polygon'; + +export interface VisualMaskConfiguration { + id: number; + name: string; + enabled?: boolean; + type: VisualMaskGeometryType; + frames: Feature[]; + style?: CustomStyle; +} + export interface Configuration { general?: { configurationMerge? : 'merge up' | 'merge down' | 'disabled'; @@ -152,6 +166,7 @@ export interface Configuration { filterTimelines?: FilterTimeline[]; timelineConfigs?: TimelineConfiguration[]; customUI?: CustomUISettings; + visualMasks?: Record; } // eslint-disable-next-line @typescript-eslint/no-explicit-any diff --git a/client/src/components/LayerManager.vue b/client/src/components/LayerManager.vue index 123f0e03..cbd4d1fb 100644 --- a/client/src/components/LayerManager.vue +++ b/client/src/components/LayerManager.vue @@ -47,6 +47,8 @@ import { useConfiguration, useAttributes, useMasks, + useVisualMaskManager, + useVisualMaskStyleManager, } from '../provides'; const useworker = true; @@ -104,6 +106,8 @@ export default defineComponent({ const selectedKeyRef = useSelectedKey(); const trackStyleManager = useTrackStyleManager(); const groupStyleManager = useGroupStyleManager(); + const visualMaskManager = useVisualMaskManager(); + const visualMaskStyleManager = useVisualMaskStyleManager(); const annotatorPrefs = useAnnotatorPreferences(); const typeStylingRef = computed(() => { if (props.colorBy === 'group') { @@ -151,6 +155,16 @@ export default defineComponent({ } } + const visualMaskRectLayer = new RectangleLayer({ + annotator, + stateStyling: visualMaskStyleManager.stateStyles, + typeStyling: visualMaskStyleManager.typeStyling, + }); + const visualMaskPolyLayer = new PolygonLayer({ + annotator, + stateStyling: visualMaskStyleManager.stateStyles, + typeStyling: visualMaskStyleManager.typeStyling, + }); const rectAnnotationLayer = new RectangleLayer({ annotator, stateStyling: trackStyleManager.stateStyles, @@ -208,6 +222,12 @@ export default defineComponent({ typeStyling: typeStylingRef, type: 'rectangle', }); + const visualMaskEditLayer = new EditAnnotationLayer({ + annotator, + stateStyling: visualMaskStyleManager.stateStyles, + typeStyling: visualMaskStyleManager.typeStyling, + type: 'rectangle', + }); const updateAttributes = () => { const newList = attributes.value.filter((item) => item.render).sort((a, b) => { @@ -259,10 +279,25 @@ export default defineComponent({ const frameData = [] as FrameDataTrack[]; const editingTracks = [] as FrameDataTrack[]; - if (currentFrameIds === undefined) { - return; - } - currentFrameIds.forEach( + const visualMaskFrameData = [] as FrameDataTrack[]; + visualMaskManager.getMasks(props.camera).forEach((visualMask) => { + if (!visualMask.enabled) { + return; + } + const features = visualMask.getFeature(frame); + if (!features) { + return; + } + visualMaskFrameData.push({ + selected: visualMaskManager.selectedMaskId.value === visualMask.id, + editing: visualMaskManager.editingMaskId.value === visualMask.id, + track: visualMask as never, + groups: [], + features, + styleType: [visualMask.styleKey, 1], + }); + }); + (currentFrameIds || []).forEach( (trackId: AnnotationId) => { if (trackStore?.annotationIds.value.length === 0) { return; // No annotations so just skip the updating @@ -313,6 +348,16 @@ export default defineComponent({ }, ); + if (visibleModes.includes('VisualMask')) { + visualMaskRectLayer.setDrawingOther(['Polygon']); + visualMaskRectLayer.changeData(visualMaskFrameData); + visualMaskPolyLayer.setDrawingOther(true); + visualMaskPolyLayer.changeData(visualMaskFrameData); + } else { + visualMaskRectLayer.disable(); + visualMaskPolyLayer.disable(); + } + if (visibleModes.includes('rectangle')) { //We modify rects opacity/thickness if polygons are visible or not rectAnnotationLayer.setDrawingOther(visibleModes); @@ -423,8 +468,26 @@ export default defineComponent({ timeLayer.disable(); } - if (selectedTrackId !== null) { - if ((editingTrack) && !currentFrameIds.includes(selectedTrackId) + const editingVisualMask = props.camera === selectedCamera.value + ? visualMaskManager.getEditingMask(props.camera) + : undefined; + if (editingVisualMask) { + const visualMaskFrame = { + selected: true, + editing: true, + track: editingVisualMask as never, + groups: [], + features: editingVisualMask.getFeature(frame), + styleType: [editingVisualMask.styleKey, 1] as [string, number], + }; + visualMaskEditLayer.setType(editingVisualMask.type); + visualMaskEditLayer.setKey(''); + visualMaskEditLayer.changeData([visualMaskFrame]); + editAnnotationLayer.disable(); + maskEditorLayer.disable(); + rectAnnotationLayer.setDisableClicking(false); + } else if (selectedTrackId !== null) { + if ((editingTrack) && !(currentFrameIds || []).includes(selectedTrackId) && props.camera === selectedCamera.value) { const editTrack = trackStore?.getPossible(selectedTrackId); if (editTrack === undefined) { @@ -448,9 +511,11 @@ export default defineComponent({ editAnnotationLayer.setKey(selectedKey); editAnnotationLayer.changeData(editingTracks); } + visualMaskEditLayer.disable(); maskEditorLayer.disable(); } else if (editingTracks.length && editingTrack === 'Mask') { maskLayer.disable(); + visualMaskEditLayer.disable(); editAnnotationLayer.disable(); rectAnnotationLayer.setDisableClicking(true); const track = editingTracks[0]; @@ -471,11 +536,13 @@ export default defineComponent({ }); getOrCreateFilter(track.styleType[0], typeStylingRef.value.color(track.styleType[0])); } else { + visualMaskEditLayer.disable(); editAnnotationLayer.disable(); maskEditorLayer.disable(); rectAnnotationLayer.setDisableClicking(false); } } else { + visualMaskEditLayer.disable(); editAnnotationLayer.disable(); rectAnnotationLayer.setDisableClicking(false); if (maskEditorLayer.checkEnabled()) { @@ -513,6 +580,7 @@ export default defineComponent({ multiSeletListRef, visibleModesRef, typeStylingRef, + visualMaskManager.revisionCounter, toRef(props, 'colorBy'), selectedCamera, ], @@ -587,6 +655,7 @@ export default defineComponent({ if (selectedCamera.value !== props.camera) { return; } + visualMaskManager.clearSelection(); //So we only want to pass the click whjen not in creation mode or editing mode for features if (editAnnotationLayer.getMode() !== 'creation' && getUISetting('UISelection')) { editAnnotationLayer.disable(); @@ -595,16 +664,42 @@ export default defineComponent({ } }; + const visualMaskClicked = (maskId: number | null, editing: boolean) => { + if (selectedCamera.value !== props.camera) { + return; + } + handler.trackSelect(null, false); + if (maskId === null) { + visualMaskManager.clearSelection(); + return; + } + visualMaskManager.selectMask(maskId); + if (editing) { + visualMaskManager.startEditing(props.camera, maskId); + } else { + visualMaskManager.stopEditing(); + } + }; + //Sync of internal geoJS state with the application editAnnotationLayer.bus.$on('editing-annotation-sync', (editing: boolean) => { handler.trackSelect(selectedTrackIdRef.value, editing); }); + visualMaskEditLayer.bus.$on('editing-annotation-sync', (editing: boolean) => { + if (!editing) { + visualMaskManager.stopEditing(); + } + }); rectAnnotationLayer.bus.$on('annotation-clicked', Clicked); rectAnnotationLayer.bus.$on('annotation-right-clicked', Clicked); timeLayer.bus.$on('annotation-clicked', Clicked); timeLayer.bus.$on('annotation-right-clicked', Clicked); polyAnnotationLayer.bus.$on('annotation-clicked', Clicked); polyAnnotationLayer.bus.$on('annotation-right-clicked', Clicked); + visualMaskRectLayer.bus.$on('annotation-clicked', (maskId: number | null) => visualMaskClicked(maskId, false)); + visualMaskRectLayer.bus.$on('annotation-right-clicked', (maskId: number | null) => visualMaskClicked(maskId, true)); + visualMaskPolyLayer.bus.$on('annotation-clicked', (maskId: number | null) => visualMaskClicked(maskId, false)); + visualMaskPolyLayer.bus.$on('annotation-right-clicked', (maskId: number | null) => visualMaskClicked(maskId, true)); editAnnotationLayer.bus.$on('update:geojson', ( mode: 'in-progress' | 'editing', geometryCompleteEvent: boolean, @@ -642,6 +737,38 @@ export default defineComponent({ ); } }); + visualMaskEditLayer.bus.$on('update:geojson', ( + _mode: 'in-progress' | 'editing', + geometryCompleteEvent: boolean, + data: GeoJSON.Feature, + type: string, + key = '', + cb: () => void, + ) => { + if (key) { + // Visual masks store a single shape per frame and do not use keyed geometries. + } + if (type === 'rectangle') { + const bounds = geojsonToBound(data as GeoJSON.Feature); + cb(); + visualMaskManager.updateRectBounds(props.camera, frameNumberRef.value, bounds); + } else if (type === 'Polygon') { + cb(); + visualMaskManager.updateGeoJSON(props.camera, frameNumberRef.value, data as GeoJSON.Feature); + } + if (geometryCompleteEvent) { + updateLayers( + frameNumberRef.value, + editingModeRef.value, + selectedTrackIdRef.value, + multiSeletListRef.value, + enabledTracksRef.value, + visibleModesRef.value, + selectedKeyRef.value, + props.colorBy, + ); + } + }); editAnnotationLayer.bus.$on( 'update:selectedIndex', (index: number, _type: EditAnnotationTypes, key = '') => handler.selectFeatureHandle(index, key), @@ -748,15 +875,15 @@ export default defineComponent({ @@ -800,7 +927,7 @@ export default defineComponent({ id="colorScale" in="SourceGraphic" type="matrix" - :values="overlay.colorScaleMatrix" + :values="overlay.colorScaleMatrix.join(' ')" /> diff --git a/client/src/layers/index.ts b/client/src/layers/index.ts index c2902314..eb142ddc 100644 --- a/client/src/layers/index.ts +++ b/client/src/layers/index.ts @@ -12,7 +12,7 @@ import * as UILayerTypes from './UILayers/UILayerTypes'; import EditAnnotationLayer from './EditAnnotationLayer'; import type { EditAnnotationTypes } from './EditAnnotationLayer'; -type VisibleAnnotationTypes = EditAnnotationTypes | 'text' | 'tooltip' | 'TrackTail' | 'overlays' | 'attributeKey'; +type VisibleAnnotationTypes = EditAnnotationTypes | 'VisualMask' | 'text' | 'tooltip' | 'TrackTail' | 'overlays' | 'attributeKey'; export { /* AnnotationLayers */ diff --git a/client/src/provides.ts b/client/src/provides.ts index f62c9806..16d1ed38 100644 --- a/client/src/provides.ts +++ b/client/src/provides.ts @@ -31,6 +31,7 @@ import CameraStore from './CameraStore'; import ConfigurationManager from './ConfigurationManager'; import { EventChartData } from './use/useEventChart'; import type { FilterTimeline } from './use/useTimelineFilters'; +import VisualMaskManager from './visualMasks'; /** * Type definitions are read only because injectors may mutate internal state, @@ -141,9 +142,11 @@ const UINotificationSymbol = Symbol('uiNotification'); const TrackStyleManagerSymbol = Symbol('trackTypeStyling'); const GroupStyleManagerSymbol = Symbol('groupTypeStyling'); +const VisualMaskStyleManagerSymbol = Symbol('visualMaskStyling'); const TrackFilterControlsSymbol = Symbol('trackFilters'); const GroupFilterControlsSymbol = Symbol('groupFilters'); +const VisualMaskManagerSymbol = Symbol('visualMaskManager'); /** * Handler interface describes all global events mutations @@ -291,6 +294,8 @@ export interface State { editingMode: EditingModeType; groupFilters: GroupFilterControls; groupStyleManager: StyleManager; + visualMaskManager: VisualMaskManager; + visualMaskStyleManager: StyleManager; multiSelectList: MultiSelectType; pendingSaveCount: pendingSaveCountType; progress: ProgressType; @@ -337,6 +342,12 @@ function dummyState(): State { groupFilterControls, lookupGroups: cameraStore.lookupGroups, }); + const visualMaskStyleManager = new StyleManager({ markChangesPending }); + const visualMaskManager = new VisualMaskManager({ + markChangesPending, + styleManager: visualMaskStyleManager, + syncConfiguration: () => {}, + }); return { annotatorPreferences: ref({ trackTails: { before: 20, after: 10 }, @@ -357,6 +368,8 @@ function dummyState(): State { latestRevisionId: ref(0), groupFilters: groupFilterControls, groupStyleManager: new StyleManager({ markChangesPending }), + visualMaskManager, + visualMaskStyleManager, selectedCamera: ref('singleCam'), selectedKey: ref(''), selectedTrackId: ref(null), @@ -424,7 +437,7 @@ function dummyState(): State { }, trackFilters: trackFilterControls, trackStyleManager: new StyleManager({ markChangesPending }), - visibleModes: ref(['rectangle', 'text'] as VisibleAnnotationTypes[]), + visibleModes: ref(['rectangle', 'VisualMask', 'text'] as VisibleAnnotationTypes[]), readOnlyMode: ref(false), imageEnhancements: ref({}), diveMetadataRootId: ref(null), @@ -450,6 +463,8 @@ function provideAnnotator(state: State, handler: Handler, attributesFilters: Att provide(EditingModeSymbol, state.editingMode); provide(GroupFilterControlsSymbol, state.groupFilters); provide(GroupStyleManagerSymbol, state.groupStyleManager); + provide(VisualMaskManagerSymbol, state.visualMaskManager); + provide(VisualMaskStyleManagerSymbol, state.visualMaskStyleManager); provide(MultiSelectSymbol, state.multiSelectList); provide(PendingSaveCountSymbol, state.pendingSaveCount); provide(ProgressSymbol, state.progress); @@ -526,6 +541,14 @@ function useGroupStyleManager() { return use(GroupStyleManagerSymbol); } +function useVisualMaskManager() { + return use(VisualMaskManagerSymbol); +} + +function useVisualMaskStyleManager() { + return use(VisualMaskStyleManagerSymbol); +} + function useHandler() { return use(HandlerSymbol); } @@ -611,6 +634,8 @@ export { useHandler, useGroupFilterControls, useGroupStyleManager, + useVisualMaskManager, + useVisualMaskStyleManager, useMultiSelectList, usePendingSaveCount, useProgress, diff --git a/client/src/visualMasks.spec.ts b/client/src/visualMasks.spec.ts new file mode 100644 index 00000000..7b6e94a3 --- /dev/null +++ b/client/src/visualMasks.spec.ts @@ -0,0 +1,65 @@ +/// +import VisualMaskManager, { VisualMask } from './visualMasks'; +import StyleManager from './StyleManager'; + +describe('VisualMask', () => { + it('persists the first shape until a later keyframe changes it', () => { + const mask = new VisualMask({ + id: 1, + name: 'Road Mask', + type: 'rectangle', + frames: [{ + frame: 10, + bounds: [0, 0, 10, 10], + keyframe: true, + }, { + frame: 20, + bounds: [5, 5, 15, 15], + keyframe: true, + }], + }); + + expect(mask.getFeature(0)?.bounds).toEqual([0, 0, 10, 10]); + expect(mask.getFeature(15)?.bounds).toEqual([0, 0, 10, 10]); + expect(mask.getFeature(20)?.bounds).toEqual([5, 5, 15, 15]); + expect(mask.getFeature(30)?.bounds).toEqual([5, 5, 15, 15]); + }); +}); + +describe('VisualMaskManager', () => { + it('serializes per-camera visual masks with their styles', () => { + const styleManager = new StyleManager({ markChangesPending: () => {} }); + let serializedMasks = {}; + const manager = new VisualMaskManager({ + markChangesPending: () => {}, + styleManager, + syncConfiguration: (visualMasks) => { + serializedMasks = visualMasks; + }, + }); + + const id = manager.addMask('singleCam', 'rectangle'); + manager.setMaskStyle('singleCam', id, { color: '#ff0000', opacity: 0.5 }); + manager.updateRectBounds('singleCam', 12, [1, 2, 3, 4], id); + + expect(serializedMasks).toEqual({ + singleCam: [{ + id, + name: 'Mask 1', + enabled: true, + type: 'rectangle', + frames: [{ + frame: 12, + bounds: [1, 2, 3, 4], + keyframe: true, + }], + style: { + color: '#ff0000', + fill: true, + opacity: 0.5, + strokeWidth: 3, + }, + }], + }); + }); +}); diff --git a/client/src/visualMasks.ts b/client/src/visualMasks.ts new file mode 100644 index 00000000..9e75f599 --- /dev/null +++ b/client/src/visualMasks.ts @@ -0,0 +1,460 @@ +/* eslint-disable max-classes-per-file */ +import { computed, ref, Ref } from 'vue'; +import type { Feature, TrackSupportedFeature } from './track'; +import type { RectBounds } from './utils'; +import { geojsonToBound } from './utils'; +import { listInsert, listRemove } from './listUtils'; +import StyleManager, { CustomStyle } from './StyleManager'; +import type { + VisualMaskConfiguration, + VisualMaskGeometryType, +} from './ConfigurationManager'; + +const DEFAULT_VISUAL_MASK_STYLE: CustomStyle = { + color: '#000000', + fill: true, + opacity: 0.35, + strokeWidth: 3, +}; + +function getVisualMaskStyleKey(id: number) { + return `visual-mask-${id}`; +} + +function normalizeStyle(style?: CustomStyle): CustomStyle { + return { + ...DEFAULT_VISUAL_MASK_STYLE, + ...style, + }; +} + +export class VisualMask { + id: number; + + name: string; + + enabled: boolean; + + type: VisualMaskGeometryType; + + features: Feature[]; + + featureIndex: number[]; + + style: CustomStyle; + + constructor({ + id, + name, + enabled = true, + type, + frames, + style, + }: VisualMaskConfiguration) { + this.id = id; + this.name = name; + this.enabled = enabled; + this.type = type; + this.features = []; + this.featureIndex = []; + this.style = normalizeStyle(style); + (frames || []).forEach((frame) => { + this.setFeature(frame, frame.geometry?.features || [], false); + }); + } + + get styleKey() { + return getVisualMaskStyleKey(this.id); + } + + getKeyframes() { + return [...this.featureIndex]; + } + + getExactFeature(frame: number) { + return this.features[frame] || null; + } + + getFeature(frame: number) { + const exact = this.getExactFeature(frame); + if (exact) { + return exact; + } + if (!this.featureIndex.length) { + return null; + } + const previousFrames = this.featureIndex.filter((item) => item <= frame); + const sourceFrame = previousFrames.length + ? previousFrames[previousFrames.length - 1] + : this.featureIndex[0]; + const source = this.features[sourceFrame]; + if (!source) { + return null; + } + return { + ...source, + frame, + keyframe: false, + }; + } + + getNextKeyframe(frame: number) { + return this.featureIndex.find((item) => item > frame); + } + + getPreviousKeyframe(frame: number) { + const previousFrames = this.featureIndex.filter((item) => item < frame); + if (!previousFrames.length) { + return undefined; + } + return previousFrames[previousFrames.length - 1]; + } + + setFeature( + feature: Feature, + geometry: GeoJSON.Feature[] = [], + ensureKeyframe = true, + ) { + const { frame } = feature; + const current = this.features[frame] || { frame }; + const next: Feature = { + ...current, + ...feature, + keyframe: ensureKeyframe ? true : feature.keyframe, + }; + if (next.bounds) { + next.bounds = [ + Math.round(next.bounds[0]), + Math.round(next.bounds[1]), + Math.round(next.bounds[2]), + Math.round(next.bounds[3]), + ]; + } + const collection = next.geometry || { type: 'FeatureCollection', features: [] }; + geometry.forEach((geo) => { + const i = collection.features.findIndex((item) => item.geometry.type === geo.geometry.type); + if (i >= 0) { + collection.features.splice(i, 1, geo); + } else { + collection.features.push(geo); + } + }); + if (collection.features.length) { + next.geometry = collection; + } + this.features[frame] = next; + if (next.keyframe) { + listInsert(this.featureIndex, frame); + } + return next; + } + + deleteFeature(frame: number) { + listRemove(this.featureIndex, frame); + delete this.features[frame]; + } + + serialize(): VisualMaskConfiguration { + const frames: Feature[] = []; + this.featureIndex.forEach((frame) => { + if (this.features[frame]) { + frames.push({ + ...this.features[frame], + frame, + keyframe: true, + }); + } + }); + return { + id: this.id, + name: this.name, + enabled: this.enabled, + type: this.type, + frames, + style: this.style, + }; + } +} + +export default class VisualMaskManager { + private styleManager: StyleManager; + + private markChangesPending: () => void; + + private syncConfiguration: (visualMasks: Record) => void; + + masksByCamera: Ref>; + + selectedMaskId: Ref; + + editingMaskId: Ref; + + editingMode: Ref; + + revisionCounter: Ref; + + hasMasks: Ref; + + constructor( + { + markChangesPending, + syncConfiguration, + styleManager, + }: { + markChangesPending: () => void; + syncConfiguration: (visualMasks: Record) => void; + styleManager: StyleManager; + }, + ) { + this.styleManager = styleManager; + this.markChangesPending = markChangesPending; + this.syncConfiguration = syncConfiguration; + this.masksByCamera = ref({}); + this.selectedMaskId = ref(null); + this.editingMaskId = ref(null); + this.editingMode = ref(false); + this.revisionCounter = ref(0); + this.hasMasks = computed(() => Object.values(this.masksByCamera.value) + .some((cameraMasks) => cameraMasks.length > 0)); + } + + private ensureCamera(camera: string) { + if (!this.masksByCamera.value[camera]) { + this.masksByCamera.value = { + ...this.masksByCamera.value, + [camera]: [], + }; + } + return this.masksByCamera.value[camera]; + } + + private syncStyleManager() { + const styles: Record = {}; + Object.values(this.masksByCamera.value).forEach((cameraMasks) => { + cameraMasks.forEach((mask) => { + styles[mask.styleKey] = normalizeStyle(mask.style); + }); + }); + this.styleManager.populateTypeStyles(styles); + } + + private commit(markPending = true) { + this.syncStyleManager(); + this.syncConfiguration(this.serialize()); + this.revisionCounter.value += 1; + if (markPending) { + this.markChangesPending(); + } + } + + load(visualMasks?: Record) { + const nextMasks: Record = {}; + Object.entries(visualMasks || {}).forEach(([camera, masks]) => { + nextMasks[camera] = masks.map((mask) => new VisualMask(mask)); + }); + this.masksByCamera.value = nextMasks; + this.selectedMaskId.value = null; + this.editingMaskId.value = null; + this.editingMode.value = false; + this.commit(false); + } + + serialize() { + const serialized: Record = {}; + Object.entries(this.masksByCamera.value).forEach(([camera, masks]) => { + if (masks.length) { + serialized[camera] = masks.map((mask) => mask.serialize()); + } + }); + return serialized; + } + + getMasks(camera: string) { + return this.masksByCamera.value[camera] || []; + } + + getMask(camera: string, id: number) { + return this.getMasks(camera).find((mask) => mask.id === id); + } + + getSelectedMask(camera: string) { + if (this.selectedMaskId.value === null) { + return undefined; + } + return this.getMask(camera, this.selectedMaskId.value); + } + + getEditingMask(camera: string) { + if (this.editingMaskId.value === null) { + return undefined; + } + return this.getMask(camera, this.editingMaskId.value); + } + + selectMask(id: number | null) { + this.selectedMaskId.value = id; + if (id === null) { + this.stopEditing(); + } + } + + clearSelection() { + this.selectedMaskId.value = null; + this.stopEditing(); + } + + startEditing(camera: string, id: number) { + const mask = this.getMask(camera, id); + if (!mask) { + return; + } + this.selectedMaskId.value = id; + this.editingMaskId.value = id; + this.editingMode.value = mask.type; + this.revisionCounter.value += 1; + } + + stopEditing() { + this.editingMaskId.value = null; + this.editingMode.value = false; + this.revisionCounter.value += 1; + } + + private getNextId() { + const ids = Object.values(this.masksByCamera.value) + .flatMap((masks) => masks.map((mask) => mask.id)); + if (!ids.length) { + return 0; + } + return Math.max(...ids) + 1; + } + + addMask(camera: string, type: VisualMaskGeometryType) { + const id = this.getNextId(); + const masks = this.ensureCamera(camera); + masks.push(new VisualMask({ + id, + name: `Mask ${id + 1}`, + type, + enabled: true, + frames: [], + style: DEFAULT_VISUAL_MASK_STYLE, + })); + this.selectedMaskId.value = id; + this.editingMaskId.value = id; + this.editingMode.value = type; + this.commit(); + return id; + } + + removeMask(camera: string, id: number) { + const masks = this.getMasks(camera); + const nextMasks = masks.filter((mask) => mask.id !== id); + this.masksByCamera.value = { + ...this.masksByCamera.value, + [camera]: nextMasks, + }; + if (this.selectedMaskId.value === id) { + this.selectedMaskId.value = null; + } + if (this.editingMaskId.value === id) { + this.stopEditing(); + } + this.commit(); + } + + renameMask(camera: string, id: number, name: string) { + const mask = this.getMask(camera, id); + if (!mask) { + return; + } + mask.name = name; + this.commit(); + } + + setMaskEnabled(camera: string, id: number, enabled: boolean) { + const mask = this.getMask(camera, id); + if (!mask) { + return; + } + mask.enabled = enabled; + this.commit(); + } + + setMaskStyle(camera: string, id: number, style: CustomStyle) { + const mask = this.getMask(camera, id); + if (!mask) { + return; + } + mask.style = normalizeStyle({ + ...mask.style, + ...style, + }); + this.commit(); + } + + isExactKeyframe(camera: string, id: number, frame: number) { + return !!this.getMask(camera, id)?.getExactFeature(frame); + } + + removeFrameChange(camera: string, id: number, frame: number) { + const mask = this.getMask(camera, id); + if (!mask) { + return; + } + if (!mask.getExactFeature(frame)) { + return; + } + mask.deleteFeature(frame); + this.commit(); + } + + updateRectBounds(camera: string, frame: number, bounds: RectBounds, id = this.editingMaskId.value) { + if (id === null) { + return; + } + const mask = this.getMask(camera, id); + if (!mask) { + return; + } + mask.type = 'rectangle'; + mask.setFeature({ + frame, + bounds, + keyframe: true, + }); + this.commit(); + } + + updateGeoJSON( + camera: string, + frame: number, + data: GeoJSON.Feature, + id = this.editingMaskId.value, + ) { + if (id === null) { + return; + } + const mask = this.getMask(camera, id); + if (!mask) { + return; + } + mask.type = 'Polygon'; + mask.setFeature({ + frame, + bounds: geojsonToBound(data), + keyframe: true, + }, [{ + type: 'Feature', + geometry: data.geometry, + properties: {}, + }]); + this.commit(); + } +} + +export { + DEFAULT_VISUAL_MASK_STYLE, + getVisualMaskStyleKey, + normalizeStyle, +}; diff --git a/server/dive_utils/models.py b/server/dive_utils/models.py index 9a59a54a..6bf19479 100644 --- a/server/dive_utils/models.py +++ b/server/dive_utils/models.py @@ -538,6 +538,15 @@ class CustomUISettings(BaseModel): width: Optional[int] +class VisualMask(BaseModel): + id: int + name: str + enabled: Optional[bool] + type: Literal['rectangle', 'Polygon'] + frames: List[Feature] = Field(default_factory=list) + style: Optional[CustomStyle] + + class DIVEConfiguration(BaseModel): general: Optional[GeneralSettings] UISettings: Optional[UISettings] @@ -546,6 +555,7 @@ class DIVEConfiguration(BaseModel): filterTimelines: Optional[List[FilterTimeline]] timelineConfigs: Optional[List[TimelineConfiguration]] customUI: Optional[CustomUISettings] + visualMasks: Optional[Dict[str, List[VisualMask]]] @validator('timelineConfigs', pre=True) @classmethod From 61280802e73bd3a940e3b9275f932638daccb701 Mon Sep 17 00:00:00 2001 From: Bryon Lewis Date: Mon, 11 May 2026 16:18:47 -0400 Subject: [PATCH 02/12] update config, remove polygon --- client/dive-common/components/EditorMenu.vue | 7 ++-- .../components/VisualMaskSidebar.vue | 18 ++------- client/src/components/LayerManager.vue | 27 +++++-------- client/src/visualMasks.spec.ts | 40 +++++++++++++++++++ client/src/visualMasks.ts | 33 +++++++++------ 5 files changed, 77 insertions(+), 48 deletions(-) diff --git a/client/dive-common/components/EditorMenu.vue b/client/dive-common/components/EditorMenu.vue index 0c267ba5..269a769a 100644 --- a/client/dive-common/components/EditorMenu.vue +++ b/client/dive-common/components/EditorMenu.vue @@ -7,7 +7,7 @@ import { Mousetrap, OverlayPreferences } from 'vue-media-annotator/types'; import { EditAnnotationTypes, VisibleAnnotationTypes } from 'vue-media-annotator/layers'; import Recipe from 'vue-media-annotator/recipe'; import { hexToRgb } from 'vue-media-annotator/utils'; -import { useMasks, useConfiguration } from 'vue-media-annotator/provides'; +import { useMasks, useConfiguration, useVisualMaskManager } from 'vue-media-annotator/provides'; import { useStore } from 'platform/web-girder/store/types'; import MaskTracking from 'dive-common/components/MaskTracking.vue'; @@ -81,6 +81,7 @@ export default defineComponent({ let toolTimeTimeout: number | undefined; const { editorOptions, editorFunctions } = useMasks(); const configMan = useConfiguration(); + const visualMaskManager = useVisualMaskManager(); const store = useStore(); const isOwnerAdmin = computed(() => { const currentUser = store.state.User.user as ({ @@ -260,7 +261,7 @@ export default defineComponent({ click: () => toggleVisible('attributeKey'), }); } - if (!isOwnerAdmin.value) { + if (!isOwnerAdmin.value || !visualMaskManager.hasMasks.value) { return buttons.filter((button) => button.id !== 'VisualMask'); } return buttons; @@ -269,7 +270,7 @@ export default defineComponent({ const isVisible = (mode: VisibleAnnotationTypes) => props.visibleModes.includes(mode); const toggleVisible = (mode: VisibleAnnotationTypes) => { - if (mode === 'VisualMask' && !isOwnerAdmin.value) { + if (mode === 'VisualMask' && (!isOwnerAdmin.value || !visualMaskManager.hasMasks.value)) { return; } if (isVisible(mode)) { diff --git a/client/dive-common/components/VisualMaskSidebar.vue b/client/dive-common/components/VisualMaskSidebar.vue index c6478b16..0f69f594 100644 --- a/client/dive-common/components/VisualMaskSidebar.vue +++ b/client/dive-common/components/VisualMaskSidebar.vue @@ -89,9 +89,9 @@ export default defineComponent({ ); }); - function addMask(type: 'rectangle' | 'Polygon') { + function addMask() { handler.trackSelect(null, false); - visualMaskManager.addMask(selectedCamera.value, type); + visualMaskManager.addMask(selectedCamera.value, 'rectangle'); showColorPicker.value = false; } @@ -179,25 +179,13 @@ export default defineComponent({ color="primary" class="mr-2" :disabled="!canEditMasks" - @click="addMask('rectangle')" + @click="addMask()" > mdi-vector-square Add Box - - - mdi-vector-polygon - - Add Polygon -
Camera: {{ selectedCamera }} diff --git a/client/src/components/LayerManager.vue b/client/src/components/LayerManager.vue index cbd4d1fb..87b08e1b 100644 --- a/client/src/components/LayerManager.vue +++ b/client/src/components/LayerManager.vue @@ -160,11 +160,6 @@ export default defineComponent({ stateStyling: visualMaskStyleManager.stateStyles, typeStyling: visualMaskStyleManager.typeStyling, }); - const visualMaskPolyLayer = new PolygonLayer({ - annotator, - stateStyling: visualMaskStyleManager.stateStyles, - typeStyling: visualMaskStyleManager.typeStyling, - }); const rectAnnotationLayer = new RectangleLayer({ annotator, stateStyling: trackStyleManager.stateStyles, @@ -349,13 +344,10 @@ export default defineComponent({ ); if (visibleModes.includes('VisualMask')) { - visualMaskRectLayer.setDrawingOther(['Polygon']); + visualMaskRectLayer.setDrawingOther([]); visualMaskRectLayer.changeData(visualMaskFrameData); - visualMaskPolyLayer.setDrawingOther(true); - visualMaskPolyLayer.changeData(visualMaskFrameData); } else { visualMaskRectLayer.disable(); - visualMaskPolyLayer.disable(); } if (visibleModes.includes('rectangle')) { @@ -480,11 +472,12 @@ export default defineComponent({ features: editingVisualMask.getFeature(frame), styleType: [editingVisualMask.styleKey, 1] as [string, number], }; - visualMaskEditLayer.setType(editingVisualMask.type); - visualMaskEditLayer.setKey(''); - visualMaskEditLayer.changeData([visualMaskFrame]); editAnnotationLayer.disable(); maskEditorLayer.disable(); + visualMaskEditLayer.setType('rectangle'); + visualMaskEditLayer.setKey(''); + visualMaskEditLayer.changeData([visualMaskFrame]); + annotator.setImageCursor('mdi-vector-rectangle'); rectAnnotationLayer.setDisableClicking(false); } else if (selectedTrackId !== null) { if ((editingTrack) && !(currentFrameIds || []).includes(selectedTrackId) @@ -698,8 +691,6 @@ export default defineComponent({ polyAnnotationLayer.bus.$on('annotation-right-clicked', Clicked); visualMaskRectLayer.bus.$on('annotation-clicked', (maskId: number | null) => visualMaskClicked(maskId, false)); visualMaskRectLayer.bus.$on('annotation-right-clicked', (maskId: number | null) => visualMaskClicked(maskId, true)); - visualMaskPolyLayer.bus.$on('annotation-clicked', (maskId: number | null) => visualMaskClicked(maskId, false)); - visualMaskPolyLayer.bus.$on('annotation-right-clicked', (maskId: number | null) => visualMaskClicked(maskId, true)); editAnnotationLayer.bus.$on('update:geojson', ( mode: 'in-progress' | 'editing', geometryCompleteEvent: boolean, @@ -738,7 +729,7 @@ export default defineComponent({ } }); visualMaskEditLayer.bus.$on('update:geojson', ( - _mode: 'in-progress' | 'editing', + mode: 'in-progress' | 'editing', geometryCompleteEvent: boolean, data: GeoJSON.Feature, type: string, @@ -748,13 +739,13 @@ export default defineComponent({ if (key) { // Visual masks store a single shape per frame and do not use keyed geometries. } + if (mode !== 'editing') { + return; + } if (type === 'rectangle') { const bounds = geojsonToBound(data as GeoJSON.Feature); cb(); visualMaskManager.updateRectBounds(props.camera, frameNumberRef.value, bounds); - } else if (type === 'Polygon') { - cb(); - visualMaskManager.updateGeoJSON(props.camera, frameNumberRef.value, data as GeoJSON.Feature); } if (geometryCompleteEvent) { updateLayers( diff --git a/client/src/visualMasks.spec.ts b/client/src/visualMasks.spec.ts index 7b6e94a3..072e83ef 100644 --- a/client/src/visualMasks.spec.ts +++ b/client/src/visualMasks.spec.ts @@ -24,6 +24,32 @@ describe('VisualMask', () => { expect(mask.getFeature(20)?.bounds).toEqual([5, 5, 15, 15]); expect(mask.getFeature(30)?.bounds).toEqual([5, 5, 15, 15]); }); + + it('normalizes polygon geometry with an empty key for editing', () => { + const mask = new VisualMask({ + id: 2, + name: 'Polygon Mask', + type: 'Polygon', + frames: [{ + frame: 10, + bounds: [0, 0, 10, 10], + keyframe: true, + geometry: { + type: 'FeatureCollection', + features: [{ + type: 'Feature', + geometry: { + type: 'Polygon', + coordinates: [[[0, 0], [10, 0], [10, 10], [0, 10], [0, 0]]], + }, + properties: {}, + }], + }, + }], + }); + + expect(mask.getFeature(10)?.geometry?.features[0]?.properties).toEqual({ key: '' }); + }); }); describe('VisualMaskManager', () => { @@ -62,4 +88,18 @@ describe('VisualMaskManager', () => { }], }); }); + + it('always creates rectangle visual masks', () => { + const styleManager = new StyleManager({ markChangesPending: () => {} }); + const manager = new VisualMaskManager({ + markChangesPending: () => {}, + styleManager, + syncConfiguration: () => {}, + }); + + const id = manager.addMask('singleCam', 'Polygon'); + + expect(manager.getMask('singleCam', id)?.type).toBe('rectangle'); + expect(manager.editingMode.value).toBe('rectangle'); + }); }); diff --git a/client/src/visualMasks.ts b/client/src/visualMasks.ts index 9e75f599..42dcd365 100644 --- a/client/src/visualMasks.ts +++ b/client/src/visualMasks.ts @@ -28,6 +28,18 @@ function normalizeStyle(style?: CustomStyle): CustomStyle { }; } +function normalizeGeometryFeature( + geometryFeature: GeoJSON.Feature, +): GeoJSON.Feature { + return { + ...geometryFeature, + properties: { + ...(geometryFeature.properties || {}), + key: geometryFeature.properties?.key ?? '', + }, + }; +} + export class VisualMask { id: number; @@ -132,11 +144,12 @@ export class VisualMask { } const collection = next.geometry || { type: 'FeatureCollection', features: [] }; geometry.forEach((geo) => { - const i = collection.features.findIndex((item) => item.geometry.type === geo.geometry.type); + const normalizedGeo = normalizeGeometryFeature(geo); + const i = collection.features.findIndex((item) => item.geometry.type === normalizedGeo.geometry.type); if (i >= 0) { - collection.features.splice(i, 1, geo); + collection.features.splice(i, 1, normalizedGeo); } else { - collection.features.push(geo); + collection.features.push(normalizedGeo); } }); if (collection.features.length) { @@ -329,20 +342,20 @@ export default class VisualMaskManager { return Math.max(...ids) + 1; } - addMask(camera: string, type: VisualMaskGeometryType) { + addMask(camera: string, _type: VisualMaskGeometryType = 'rectangle') { const id = this.getNextId(); const masks = this.ensureCamera(camera); masks.push(new VisualMask({ id, name: `Mask ${id + 1}`, - type, + type: 'rectangle', enabled: true, frames: [], style: DEFAULT_VISUAL_MASK_STYLE, })); this.selectedMaskId.value = id; this.editingMaskId.value = id; - this.editingMode.value = type; + this.editingMode.value = 'rectangle'; this.commit(); return id; } @@ -439,16 +452,12 @@ export default class VisualMaskManager { if (!mask) { return; } - mask.type = 'Polygon'; + mask.type = 'rectangle'; mask.setFeature({ frame, bounds: geojsonToBound(data), keyframe: true, - }, [{ - type: 'Feature', - geometry: data.geometry, - properties: {}, - }]); + }); this.commit(); } } From 5418cc491696ea9f4682eb74cf503f41cc904d05 Mon Sep 17 00:00:00 2001 From: Bryon Lewis Date: Mon, 11 May 2026 16:19:05 -0400 Subject: [PATCH 03/12] add documentation --- docs/Annotation-QuickStart.md | 6 +++ docs/UI-Configuration.md | 2 + docs/UI-Navigation-Editing-Bar.md | 1 + docs/UI-Visual-Masks.md | 71 +++++++++++++++++++++++++++++++ mkdocs.yml | 1 + 5 files changed, 81 insertions(+) create mode 100644 docs/UI-Visual-Masks.md diff --git a/docs/Annotation-QuickStart.md b/docs/Annotation-QuickStart.md index e99ee8d3..ab59236d 100644 --- a/docs/Annotation-QuickStart.md +++ b/docs/Annotation-QuickStart.md @@ -134,6 +134,12 @@ Every track is required to have a bounding box, but a polygon region may be adde ![Polygon Demo](videos/CreationModes/PolygonAnnotation.gif) +## Visual Masks + +Visual masks are configuration-backed rectangular overlays that can be managed by configuration owners and administrators. + +For the full workflow, permissions, and visibility behavior, see [Visual Masks](UI-Visual-Masks.md). + ## Segmentation Masks ![Polygon Demo](videos/CreationModes/SegmentationAnnotation.gif) diff --git a/docs/UI-Configuration.md b/docs/UI-Configuration.md index d129387b..38662d18 100644 --- a/docs/UI-Configuration.md +++ b/docs/UI-Configuration.md @@ -9,6 +9,8 @@ DIVE-DSA has the capability to specify configurations for datasets. Configurati Along with launch actions, special keyboard shortcuts can be configured to perform actions, like selecting tracks or going to specific frames in tracks. +Visual masks are also stored in the dataset configuration. They are owner/admin-managed rectangular overlays that persist until changed on a later frame. For usage details, see [Visual Masks](UI-Visual-Masks.md). + ## General ![General](images/Configuration//General/General.png) diff --git a/docs/UI-Navigation-Editing-Bar.md b/docs/UI-Navigation-Editing-Bar.md index 48e0a284..a86e9193 100644 --- a/docs/UI-Navigation-Editing-Bar.md +++ b/docs/UI-Navigation-Editing-Bar.md @@ -39,4 +39,5 @@ The **:material-eye: visibility** section contains toggle buttons that control t * ==:material-vector-line:== toggles **head/tail line** visibility * ==:material-format-text:== toggles annotation type & confidence **text** visibility * ==:material-comment-text-outline:== toggles a **cursor hover tooltip**, helpful for reviewing very dense scenes with lots of overlap. +* ==:material-image-filter-center-focus-strong:== toggles **visual mask** visibility when visual masks exist and the current user is a configuration owner or administrator. See [Visual Masks](UI-Visual-Masks.md). * ==:material-navigation:== toggles **track trail** visibility. The track trail is configurable to show up to 100 frames both ahead and behind each bounding box. The trail line is made of bounding box midpoints. diff --git a/docs/UI-Visual-Masks.md b/docs/UI-Visual-Masks.md new file mode 100644 index 00000000..614e5a35 --- /dev/null +++ b/docs/UI-Visual-Masks.md @@ -0,0 +1,71 @@ +# Visual Masks + +Visual masks are configuration-backed overlays that let dataset owners and administrators block out portions of the image or video view with styled rectangles. + +Unlike track annotations, visual masks are stored with the dataset configuration and are not tied to a track ID. They are intended for display and review workflows where a persistent masked region should appear on top of the media. + +## Permissions + +Visual masks are restricted to configuration owners and administrators. + +Standard users can still see existing visual masks in the annotation view, but they cannot: + +- create masks +- edit mask geometry +- rename masks +- change mask styling +- enable or disable masks + +## Where to Find Them + +When visual masks are enabled in the dataset configuration, owners and administrators can manage them from the **Visual Masks** context sidebar. + +If at least one visual mask exists for the current dataset, owners and administrators will also see a visibility toggle in the editing bar: + +- ==:material-image-filter-center-focus-strong:== toggles visual mask display + +This visibility button is hidden when no visual masks exist, and it is also hidden for non-owner/admin users. + +## Creating a Visual Mask + +Visual masks are currently **rectangle-only**. + +1. Open the **Visual Masks** context sidebar. +1. Click **Add Box**. +1. Move to the annotation view. The cursor will show the rectangle drawing icon when the tool is ready. +1. Click and drag to place the visual mask. +1. Save your changes using the standard save button in the navigation bar. + +## Editing a Visual Mask + +1. Select a visual mask from the **Visual Masks** sidebar. +1. Click **Edit Current Frame** or right-click the visual mask in the annotation view. +1. Drag the handles to resize the box or drag the center to move it. +1. Save when you are finished. + +Visual masks support frame-specific changes. If a mask is edited on a later frame, that change becomes a new keyframe for the mask. + +## Frame Behavior + +Visual masks persist until changed. + +- If a mask is defined on frame 10, it will continue to display on later frames. +- If the mask is changed on frame 25, that updated shape is used from frame 25 onward. +- The **Shape changes** list in the sidebar shows frames where the mask has an explicit change. + +## Styling + +Each visual mask can store its own style settings in the configuration: + +- color +- fill on/off +- opacity +- line thickness + +These settings are shared with the dataset configuration, so they persist across sessions just like other configuration-driven UI settings. + +## Current Limitations + +- Only rectangle visual masks are supported currently. +- Visual masks are configured per camera. +- Visual masks are stored in configuration data, not in the annotation track data. diff --git a/mkdocs.yml b/mkdocs.yml index 4605fa17..6ddd2cbc 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -56,6 +56,7 @@ nav: - Introduction: Annotation-User-Interface-Overview.md - Navigation and Editing Bar: UI-Navigation-Editing-Bar.md - Annotation Window: UI-Annotation-View.md + - Visual Masks: UI-Visual-Masks.md - Type List: UI-Type-List.md - Track List: UI-Track-List.md - Timeline: UI-Timeline.md From aa92b3204fd554ea5601e74886854e6ac3c10abc Mon Sep 17 00:00:00 2001 From: Bryon Lewis Date: Mon, 11 May 2026 16:25:29 -0400 Subject: [PATCH 04/12] remove polygon references --- client/dive-common/components/DatasetInfoAttributes.vue | 2 +- client/dive-common/components/VisualMaskSidebar.vue | 2 +- client/src/visualMasks.spec.ts | 2 +- client/src/visualMasks.ts | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/client/dive-common/components/DatasetInfoAttributes.vue b/client/dive-common/components/DatasetInfoAttributes.vue index 89ee174a..07478de0 100644 --- a/client/dive-common/components/DatasetInfoAttributes.vue +++ b/client/dive-common/components/DatasetInfoAttributes.vue @@ -230,8 +230,8 @@ export default defineComponent({ small class="ml-2" v-bind="attrs" - v-on="on" aria-label="Detection attribute settings" + v-on="on" @click.stop > diff --git a/client/dive-common/components/VisualMaskSidebar.vue b/client/dive-common/components/VisualMaskSidebar.vue index 0f69f594..61036020 100644 --- a/client/dive-common/components/VisualMaskSidebar.vue +++ b/client/dive-common/components/VisualMaskSidebar.vue @@ -91,7 +91,7 @@ export default defineComponent({ function addMask() { handler.trackSelect(null, false); - visualMaskManager.addMask(selectedCamera.value, 'rectangle'); + visualMaskManager.addMask(selectedCamera.value); showColorPicker.value = false; } diff --git a/client/src/visualMasks.spec.ts b/client/src/visualMasks.spec.ts index 072e83ef..da8714e0 100644 --- a/client/src/visualMasks.spec.ts +++ b/client/src/visualMasks.spec.ts @@ -64,7 +64,7 @@ describe('VisualMaskManager', () => { }, }); - const id = manager.addMask('singleCam', 'rectangle'); + const id = manager.addMask('singleCam'); manager.setMaskStyle('singleCam', id, { color: '#ff0000', opacity: 0.5 }); manager.updateRectBounds('singleCam', 12, [1, 2, 3, 4], id); diff --git a/client/src/visualMasks.ts b/client/src/visualMasks.ts index 42dcd365..fca038df 100644 --- a/client/src/visualMasks.ts +++ b/client/src/visualMasks.ts @@ -342,7 +342,7 @@ export default class VisualMaskManager { return Math.max(...ids) + 1; } - addMask(camera: string, _type: VisualMaskGeometryType = 'rectangle') { + addMask(camera: string) { const id = this.getNextId(); const masks = this.ensureCamera(camera); masks.push(new VisualMask({ From 04b9c460a891579610f1737ca4952cc08dd706c5 Mon Sep 17 00:00:00 2001 From: Bryon Lewis Date: Tue, 12 May 2026 07:21:23 -0400 Subject: [PATCH 05/12] remove polygon test --- client/src/visualMasks.spec.ts | 27 --------------------------- 1 file changed, 27 deletions(-) diff --git a/client/src/visualMasks.spec.ts b/client/src/visualMasks.spec.ts index da8714e0..61396180 100644 --- a/client/src/visualMasks.spec.ts +++ b/client/src/visualMasks.spec.ts @@ -25,33 +25,6 @@ describe('VisualMask', () => { expect(mask.getFeature(30)?.bounds).toEqual([5, 5, 15, 15]); }); - it('normalizes polygon geometry with an empty key for editing', () => { - const mask = new VisualMask({ - id: 2, - name: 'Polygon Mask', - type: 'Polygon', - frames: [{ - frame: 10, - bounds: [0, 0, 10, 10], - keyframe: true, - geometry: { - type: 'FeatureCollection', - features: [{ - type: 'Feature', - geometry: { - type: 'Polygon', - coordinates: [[[0, 0], [10, 0], [10, 10], [0, 10], [0, 0]]], - }, - properties: {}, - }], - }, - }], - }); - - expect(mask.getFeature(10)?.geometry?.features[0]?.properties).toEqual({ key: '' }); - }); -}); - describe('VisualMaskManager', () => { it('serializes per-camera visual masks with their styles', () => { const styleManager = new StyleManager({ markChangesPending: () => {} }); From 01ab6b6b63c93a8d504b5fca194a7eb96c7335d6 Mon Sep 17 00:00:00 2001 From: Bryon Lewis Date: Tue, 12 May 2026 07:43:52 -0400 Subject: [PATCH 06/12] unify config owner/admin testing to configurationManager --- .../components/ConfigurationEditor.vue | 20 ++++--------- client/dive-common/components/EditorMenu.vue | 15 +--------- client/dive-common/components/Viewer.vue | 15 +--------- .../components/VisualMaskSidebar.vue | 28 +++---------------- client/src/ConfigurationManager.ts | 22 ++++++++++++++- 5 files changed, 32 insertions(+), 68 deletions(-) diff --git a/client/dive-common/components/ConfigurationEditor.vue b/client/dive-common/components/ConfigurationEditor.vue index 6fef9d16..91c6182f 100644 --- a/client/dive-common/components/ConfigurationEditor.vue +++ b/client/dive-common/components/ConfigurationEditor.vue @@ -41,21 +41,11 @@ export default defineComponent({ const getUISetting = (key: UISettingsKey) => (configMan.getUISetting(key)); const girderRest = useGirderRest(); const isAdminOwner = computed(() => { - let ownerAdmin = false; - if (girderRest.user) { - ownerAdmin = girderRest.user.admin; - } - const id = girderRest.user._id; - const groups = girderRest.user.groups as string[]; - if (configMan.configOwners.value.users.findIndex((item) => item.id === id) !== -1) { - ownerAdmin = true; - } - groups.forEach((group) => { - if (configMan.configOwners.value.groups.findIndex((item) => item.id === group) !== -1) { - ownerAdmin = true; - } - }); - return ownerAdmin; + return configMan.isConfigOwnerAdmin(girderRest.user as ({ + admin?: boolean; + _id?: string; + groups?: string[]; + } | null)); }); const hasConfig = computed(() => !!configMan.configuration.value); const menuOpen = ref(false); diff --git a/client/dive-common/components/EditorMenu.vue b/client/dive-common/components/EditorMenu.vue index 269a769a..e45ab595 100644 --- a/client/dive-common/components/EditorMenu.vue +++ b/client/dive-common/components/EditorMenu.vue @@ -89,20 +89,7 @@ export default defineComponent({ _id?: string; groups?: string[]; } | null); - if (!currentUser) { - return false; - } - let ownerAdmin = !!currentUser.admin; - if (configMan.configOwners.value.users - .findIndex((item) => item.id === currentUser._id) !== -1) { - ownerAdmin = true; - } - (currentUser.groups || []).forEach((group: string) => { - if (configMan.configOwners.value.groups.findIndex((item) => item.id === group) !== -1) { - ownerAdmin = true; - } - }); - return ownerAdmin; + return configMan.isConfigOwnerAdmin(currentUser); }); const modeToolTips = { Creating: { diff --git a/client/dive-common/components/Viewer.vue b/client/dive-common/components/Viewer.vue index c1ba79c4..a4002f32 100644 --- a/client/dive-common/components/Viewer.vue +++ b/client/dive-common/components/Viewer.vue @@ -180,20 +180,7 @@ export default defineComponent({ _id?: string; groups?: string[]; } | null); - if (!currentUser) { - return false; - } - let ownerAdmin = !!currentUser.admin; - if (configurationManager.configOwners.value.users - .findIndex((item) => item.id === currentUser._id) !== -1) { - ownerAdmin = true; - } - (currentUser.groups || []).forEach((group: string) => { - if (configurationManager.configOwners.value.groups.findIndex((item) => item.id === group) !== -1) { - ownerAdmin = true; - } - }); - return ownerAdmin; + return configurationManager.isConfigOwnerAdmin(currentUser); }); const { diff --git a/client/dive-common/components/VisualMaskSidebar.vue b/client/dive-common/components/VisualMaskSidebar.vue index 61036020..bb0c137b 100644 --- a/client/dive-common/components/VisualMaskSidebar.vue +++ b/client/dive-common/components/VisualMaskSidebar.vue @@ -36,21 +36,7 @@ export default defineComponent({ _id?: string; groups?: string[]; } | null); - let ownerAdmin = false; - if (currentUser) { - ownerAdmin = !!currentUser.admin; - } - const userId = currentUser?._id; - const groups = currentUser?.groups || []; - if (userId && configMan.configOwners.value.users.findIndex((item) => item.id === userId) !== -1) { - ownerAdmin = true; - } - groups.forEach((group) => { - if (configMan.configOwners.value.groups.findIndex((item) => item.id === group) !== -1) { - ownerAdmin = true; - } - }); - return ownerAdmin; + return configMan.isConfigOwnerAdmin(currentUser); }); const canEditMasks = computed(() => isOwnerAdmin.value && !readOnlyMode.value); @@ -161,15 +147,9 @@ export default defineComponent({
- - Visual masks are viewable here, but only configuration owners/admins can edit them. - +
+ Visual mask changes are stored in the dataset configuration and saved with the standard save button. +
Add Visual Mask
diff --git a/client/src/ConfigurationManager.ts b/client/src/ConfigurationManager.ts index 55c4f1e4..8a049afb 100644 --- a/client/src/ConfigurationManager.ts +++ b/client/src/ConfigurationManager.ts @@ -142,7 +142,13 @@ export interface CustomUISettings { width? : number; } -export type VisualMaskGeometryType = 'rectangle' | 'Polygon'; +export interface ConfigurationUser { + admin?: boolean; + _id?: string; + groups?: string[]; +} + +export type VisualMaskGeometryType = 'rectangle'; export interface VisualMaskConfiguration { id: number; @@ -258,6 +264,20 @@ export default class ConfigurationManager { this.configOwners.value = data; } + isConfigOwnerAdmin(user: ConfigurationUser | null | undefined) { + if (!user) { + return false; + } + if (user.admin) { + return true; + } + if (user._id && this.configOwners.value.users.some((item) => item.id === user._id)) { + return true; + } + return (user.groups || []).some((group) => this.configOwners.value.groups + .some((item) => item.id === group)); + } + setPrevNext(data: DiveConfiguration['prevNext']) { this.prevNext.value = data; } From 47bba1528de192e0d7029753003697ecb03e05f8 Mon Sep 17 00:00:00 2001 From: Bryon Lewis Date: Tue, 12 May 2026 07:44:18 -0400 Subject: [PATCH 07/12] remove polygon references --- client/src/visualMasks.spec.ts | 3 ++- server/dive_utils/models.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/client/src/visualMasks.spec.ts b/client/src/visualMasks.spec.ts index 61396180..16069421 100644 --- a/client/src/visualMasks.spec.ts +++ b/client/src/visualMasks.spec.ts @@ -24,6 +24,7 @@ describe('VisualMask', () => { expect(mask.getFeature(20)?.bounds).toEqual([5, 5, 15, 15]); expect(mask.getFeature(30)?.bounds).toEqual([5, 5, 15, 15]); }); +}); describe('VisualMaskManager', () => { it('serializes per-camera visual masks with their styles', () => { @@ -70,7 +71,7 @@ describe('VisualMaskManager', () => { syncConfiguration: () => {}, }); - const id = manager.addMask('singleCam', 'Polygon'); + const id = manager.addMask('singleCam'); expect(manager.getMask('singleCam', id)?.type).toBe('rectangle'); expect(manager.editingMode.value).toBe('rectangle'); diff --git a/server/dive_utils/models.py b/server/dive_utils/models.py index 6bf19479..92998410 100644 --- a/server/dive_utils/models.py +++ b/server/dive_utils/models.py @@ -542,7 +542,7 @@ class VisualMask(BaseModel): id: int name: str enabled: Optional[bool] - type: Literal['rectangle', 'Polygon'] + type: Literal['rectangle'] frames: List[Feature] = Field(default_factory=list) style: Optional[CustomStyle] From a5824e10493e2fe6fee2790bb1aac1ac3515b60a Mon Sep 17 00:00:00 2001 From: Bryon Lewis Date: Tue, 12 May 2026 07:44:28 -0400 Subject: [PATCH 08/12] update documentation --- docs/UI-Visual-Masks.md | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/docs/UI-Visual-Masks.md b/docs/UI-Visual-Masks.md index 614e5a35..bf7148a2 100644 --- a/docs/UI-Visual-Masks.md +++ b/docs/UI-Visual-Masks.md @@ -6,21 +6,15 @@ Unlike track annotations, visual masks are stored with the dataset configuration ## Permissions -Visual masks are restricted to configuration owners and administrators. +Visual mask management is restricted to configuration owners and administrators. -Standard users can still see existing visual masks in the annotation view, but they cannot: - -- create masks -- edit mask geometry -- rename masks -- change mask styling -- enable or disable masks +Non-owner/admin users do not get the **Visual Masks** sidebar or the visual-mask visibility toggle in the editing bar. Existing masks may still appear in the annotation view when they are enabled in the current configuration. ## Where to Find Them -When visual masks are enabled in the dataset configuration, owners and administrators can manage them from the **Visual Masks** context sidebar. +When visual masks are enabled in the dataset configuration, configuration owners and administrators can manage them from the **Visual Masks** context sidebar. -If at least one visual mask exists for the current dataset, owners and administrators will also see a visibility toggle in the editing bar: +If at least one visual mask exists for the current dataset, configuration owners and administrators will also see a visibility toggle in the editing bar: - ==:material-image-filter-center-focus-strong:== toggles visual mask display From ffc02640b1344c0d83e6dea0fe1cb8d663f0485c Mon Sep 17 00:00:00 2001 From: Bryon Lewis Date: Tue, 12 May 2026 08:41:28 -0400 Subject: [PATCH 09/12] default opacity --- client/src/visualMasks.spec.ts | 11 +++++++++++ client/src/visualMasks.ts | 2 +- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/client/src/visualMasks.spec.ts b/client/src/visualMasks.spec.ts index 16069421..93d0d2d5 100644 --- a/client/src/visualMasks.spec.ts +++ b/client/src/visualMasks.spec.ts @@ -24,6 +24,17 @@ describe('VisualMask', () => { expect(mask.getFeature(20)?.bounds).toEqual([5, 5, 15, 15]); expect(mask.getFeature(30)?.bounds).toEqual([5, 5, 15, 15]); }); + + it('defaults style opacity to fully opaque', () => { + const mask = new VisualMask({ + id: 2, + name: 'Opaque Mask', + type: 'rectangle', + frames: [], + }); + + expect(mask.style.opacity).toBe(1); + }); }); describe('VisualMaskManager', () => { diff --git a/client/src/visualMasks.ts b/client/src/visualMasks.ts index fca038df..2a9a7bab 100644 --- a/client/src/visualMasks.ts +++ b/client/src/visualMasks.ts @@ -13,7 +13,7 @@ import type { const DEFAULT_VISUAL_MASK_STYLE: CustomStyle = { color: '#000000', fill: true, - opacity: 0.35, + opacity: 1, strokeWidth: 3, }; From e69a10e6de6b9c4dee0c9519a4110355d55a710c Mon Sep 17 00:00:00 2001 From: Bryon Lewis Date: Tue, 12 May 2026 08:54:29 -0400 Subject: [PATCH 10/12] update visual mask documentation --- docs/UI-Visual-Masks.md | 48 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/docs/UI-Visual-Masks.md b/docs/UI-Visual-Masks.md index bf7148a2..b36dedea 100644 --- a/docs/UI-Visual-Masks.md +++ b/docs/UI-Visual-Masks.md @@ -58,6 +58,54 @@ Each visual mask can store its own style settings in the configuration: These settings are shared with the dataset configuration, so they persist across sessions just like other configuration-driven UI settings. +## Configuration Format + +Visual masks live under the top-level `visualMasks` key in the dataset configuration. The value is an object keyed by camera name, where each camera contains an array of masks. + +Example: + +```json +{ + "visualMasks": { + "cam1": [ + { + "id": 0, + "name": "Doorway Mask", + "enabled": true, + "type": "rectangle", + "style": { + "color": "#000000", + "fill": true, + "opacity": 1, + "strokeWidth": 3 + }, + "frames": [ + { + "frame": 10, + "bounds": [120, 80, 420, 260], + "keyframe": true + }, + { + "frame": 25, + "bounds": [140, 80, 420, 260], + "keyframe": true + } + ] + } + ] + } +} +``` + +Quick notes: + +- `visualMasks.cam1` can be replaced with any camera name from the dataset configuration. +- Each mask needs an `id`, `name`, `type`, and `frames` array. +- `type` is currently always `"rectangle"`. +- `frames` stores the explicit shape changes for that mask. The mask remains in effect until a later frame entry changes it. +- Each frame entry uses `bounds: [x1, y1, x2, y2]` in image/display coordinates. +- `style` is optional. If omitted, the application fills in defaults such as black, filled, and fully opaque (`opacity: 1`). + ## Current Limitations - Only rectangle visual masks are supported currently. From bec19eb0d330ceda2f4b82fbdaef84f349f21580 Mon Sep 17 00:00:00 2001 From: Bryon Lewis Date: Tue, 12 May 2026 10:31:05 -0400 Subject: [PATCH 11/12] add relative support for masking --- .../components/ConfigurationEditor.vue | 6 +- .../components/VisualMaskSidebar.vue | 93 +++++++----- client/src/ConfigurationManager.ts | 1 + client/src/components/LayerManager.vue | 13 +- client/src/visualMasks.spec.ts | 99 ++++++++++++- client/src/visualMasks.ts | 136 ++++++++++++++++-- docs/UI-Visual-Masks.md | 23 ++- server/dive_utils/models.py | 1 + 8 files changed, 313 insertions(+), 59 deletions(-) diff --git a/client/dive-common/components/ConfigurationEditor.vue b/client/dive-common/components/ConfigurationEditor.vue index 91c6182f..71dabb23 100644 --- a/client/dive-common/components/ConfigurationEditor.vue +++ b/client/dive-common/components/ConfigurationEditor.vue @@ -40,13 +40,11 @@ export default defineComponent({ const configMan = useConfiguration(); const getUISetting = (key: UISettingsKey) => (configMan.getUISetting(key)); const girderRest = useGirderRest(); - const isAdminOwner = computed(() => { - return configMan.isConfigOwnerAdmin(girderRest.user as ({ + const isAdminOwner = computed(() => configMan.isConfigOwnerAdmin(girderRest.user as ({ admin?: boolean; _id?: string; groups?: string[]; - } | null)); - }); + } | null))); const hasConfig = computed(() => !!configMan.configuration.value); const menuOpen = ref(false); const additive = ref(false); diff --git a/client/dive-common/components/VisualMaskSidebar.vue b/client/dive-common/components/VisualMaskSidebar.vue index bb0c137b..02613502 100644 --- a/client/dive-common/components/VisualMaskSidebar.vue +++ b/client/dive-common/components/VisualMaskSidebar.vue @@ -11,6 +11,7 @@ import { useTime, useVisualMaskManager, } from 'vue-media-annotator/provides'; +import { injectAggregateController } from 'vue-media-annotator/components/annotators/useMediaController'; export default defineComponent({ name: 'VisualMaskSidebar', @@ -22,6 +23,7 @@ export default defineComponent({ }, setup() { const configMan = useConfiguration(); + const aggregateController = injectAggregateController().value; const handler = useHandler(); const readOnlyMode = useReadOnlyMode(); const selectedCamera = useSelectedCamera(); @@ -41,6 +43,9 @@ export default defineComponent({ const canEditMasks = computed(() => isOwnerAdmin.value && !readOnlyMode.value); const currentFrame = computed(() => time.frame.value); + const selectedCameraFrameSize = computed(() => ( + aggregateController.getController(selectedCamera.value).frameSize.value + )); const cameraMasks = computed(() => { const revision = visualMaskManager.revisionCounter.value; if (revision >= 0) { @@ -104,11 +109,24 @@ export default defineComponent({ }); } + function setRelativePositioning(value: boolean) { + if (!selectedMask.value) { + return; + } + visualMaskManager.setMaskRelativePositioning( + selectedCamera.value, + selectedMask.value.id, + value, + selectedCameraFrameSize.value, + ); + } + return { addMask, cameraMasks, canEditMasks, currentFrame, + selectedCameraFrameSize, editingMaskId: visualMaskManager.editingMaskId, isEditingSelected, isExactKeyframe, @@ -120,6 +138,7 @@ export default defineComponent({ selectMask, showColorPicker, toggleEditing, + setRelativePositioning, updateStyle, visualMaskManager, seekFrame: handler.seekFrame, @@ -190,16 +209,30 @@ export default defineComponent({ - + top + > + + Visibility + -
- - {{ selectedMask.type }} - - - {{ isEditingSelected ? 'editing' : 'selected' }} - - - frame {{ currentFrame }} - - - {{ isExactKeyframe ? 'shape changes here' : 'inherits previous shape' }} - + +
+ Store mask bounds as a percentage of the video size so the mask scales across videos with different resolutions.
diff --git a/client/src/ConfigurationManager.ts b/client/src/ConfigurationManager.ts index 8a049afb..957f20c1 100644 --- a/client/src/ConfigurationManager.ts +++ b/client/src/ConfigurationManager.ts @@ -154,6 +154,7 @@ export interface VisualMaskConfiguration { id: number; name: string; enabled?: boolean; + useRelativePositioning?: boolean; type: VisualMaskGeometryType; frames: Feature[]; style?: CustomStyle; diff --git a/client/src/components/LayerManager.vue b/client/src/components/LayerManager.vue index 87b08e1b..efad9d75 100644 --- a/client/src/components/LayerManager.vue +++ b/client/src/components/LayerManager.vue @@ -275,11 +275,12 @@ export default defineComponent({ const frameData = [] as FrameDataTrack[]; const editingTracks = [] as FrameDataTrack[]; const visualMaskFrameData = [] as FrameDataTrack[]; + const frameSize = annotator.frameSize.value; visualMaskManager.getMasks(props.camera).forEach((visualMask) => { if (!visualMask.enabled) { return; } - const features = visualMask.getFeature(frame); + const features = visualMask.getFeature(frame, frameSize); if (!features) { return; } @@ -469,7 +470,7 @@ export default defineComponent({ editing: true, track: editingVisualMask as never, groups: [], - features: editingVisualMask.getFeature(frame), + features: editingVisualMask.getFeature(frame, frameSize), styleType: [editingVisualMask.styleKey, 1] as [string, number], }; editAnnotationLayer.disable(); @@ -745,7 +746,13 @@ export default defineComponent({ if (type === 'rectangle') { const bounds = geojsonToBound(data as GeoJSON.Feature); cb(); - visualMaskManager.updateRectBounds(props.camera, frameNumberRef.value, bounds); + visualMaskManager.updateRectBounds( + props.camera, + frameNumberRef.value, + bounds, + undefined, + annotator.frameSize.value, + ); } if (geometryCompleteEvent) { updateLayers( diff --git a/client/src/visualMasks.spec.ts b/client/src/visualMasks.spec.ts index 93d0d2d5..c38347ea 100644 --- a/client/src/visualMasks.spec.ts +++ b/client/src/visualMasks.spec.ts @@ -35,10 +35,26 @@ describe('VisualMask', () => { expect(mask.style.opacity).toBe(1); }); + + it('converts relative mask bounds to the current frame size for display', () => { + const mask = new VisualMask({ + id: 3, + name: 'Relative Mask', + type: 'rectangle', + useRelativePositioning: true, + frames: [{ + frame: 10, + bounds: [10, 20, 30, 40], + keyframe: true, + }], + }); + + expect(mask.getFeature(10, [200, 50])?.bounds).toEqual([20, 10, 60, 20]); + }); }); describe('VisualMaskManager', () => { - it('serializes per-camera visual masks with their styles', () => { + it('serializes per-camera visual masks with relative bounds by default', () => { const styleManager = new StyleManager({ markChangesPending: () => {} }); let serializedMasks = {}; const manager = new VisualMaskManager({ @@ -51,17 +67,18 @@ describe('VisualMaskManager', () => { const id = manager.addMask('singleCam'); manager.setMaskStyle('singleCam', id, { color: '#ff0000', opacity: 0.5 }); - manager.updateRectBounds('singleCam', 12, [1, 2, 3, 4], id); + manager.updateRectBounds('singleCam', 12, [10, 20, 30, 40], id, [200, 200]); expect(serializedMasks).toEqual({ singleCam: [{ id, name: 'Mask 1', enabled: true, + useRelativePositioning: true, type: 'rectangle', frames: [{ frame: 12, - bounds: [1, 2, 3, 4], + bounds: [5, 10, 15, 20], keyframe: true, }], style: { @@ -74,6 +91,81 @@ describe('VisualMaskManager', () => { }); }); + it('converts stored bounds when toggling relative positioning', () => { + const styleManager = new StyleManager({ markChangesPending: () => {} }); + let serializedMasks = {}; + const manager = new VisualMaskManager({ + markChangesPending: () => {}, + styleManager, + syncConfiguration: (visualMasks) => { + serializedMasks = visualMasks; + }, + }); + + const id = manager.addMask('singleCam'); + manager.updateRectBounds('singleCam', 12, [10, 20, 30, 40], id, [200, 200]); + manager.setMaskRelativePositioning('singleCam', id, false, [200, 200]); + + expect(serializedMasks).toEqual({ + singleCam: [{ + id, + name: 'Mask 1', + enabled: true, + useRelativePositioning: false, + type: 'rectangle', + frames: [{ + frame: 12, + bounds: [10, 20, 30, 40], + keyframe: true, + }], + style: { + color: '#000000', + fill: true, + opacity: 1, + strokeWidth: 3, + }, + }], + }); + }); + + it('converts absolute masks to percentage bounds when enabling relative positioning', () => { + const styleManager = new StyleManager({ markChangesPending: () => {} }); + let serializedMasks = {}; + const manager = new VisualMaskManager({ + markChangesPending: () => {}, + styleManager, + syncConfiguration: (visualMasks) => { + serializedMasks = visualMasks; + }, + }); + + const id = manager.addMask('singleCam'); + manager.setMaskRelativePositioning('singleCam', id, false, [200, 200]); + manager.updateRectBounds('singleCam', 12, [10, 20, 30, 40], id, [200, 200]); + manager.setMaskRelativePositioning('singleCam', id, true, [200, 200]); + + expect(serializedMasks).toEqual({ + singleCam: [{ + id, + name: 'Mask 1', + enabled: true, + useRelativePositioning: true, + type: 'rectangle', + frames: [{ + frame: 12, + bounds: [5, 10, 15, 20], + keyframe: true, + }], + style: { + color: '#000000', + fill: true, + opacity: 1, + strokeWidth: 3, + }, + }], + }); + }); + it('always creates rectangle visual masks', () => { const styleManager = new StyleManager({ markChangesPending: () => {} }); const manager = new VisualMaskManager({ @@ -85,6 +177,7 @@ describe('VisualMaskManager', () => { const id = manager.addMask('singleCam'); expect(manager.getMask('singleCam', id)?.type).toBe('rectangle'); + expect(manager.getMask('singleCam', id)?.useRelativePositioning).toBe(true); expect(manager.editingMode.value).toBe('rectangle'); }); }); diff --git a/client/src/visualMasks.ts b/client/src/visualMasks.ts index 2a9a7bab..598533a0 100644 --- a/client/src/visualMasks.ts +++ b/client/src/visualMasks.ts @@ -17,6 +17,8 @@ const DEFAULT_VISUAL_MASK_STYLE: CustomStyle = { strokeWidth: 3, }; +const RELATIVE_POSITION_PERCENT_SCALE = 100; + function getVisualMaskStyleKey(id: number) { return `visual-mask-${id}`; } @@ -28,6 +30,64 @@ function normalizeStyle(style?: CustomStyle): CustomStyle { }; } +function normalizeRelativeValue(value: number) { + return Number(value.toFixed(6)); +} + +function hasFrameSize(frameSize?: readonly [number, number]) { + return !!frameSize && frameSize[0] > 0 && frameSize[1] > 0; +} + +function convertBoundsToRelative( + bounds: RectBounds, + frameSize?: readonly [number, number], +): RectBounds { + if (!hasFrameSize(frameSize)) { + return [...bounds] as RectBounds; + } + const width = frameSize![0]; + const height = frameSize![1]; + return [ + normalizeRelativeValue((bounds[0] / width) * RELATIVE_POSITION_PERCENT_SCALE), + normalizeRelativeValue((bounds[1] / height) * RELATIVE_POSITION_PERCENT_SCALE), + normalizeRelativeValue((bounds[2] / width) * RELATIVE_POSITION_PERCENT_SCALE), + normalizeRelativeValue((bounds[3] / height) * RELATIVE_POSITION_PERCENT_SCALE), + ]; +} + +function convertBoundsToAbsolute( + bounds: RectBounds, + frameSize?: readonly [number, number], +): RectBounds { + if (!hasFrameSize(frameSize)) { + return [...bounds] as RectBounds; + } + const width = frameSize![0]; + const height = frameSize![1]; + return [ + Math.round((bounds[0] / RELATIVE_POSITION_PERCENT_SCALE) * width), + Math.round((bounds[1] / RELATIVE_POSITION_PERCENT_SCALE) * height), + Math.round((bounds[2] / RELATIVE_POSITION_PERCENT_SCALE) * width), + Math.round((bounds[3] / RELATIVE_POSITION_PERCENT_SCALE) * height), + ]; +} + +function normalizeStoredBounds( + bounds: RectBounds, + useRelativePositioning: boolean, + frameSize?: readonly [number, number], +): RectBounds { + if (useRelativePositioning) { + return convertBoundsToRelative(bounds, frameSize); + } + return [ + Math.round(bounds[0]), + Math.round(bounds[1]), + Math.round(bounds[2]), + Math.round(bounds[3]), + ]; +} + function normalizeGeometryFeature( geometryFeature: GeoJSON.Feature, ): GeoJSON.Feature { @@ -47,6 +107,8 @@ export class VisualMask { enabled: boolean; + useRelativePositioning: boolean; + type: VisualMaskGeometryType; features: Feature[]; @@ -59,6 +121,7 @@ export class VisualMask { id, name, enabled = true, + useRelativePositioning = false, type, frames, style, @@ -66,6 +129,7 @@ export class VisualMask { this.id = id; this.name = name; this.enabled = enabled; + this.useRelativePositioning = useRelativePositioning; this.type = type; this.features = []; this.featureIndex = []; @@ -87,9 +151,15 @@ export class VisualMask { return this.features[frame] || null; } - getFeature(frame: number) { + getFeature(frame: number, frameSize?: readonly [number, number]) { const exact = this.getExactFeature(frame); if (exact) { + if (exact.bounds && this.useRelativePositioning) { + return { + ...exact, + bounds: convertBoundsToAbsolute(exact.bounds, frameSize), + }; + } return exact; } if (!this.featureIndex.length) { @@ -103,11 +173,15 @@ export class VisualMask { if (!source) { return null; } - return { + const nextFeature: Feature = { ...source, frame, keyframe: false, }; + if (nextFeature.bounds && this.useRelativePositioning) { + nextFeature.bounds = convertBoundsToAbsolute(nextFeature.bounds, frameSize); + } + return nextFeature; } getNextKeyframe(frame: number) { @@ -126,6 +200,7 @@ export class VisualMask { feature: Feature, geometry: GeoJSON.Feature[] = [], ensureKeyframe = true, + frameSize?: readonly [number, number], ) { const { frame } = feature; const current = this.features[frame] || { frame }; @@ -135,12 +210,7 @@ export class VisualMask { keyframe: ensureKeyframe ? true : feature.keyframe, }; if (next.bounds) { - next.bounds = [ - Math.round(next.bounds[0]), - Math.round(next.bounds[1]), - Math.round(next.bounds[2]), - Math.round(next.bounds[3]), - ]; + next.bounds = normalizeStoredBounds(next.bounds, this.useRelativePositioning, frameSize); } const collection = next.geometry || { type: 'FeatureCollection', features: [] }; geometry.forEach((geo) => { @@ -167,6 +237,25 @@ export class VisualMask { delete this.features[frame]; } + setUseRelativePositioning( + useRelativePositioning: boolean, + frameSize?: readonly [number, number], + ) { + if (this.useRelativePositioning === useRelativePositioning) { + return; + } + this.featureIndex.forEach((frame) => { + const feature = this.features[frame]; + if (!feature?.bounds) { + return; + } + feature.bounds = useRelativePositioning + ? convertBoundsToRelative(feature.bounds, frameSize) + : convertBoundsToAbsolute(feature.bounds, frameSize); + }); + this.useRelativePositioning = useRelativePositioning; + } + serialize(): VisualMaskConfiguration { const frames: Feature[] = []; this.featureIndex.forEach((frame) => { @@ -182,6 +271,7 @@ export class VisualMask { id: this.id, name: this.name, enabled: this.enabled, + useRelativePositioning: this.useRelativePositioning, type: this.type, frames, style: this.style, @@ -350,6 +440,7 @@ export default class VisualMaskManager { name: `Mask ${id + 1}`, type: 'rectangle', enabled: true, + useRelativePositioning: true, frames: [], style: DEFAULT_VISUAL_MASK_STYLE, })); @@ -394,6 +485,20 @@ export default class VisualMaskManager { this.commit(); } + setMaskRelativePositioning( + camera: string, + id: number, + useRelativePositioning: boolean, + frameSize?: readonly [number, number], + ) { + const mask = this.getMask(camera, id); + if (!mask) { + return; + } + mask.setUseRelativePositioning(useRelativePositioning, frameSize); + this.commit(); + } + setMaskStyle(camera: string, id: number, style: CustomStyle) { const mask = this.getMask(camera, id); if (!mask) { @@ -422,7 +527,13 @@ export default class VisualMaskManager { this.commit(); } - updateRectBounds(camera: string, frame: number, bounds: RectBounds, id = this.editingMaskId.value) { + updateRectBounds( + camera: string, + frame: number, + bounds: RectBounds, + id = this.editingMaskId.value, + frameSize?: readonly [number, number], + ) { if (id === null) { return; } @@ -435,7 +546,7 @@ export default class VisualMaskManager { frame, bounds, keyframe: true, - }); + }, [], true, frameSize); this.commit(); } @@ -444,6 +555,7 @@ export default class VisualMaskManager { frame: number, data: GeoJSON.Feature, id = this.editingMaskId.value, + frameSize?: readonly [number, number], ) { if (id === null) { return; @@ -457,13 +569,15 @@ export default class VisualMaskManager { frame, bounds: geojsonToBound(data), keyframe: true, - }); + }, [], true, frameSize); this.commit(); } } export { DEFAULT_VISUAL_MASK_STYLE, + convertBoundsToAbsolute, + convertBoundsToRelative, getVisualMaskStyleKey, normalizeStyle, }; diff --git a/docs/UI-Visual-Masks.md b/docs/UI-Visual-Masks.md index b36dedea..8bac46b7 100644 --- a/docs/UI-Visual-Masks.md +++ b/docs/UI-Visual-Masks.md @@ -20,6 +20,8 @@ If at least one visual mask exists for the current dataset, configuration owners This visibility button is hidden when no visual masks exist, and it is also hidden for non-owner/admin users. +Within the **Visual Masks** sidebar, each mask row also includes a checkbox with a `Visibility` tooltip so owners/admins can enable or disable that specific mask in the saved configuration. + ## Creating a Visual Mask Visual masks are currently **rectangle-only**. @@ -30,15 +32,27 @@ Visual masks are currently **rectangle-only**. 1. Click and drag to place the visual mask. 1. Save your changes using the standard save button in the navigation bar. +New masks default to **Relative positioning**, which stores their bounds as percentages of the current video size instead of raw pixels. + ## Editing a Visual Mask 1. Select a visual mask from the **Visual Masks** sidebar. 1. Click **Edit Current Frame** or right-click the visual mask in the annotation view. 1. Drag the handles to resize the box or drag the center to move it. +1. Optionally toggle **Relative positioning** on or off for the selected mask. 1. Save when you are finished. Visual masks support frame-specific changes. If a mask is edited on a later frame, that change becomes a new keyframe for the mask. +## Positioning Mode + +Each mask has a **Relative positioning** toggle in the sidebar. + +- When enabled, mask bounds are stored as percentages of the video width and height. +- This mode is useful when the same overall configuration folder may be applied to videos with different resolutions. +- When disabled, bounds are stored as absolute image coordinates. +- Existing masks may still use absolute coordinates if they were created before relative positioning was added or if the toggle is turned off. + ## Frame Behavior Visual masks persist until changed. @@ -72,6 +86,7 @@ Example: "id": 0, "name": "Doorway Mask", "enabled": true, + "useRelativePositioning": true, "type": "rectangle", "style": { "color": "#000000", @@ -82,12 +97,12 @@ Example: "frames": [ { "frame": 10, - "bounds": [120, 80, 420, 260], + "bounds": [11.1111, 7.4074, 38.8889, 24.0741], "keyframe": true }, { "frame": 25, - "bounds": [140, 80, 420, 260], + "bounds": [12.963, 7.4074, 38.8889, 24.0741], "keyframe": true } ] @@ -101,9 +116,11 @@ Quick notes: - `visualMasks.cam1` can be replaced with any camera name from the dataset configuration. - Each mask needs an `id`, `name`, `type`, and `frames` array. +- `useRelativePositioning` controls whether frame bounds are stored as percentages or absolute coordinates. - `type` is currently always `"rectangle"`. - `frames` stores the explicit shape changes for that mask. The mask remains in effect until a later frame entry changes it. -- Each frame entry uses `bounds: [x1, y1, x2, y2]` in image/display coordinates. +- When `useRelativePositioning` is `true`, each frame entry uses `bounds: [x1, y1, x2, y2]` as percentage values of the video size, where `50` means `50%`. +- When `useRelativePositioning` is `false`, each frame entry uses `bounds: [x1, y1, x2, y2]` in image/display coordinates. - `style` is optional. If omitted, the application fills in defaults such as black, filled, and fully opaque (`opacity: 1`). ## Current Limitations diff --git a/server/dive_utils/models.py b/server/dive_utils/models.py index 92998410..693f65c6 100644 --- a/server/dive_utils/models.py +++ b/server/dive_utils/models.py @@ -542,6 +542,7 @@ class VisualMask(BaseModel): id: int name: str enabled: Optional[bool] + useRelativePositioning: Optional[bool] type: Literal['rectangle'] frames: List[Feature] = Field(default_factory=list) style: Optional[CustomStyle] From 0362330313eb7bec55823c3b37133bab9a6bb95d Mon Sep 17 00:00:00 2001 From: Bryon Lewis Date: Tue, 12 May 2026 10:31:37 -0400 Subject: [PATCH 12/12] increment version --- client/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/package.json b/client/package.json index 9b0ae54d..0021b34f 100644 --- a/client/package.json +++ b/client/package.json @@ -1,6 +1,6 @@ { "name": "dive-dsa", - "version": "1.11.29", + "version": "1.11.30", "author": { "name": "Kitware, Inc.", "email": "Bryon.Lewis@kitware.com"