5.0.0-dev.8

* audio playback markdown commands

* trigger markdown commands from MVS primitives

* docs

* fix usage of mutative

CHANGELOG.md

@@ -41,14 +41,16 @@ Note that since we don't clearly distinguish between a public and private interf
   - MVSX - use Murmur hash instead of FNV in archive URI
   - Support additional file formats (pdbqt, gro, xyz, mol, sdf, mol2, xtc, lammpstrj)
   - Support loading trajectory coordinates from separate nodes
+  - Trigger markdown commands from primitives using `molstar_markdown_commands` custom extensions
 - Added new color schemes, synchronized with D3.js ('inferno', 'magma', 'turbo', 'rainbow', 'sinebow', 'warm', 'cool', 'cubehelix-default', 'category-10', 'observable-10', 'tableau-10')
 - Snapshot Markdown improvements
   - Add `MarkdownExtensionManager` (`PluginContext.managers.markdownExtensions`)
   - Support custom markdown commands to control the plugin via the `[link](!command)` pattern
   - Support rendering custom elements via the `![alt](!parameters)` pattern
   - Support tables
-  - Support loading images from MVSX files
+  - Support loading images and audio from MVSX files
   - Indicate external links with ⤴
+  - Audio support
 - Avoid calculating rings for coarse-grained structures
 - Fix isosurface compute shader normals when transformation matrix is applied to volume
 - Symmetry operator naming for spacegroup symmetry - parenthesize multi-character indices (1_111-1 -> 1_(11)1(-1))

docs/docs/plugin/managers/markdown-extensions.md

@@ -28,6 +28,7 @@ Generally, the command should be URL encoded, e.g., `a b` => `a%20b` (in JS, `en
   - (optional) `action` is an array of `highlight` (default), `focus` (multiple actions can be specified)
   - (optional) `focus-radius` is extra distance applied when focusing the selection (default is `3`)
   - Example: `[HEM](!query%3Dresn%20HEM%26lang%3Dpymol%26action%3Dhighlight%2Cfocus)` highlights or focuses the HEM residue (the command must be URL encoded because it contains spaces and possibly other special characters)
+- `play-audio=src`, `toggle-audio[=str]`, `stop-audio`, `pause-audio` - Audio playback support
 
 ## Custom Content
 

package-lock.json

@@ -1,12 +1,12 @@
 {
   "name": "molstar",
-  "version": "5.0.0-dev.7",
+  "version": "5.0.0-dev.8",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "molstar",
-      "version": "5.0.0-dev.7",
+      "version": "5.0.0-dev.8",
       "license": "MIT",
       "dependencies": {
         "@types/argparse": "^2.0.17",

package.json

@@ -1,6 +1,6 @@
 {
   "name": "molstar",
-  "version": "5.0.0-dev.7",
+  "version": "5.0.0-dev.8",
   "description": "A comprehensive macromolecular library.",
   "homepage": "https://github.com/molstar/molstar#readme",
   "repository": {

src/examples/mvs-stories/stories/index.ts

@@ -7,9 +7,11 @@
 import { buildStory as kinase } from './kinase';
 import { buildStory as tbp } from './tbp';
 import { buildStory as animation } from './animation';
+import { buildStory as audio } from './audio';
 
 export const Stories = [
     { id: 'kinase', name: 'BCR-ABL: A Kinase Out of Control', buildStory: kinase },
     { id: 'tata', name: 'TATA-Binding Protein and its Role in Transcription Initiation ', buildStory: tbp },
     { id: 'animation', name: 'Molecular Animation', buildStory: animation },
+    { id: 'audio', name: 'Audio Playback', buildStory: audio },
 ] as const;

src/extensions/mvs/components/primitives.ts

@@ -149,6 +149,7 @@ export const MVSBuildPrimitiveShape = MVSTransform({
         const context: PrimitiveBuilderContext = { ...a.data, structureRefs };
 
         const snapshotKey = { snapshotKey: { ...SnapshotKey, defaultValue: a.data.options?.snapshot_key ?? '' } };
+        const markdownCommands = { markdownCommands: { ...MarkdownCommands, defaultValue: a.data.node?.custom?.molstar_markdown_commands } };
 
         const label = capitalize(params.kind);
         if (params.kind === 'mesh') {
@@ -161,6 +162,7 @@ export const MVSBuildPrimitiveShape = MVSTransform({
                 params: {
                     ...PD.withDefaults(Mesh.Params, { alpha: a.data.options?.opacity ?? 1, ...customMeshParams }),
                     ...snapshotKey,
+                    ...markdownCommands,
                 },
                 getShape: (_, data, __, prev: any) => buildPrimitiveMesh(data, prev?.geometry),
                 geometryUtils: Mesh.Utils,
@@ -185,6 +187,7 @@ export const MVSBuildPrimitiveShape = MVSTransform({
                         ...customLabelParams,
                     }),
                     ...snapshotKey,
+                    ...markdownCommands,
                 },
                 getShape: (_, data, props, prev: any) => buildPrimitiveLabels(data, prev?.geometry, props),
                 geometryUtils: Text.Utils,
@@ -199,6 +202,7 @@ export const MVSBuildPrimitiveShape = MVSTransform({
                 params: {
                     ...PD.withDefaults(Lines.Params, { alpha: a.data.options?.opacity ?? 1, ...customLineParams }),
                     ...snapshotKey,
+                    ...markdownCommands,
                 },
                 getShape: (_, data, __, prev: any) => buildPrimitiveLines(data, prev?.geometry),
                 geometryUtils: Lines.Utils,
@@ -227,7 +231,7 @@ export const MVSShapeRepresentation3D = MVSTransform({
             const repr = ShapeRepresentation(a.data.getShape, a.data.geometryUtils);
             await repr.createOrUpdate(props, a.data.data).runInContext(ctx);
 
-            const pickable = !!(params as any).snapshotKey?.trim();
+            const pickable = !!(params as any).snapshotKey?.trim() || !!(params as any).markdownCommands;
             if (pickable) {
                 repr.setState({ pickable, markerActions: MarkerActions.Highlighting });
             }
@@ -241,7 +245,7 @@ export const MVSShapeRepresentation3D = MVSTransform({
             await b.data.repr.createOrUpdate(props, a.data.data).runInContext(ctx);
             b.data.sourceData = a.data;
 
-            const pickable = !!(newParams as any).snapshotKey?.trim();
+            const pickable = !!(newParams as any).snapshotKey?.trim() || !!(newParams as any).markdownCommands;
             if (pickable) {
                 b.data.repr.setState({ pickable, markerActions: MarkerActions.Highlighting });
             }
@@ -252,6 +256,7 @@ export const MVSShapeRepresentation3D = MVSTransform({
 });
 
 const SnapshotKey = PD.Text('', { isEssential: true, disableInteractiveUpdates: true, description: 'Activate the snapshot with the provided key when clicking on the label' });
+const MarkdownCommands = PD.Value<any>(undefined, { isHidden: true });
 
 /* **************************************************** */
 

src/extensions/mvs/helpers/animation.ts

@@ -5,7 +5,6 @@
  * @author Ludovic Autin <ludovic.autin@gmail.com>
  */
 
-import { create } from 'mutative';
 import { Snapshot } from '../mvs-data';
 import { Tree } from '../tree/generic/tree-schema';
 import { clamp, lerp } from '../../../mol-math/interpolate';
@@ -20,6 +19,7 @@ import { makeContinuousPaletteCheckpoints, MVSContinuousPaletteProps, MVSDiscret
 import { palettePropsFromMVSPalette } from '../load-helpers';
 import { SortedArray } from '../../../mol-data/int';
 import { ColorT } from '../tree/mvs/param-types';
+import { produce } from '../../../mol-util/produce';
 
 export async function generateStateTransition(ctx: RuntimeContext, snapshot: Snapshot, snapshotIndex: number, snapshotCount: number) {
     if (!snapshot.animation) return undefined;
@@ -81,7 +81,7 @@ interface InterpolationCacheEntry {
 }
 
 function createSnapshot(tree: MVSTree, transitions: MVSAnimationNode<'interpolate'>[], time: number, cache: Map<any, InterpolationCacheEntry>, nodeMap: Map<string, (string | number)[]>) {
-    return create(tree, (draft) => {
+    return produce(tree, (draft) => {
         for (const transition of transitions) {
             const nodePath = nodeMap.get(transition.params.target_ref);
             if (!nodePath) continue;

src/extensions/mvs/load.ts

@@ -66,6 +66,9 @@ async function _loadMVS(ctx: RuntimeContext, plugin: PluginContext, data: MVSDat
         const mvsExtensionLoaded = plugin.state.hasBehavior(MolViewSpec);
         if (!mvsExtensionLoaded) throw new Error('MolViewSpec extension is not loaded.');
 
+        // Stop any currently running audio
+        plugin.managers.markdownExtensions.audio.dispose();
+
         // Reset canvas props to default so that modifyCanvasProps works as expected
         resetCanvasProps(plugin);
 

src/mol-canvas3d/canvas3d.ts

@@ -34,7 +34,6 @@ import { ImagePass, ImageProps } from './passes/image';
 import { Sphere3D } from '../mol-math/geometry';
 import { addConsoleStatsProvider, isDebugMode, isTimingMode, removeConsoleStatsProvider } from '../mol-util/debug';
 import { CameraHelperParams } from './helper/camera-helper';
-import { create as produce } from 'mutative';
 import { HandleHelperParams } from './helper/handle-helper';
 import { StereoCamera, StereoCameraParams } from './camera/stereo';
 import { Helper } from './helper/helper';
@@ -49,6 +48,7 @@ import { IlluminationParams } from './passes/illumination';
 import { isMobileBrowser } from '../mol-util/browser';
 import { Ray3D } from '../mol-math/geometry/primitives/ray3d';
 import { RayHelper } from './helper/ray-helper';
+import { produce } from '../mol-util/produce';
 
 export const CameraFogParams = {
     intensity: PD.Numeric(15, { min: 1, max: 100, step: 1 }),

src/mol-canvas3d/helper/camera-helper.ts

@@ -4,7 +4,7 @@
  * @author Alexander Rose <alexander.rose@weirdbyte.de>
  */
 
-import { create as produce } from 'mutative';
+import { produce } from '../../mol-util/produce';
 import { Interval } from '../../mol-data/int/interval';
 import { addCylinder } from '../../mol-geo/geometry/mesh/builder/cylinder';
 import { addSphere } from '../../mol-geo/geometry/mesh/builder/sphere';

src/mol-canvas3d/helper/handle-helper.ts

@@ -16,7 +16,7 @@ import { addCylinder } from '../../mol-geo/geometry/mesh/builder/cylinder';
 import { ValueCell } from '../../mol-util';
 import { Sphere3D } from '../../mol-math/geometry';
 import { ParamDefinition as PD } from '../../mol-util/param-definition';
-import { create as produce } from 'mutative';
+import { produce } from '../../mol-util/produce';
 import { Shape } from '../../mol-model/shape';
 import { PickingId } from '../../mol-geo/geometry/picking';
 import { Camera } from '../camera';

src/mol-plugin-state/manager/markdown-extensions.ts

@@ -10,6 +10,7 @@ import { StateObjectCell } from '../../mol-state';
 import { PluginContext } from '../../mol-plugin/context';
 import { Script } from '../../mol-script/script';
 import { QueryContext, QueryFn, StructureElement, StructureSelection } from '../../mol-model/structure';
+import { BehaviorSubject } from 'rxjs';
 
 export type MarkdownExtensionEvent = 'click' | 'mouse-enter' | 'mouse-leave';
 
@@ -150,9 +151,46 @@ export const BuiltInMarkdownExtension: MarkdownExtension[] = [
             }
         },
     },
+    {
+        name: 'play-audio',
+        execute: ({ event, args, manager }) => {
+            if (event !== 'click') return;
+
+            const src = args['play-audio'];
+            if (!src?.length) return;
+            manager.audio.play(src);
+        }
+    },
+    {
+        name: 'toggle-audio',
+        execute: ({ event, args, manager }) => {
+            if (event !== 'click' || !('toggle-audio' in args)) return;
+
+            const src = args['toggle-audio'];
+            manager.audio.play(src, { toggle: true });
+        }
+    },
+    {
+        name: 'pause-audio',
+        execute: ({ event, args, manager }) => {
+            if (event !== 'click' || !('pause-audio' in args)) return;
+            manager.audio.pause();
+        }
+    },
+    {
+        name: 'stop-audio',
+        execute: ({ event, args, manager }) => {
+            if (event !== 'click' || !('stop-audio' in args)) return;
+            manager.audio.stop();
+        }
+    },
 ];
 
 export class MarkdownExtensionManager {
+    state = {
+        audioPlayer: new BehaviorSubject<HTMLAudioElement | null>(null),
+    };
+
     private extension: MarkdownExtension[] = [];
     private refResolvers: Record<string, (plugin: PluginContext, refs: string[]) => StateObjectCell[]> = {
         default: (plugin: PluginContext, refs: string[]) => refs
@@ -286,6 +324,76 @@ export class MarkdownExtensionManager {
         return ret;
     }
 
+    private resolveAudioPlayer() {
+        if (this.state.audioPlayer.value) {
+            return this.state.audioPlayer.value;
+        }
+
+        const audio = document.createElement('audio');
+        audio.controls = true;
+        audio.preload = 'auto';
+        audio.style.width = '100%';
+        audio.style.height = '32px';
+        this.state.audioPlayer.next(audio);
+        return audio;
+    }
+
+    get audioPlayer() {
+        return this.state.audioPlayer.value;
+    }
+
+    audio = {
+        play: async (src: string, options?: { toggle?: boolean }) => {
+            try {
+                const audio = this.resolveAudioPlayer();
+
+                let newSource = false;
+                if (src?.trim()) {
+                    const resolved = this.tryResolveUri(src);
+                    let uri: string = src;
+                    if (typeof (resolved as Promise<string>)?.then === 'function') {
+                        uri = (await resolved) as string;
+                    } else if (resolved) {
+                        uri = resolved as string;
+                    }
+                    newSource = audio.src !== uri;
+                    if (newSource) {
+                        audio.src = uri;
+                        audio.load();
+                    }
+                }
+
+                if (!newSource && options?.toggle) {
+                    if (audio.paused) {
+                        await audio.play();
+                    } else {
+                        audio.pause();
+                    }
+                } else {
+                    audio.currentTime = 0;
+                    await audio.play();
+                }
+            } catch (e) {
+                console.error('Failed to play audio', e);
+            }
+        },
+        pause: () => {
+            this.audioPlayer?.pause();
+        },
+        stop: () => {
+            if (!this.audioPlayer) return;
+            this.audioPlayer.pause();
+            this.audioPlayer.currentTime = 0;
+        },
+        dispose: () => {
+            if (this.audioPlayer) {
+                this.audioPlayer.pause();
+                this.audioPlayer.currentTime = 0;
+                this.state.audioPlayer.next(null);
+            }
+        }
+    };
+
     constructor(public plugin: PluginContext) {
         for (const command of BuiltInMarkdownExtension) {
             this.registerExtension(command);

src/mol-plugin-ui/controls/markdown.tsx

@@ -4,7 +4,7 @@
  * @author David Sehnal <david.sehnal@gmail.com>
  */
 
-import { useContext, useEffect, useState } from 'react';
+import { useContext, useEffect, useRef, useState } from 'react';
 import ReactMarkdown, { Components } from 'react-markdown';
 import remarkGfm from 'remark-gfm';
 import { PluginReactContext } from '../base';
@@ -13,9 +13,11 @@ import { PluginContext } from '../../mol-plugin/context';
 import { MarkdownExtension } from '../../mol-plugin-state/manager/markdown-extensions';
 import { ColorLists } from '../../mol-util/color/lists';
 import { getColorGradient, getColorGradientBanded, parseColorList } from '../../mol-util/color/utils';
+import { useBehavior } from '../hooks/use-behavior';
 
 export function Markdown({ children, components }: { children?: string, components?: Components }) {
     return <div className='msp-markdown'>
+        <MarkdownAudioPlayer />
         <ReactMarkdown
             skipHtml
             components={{ a: MarkdownAnchor, img: MarkdownImg, ...components }}
@@ -26,6 +28,21 @@ export function Markdown({ children, components }: { children?: string, componen
     </div>;
 }
 
+export function MarkdownAudioPlayer() {
+    const parent = useRef<HTMLDivElement>(null);
+    const plugin: PluginUIContext | undefined = useContext(PluginReactContext);
+    const audio = useBehavior(plugin?.managers.markdownExtensions.state.audioPlayer);
+
+    useEffect(() => {
+        if (!parent.current) return;
+        parent.current.appendChild(audio!);
+        return () => { audio?.remove(); };
+    }, [audio]);
+    if (!audio) return null;
+
+    return <div className='msp-markdown-audio-player' ref={parent} />;
+}
+
 export function MarkdownImg({ src, element, alt }: { src?: string, element?: any, alt?: string }) {
     const plugin: PluginUIContext | undefined = useContext(PluginReactContext);
 

src/mol-plugin-ui/viewport/simple-settings.tsx

@@ -5,7 +5,7 @@
  * @author David Sehnal <david.sehnal@gmail.com>
  */
 
-import { create as produce } from 'mutative';
+import { produce } from '../../mol-util/produce';
 import { throttleTime } from 'rxjs';
 import { Canvas3DContext, Canvas3DParams, Canvas3DProps } from '../../mol-canvas3d/canvas3d';
 import { PluginCommands } from '../../mol-plugin/commands';

src/mol-plugin/behavior/dynamic/camera.ts

@@ -79,6 +79,13 @@ export const FocusLoci = PluginBehavior.create<FocusLociProps>({
                     return;
                 }
 
+                // Prevent when interaction props are set
+                const snapshotKey = current.repr?.props?.snapshotKey?.trim() ?? '';
+                const markdownCommands = current.repr?.props?.markdownCommands;
+                if (snapshotKey || (typeof markdownCommands === 'object' && Object.keys(markdownCommands).length > 0)) {
+                    return;
+                }
+
                 if (Binding.match(binding, button, modifiers)) {
                     const loci = Loci.normalize(current.loci, this.ctx.managers.interactivity.props.granularity);
                     this.ctx.managers.camera.focusLoci(loci, this.params);

src/mol-plugin/behavior/dynamic/representation.ts

@@ -290,6 +290,15 @@ export const FocusLoci = PluginBehavior.create<FocusLociProps>({
                     return;
                 }
 
+                // Support executing markdown commands associated with a visual
+                const markdownCommands = current.repr?.props?.markdownCommands;
+                if (!this.ctx.selectionMode && matched && typeof markdownCommands === 'object') {
+                    if (Object.keys(markdownCommands).length > 0) {
+                        this.ctx.managers.markdownExtensions.tryExecute('click', markdownCommands);
+                        return;
+                    }
+                }
+
                 // only apply structure focus for appropriate granularity
                 const { granularity } = this.ctx.managers.interactivity.props;
                 if (granularity !== 'residue' && granularity !== 'element') return;

src/mol-plugin/context.ts

@@ -5,7 +5,7 @@
  * @author Alexander Rose <alexander.rose@weirdbyte.de>
  */
 
-import { create as produce } from 'mutative';
+import { produce } from '../mol-util/produce';
 import { List } from 'immutable';
 import { merge, Subscription } from 'rxjs';
 import { debounceTime, filter, take, throttleTime } from 'rxjs/operators';
@@ -382,6 +382,7 @@ export class PluginContext {
         }
         this.subs = [];
 
+        this.managers.markdownExtensions.audio.dispose();
         this.animationLoop.stop();
         this.commands.dispose();
         this.canvas3d?.dispose();

src/mol-plugin/state.ts

@@ -6,7 +6,7 @@
  * @author Adam Midlik <midlik@gmail.com>
  */
 
-import { create as produce } from 'mutative';
+import { produce } from '../mol-util/produce';
 import { merge } from 'rxjs';
 import { Camera } from '../mol-canvas3d/camera';
 import { Canvas3DContext, Canvas3DParams, Canvas3DProps } from '../mol-canvas3d/canvas3d';

src/mol-state/state/builder.ts

@@ -11,7 +11,7 @@ import { StateObject, StateObjectCell, StateObjectSelector, StateObjectRef } fro
 import { StateTransform } from '../transform';
 import { StateTransformer } from '../transformer';
 import { State } from '../state';
-import { create as produce } from 'mutative';
+import { produce } from '../../mol-util/produce';
 
 export { StateBuilder };
 

src/mol-util/file.ts

@@ -7,6 +7,7 @@
 export async function fileToDataUri(file: File): Promise<string> {
     const filename = file.name.toLowerCase() || 'file';
     const isImage = ['jpg', 'jpeg', 'png', 'gif', 'webp'].some(ext => filename.endsWith(`.${ext}`));
+    const isAudio = ['mp3', 'wav', 'ogg'].some(ext => filename.endsWith(`.${ext}`));
 
     let type = 'application/octet-stream';
     if (isImage) {
@@ -19,6 +20,16 @@ export async function fileToDataUri(file: File): Promise<string> {
                 type = `image/${ext}`;
                 break;
         }
+    } else if (isAudio) {
+        const ext = filename.split('.').pop()?.toLowerCase();
+        switch (ext) {
+            case 'mp3':
+                type = 'audio/mpeg';
+                break;
+            default:
+                type = `audio/${ext}`;
+                break;
+        }
     }
 
     const bytes = await file.arrayBuffer();

src/mol-util/param-definition.ts

@@ -534,7 +534,7 @@ export namespace ParamDefinition {
         if (a === undefined) return { ...b };
         if (b === undefined) return { ...a };
 
-        const o = Object.create(null);
+        const o = {} as any;
         for (const k of Object.keys(params)) {
             o[k] = mergeParam(params[k], a[k], b[k]);
         }
@@ -587,7 +587,7 @@ export namespace ParamDefinition {
         if (p.type === 'value') {
             return value;
         } else if (p.type === 'group') {
-            const ret = Object.create(null);
+            const ret = {} as any;
             for (const key of Object.keys(p.params)) {
                 const param = p.params[key];
                 if (value[key] === void 0) {
@@ -638,7 +638,7 @@ export namespace ParamDefinition {
             return defaultIfUndefined ? getDefaultValues(p) : value;
         }
 
-        const ret = Object.create(null);
+        const ret = {} as any;
         for (const key of Object.keys(p)) {
             const param = p[key];
             if (value[key] === void 0) {

src/mol-util/param-mapping.ts

@@ -5,7 +5,7 @@
  */
 
 import { ParamDefinition as PD } from './param-definition';
-import { create as produce } from 'mutative';
+import { produce } from './produce';
 import { Mutable } from './type-helpers';
 
 export interface ParamMapping<S, T, Ctx> {

src/mol-util/produce.ts

@@ -0,0 +1,23 @@
+/**
+ * Copyright (c) 2025 mol* contributors, licensed under MIT, See LICENSE file for more info.
+ *
+ * @author David Sehnal <david.sehnal@gmail.com>
+ */
+
+import { create, rawReturn } from 'mutative';
+
+let currentRecipe: any = undefined;
+function recipeWrapper(draft: any) {
+    const r = currentRecipe(draft);
+    if (r !== undefined && r !== draft) return rawReturn(r);
+    return r;
+}
+
+/** Apply changes to an immutable-like object */
+export function produce<T>(base: T, recipe: (draft: T) => T | void): T {
+    currentRecipe = recipe;
+    if (typeof base === 'object' && !('prototype' in (base as any))) {
+        return create({ ...base }, recipeWrapper) as T;
+    }
+    return create(base, recipeWrapper) as T;
+}