merge: Optimized pattern drawing in MkModPlayer (!410)

View MR for information: https://activitypub.software/TransFem-org/Sharkey/-/merge_requests/410

Approved-by: Amelia Yukii <amelia.yukii@shourai.de>
Approved-by: dakkar <dakkar@thenautilus.net>
This commit is contained in:
Amelia Yukii 2024-02-21 16:17:13 +00:00
commit a1a3eec262
2 changed files with 386 additions and 124 deletions

View File

@ -7,14 +7,17 @@
</div>
<div v-else class="mod-player-enabled">
<div class="pattern-display" @click="togglePattern()">
<div class="pattern-display" @click="togglePattern()" @scroll="scrollHandler" @scrollend="scrollEndHandle">
<div v-if="patternHide" class="pattern-hide">
<b><i class="ph-eye ph-bold ph-lg"></i> Pattern Hidden</b>
<span>{{ i18n.ts.clickToShow }}</span>
</div>
<span class="patternShadowTop"></span>
<span class="patternShadowBottom"></span>
<canvas ref="displayCanvas" class="pattern-canvas"></canvas>
</div>
<div class="controls">
<input v-if="patternScrollSliderShow" ref="patternScrollSlider" v-model="patternScrollSliderPos" class="pattern-slider" type="range" min="0" max="100" step="0.01" style=""/>
<button class="play" @click="playPause()">
<i v-if="playing" class="ph-pause ph-bold ph-lg"></i>
<i v-else class="ph-play ph-bold ph-lg"></i>
@ -33,43 +36,30 @@
</template>
<script lang="ts" setup>
import { ref, nextTick, computed } from 'vue';
import { ref, nextTick, computed, watch, onDeactivated, onMounted } from 'vue';
import * as Misskey from 'misskey-js';
import { i18n } from '@/i18n.js';
import { defaultStore } from '@/store.js';
import { ChiptuneJsPlayer, ChiptuneJsConfig } from '@/scripts/chiptune2.js';
import { isTouchUsing } from '@/scripts/touch.js';
const colours = {
background: '#000000',
foreground: {
default: '#ffffff',
quarter: '#ffff00',
instr: '#80e0ff',
volume: '#80ff80',
fx: '#ff80e0',
operant: '#ffe080',
},
};
const CHAR_WIDTH = 6;
const CHAR_HEIGHT = 12;
const ROW_OFFSET_Y = 10;
const colours = {
background: '#000000',
default: {
active: '#ffffff',
inactive: '#808080',
},
quarter: {
active: '#ffff00',
inactive: '#ffe135',
},
instr: {
active: '#80e0ff',
inactive: '#0099cc',
},
volume: {
active: '#80ff80',
inactive: '#008000',
},
fx: {
active: '#ff80e0',
inactive: '#800060',
},
operant: {
active: '#ffe080',
inactive: '#806000',
},
};
const MAX_TIME_SPENT = 50;
const MAX_TIME_PER_ROW = 15;
const props = defineProps<{
module: Misskey.entities.DriveFile
@ -79,29 +69,57 @@ const isSensitive = computed(() => { return props.module.isSensitive; });
const url = computed(() => { return props.module.url; });
let hide = ref((defaultStore.state.nsfw === 'force') ? true : isSensitive.value && (defaultStore.state.nsfw !== 'ignore'));
let patternHide = ref(false);
let firstFrame = ref(true);
let playing = ref(false);
let displayCanvas = ref<HTMLCanvasElement>();
let progress = ref<HTMLProgressElement>();
let position = ref(0);
let patternScrollSlider = ref<HTMLProgressElement>();
let patternScrollSliderShow = ref(false);
let patternScrollSliderPos = ref(0);
const player = ref(new ChiptuneJsPlayer(new ChiptuneJsConfig()));
const rowBuffer = 24;
const maxRowNumbers = 0xFF;
const rowBuffer = 26;
let buffer = null;
let isSeeking = false;
let firstFrame = true;
let lastPattern = -1;
let lastDrawnRow = -1;
let numberRowCanvas = new OffscreenCanvas(2 * CHAR_WIDTH + 1, maxRowNumbers * CHAR_HEIGHT + 1);
let alreadyHiddenOnce = false;
let alreadyDrawn = [false];
let patternTime = { 'current': 0, 'max': 0, 'initial': 0 };
player.value.load(url.value).then((result) => {
buffer = result;
try {
player.value.play(buffer);
progress.value!.max = player.value.duration();
display();
} catch (err) {
console.warn(err);
function bakeNumberRow() {
let ctx = numberRowCanvas.getContext('2d', { alpha: false }) as OffscreenCanvasRenderingContext2D;
ctx.font = '10px monospace';
for (let i = 0; i < maxRowNumbers; i++) {
let rowText = i.toString(16);
if (rowText.length === 1) rowText = '0' + rowText;
ctx.fillStyle = colours.foreground.default;
if (i % 4 === 0) ctx.fillStyle = colours.foreground.quarter;
ctx.fillText(rowText, 0, 10 + i * 12);
}
player.value.stop();
}).catch((error) => {
console.error(error);
}
onMounted(() => {
player.value.load(url.value).then((result) => {
buffer = result;
try {
player.value.play(buffer);
progress.value!.max = player.value.duration();
bakeNumberRow();
display();
} catch (err) {
console.warn(err);
}
player.value.stop();
}).catch((error) => {
console.error(error);
});
});
function playPause() {
@ -133,7 +151,7 @@ function stop(noDisplayUpdate = false) {
if (!noDisplayUpdate) {
try {
player.value.play(buffer);
display();
display(true);
} catch (err) {
console.warn(err);
}
@ -162,104 +180,256 @@ function performSeek() {
function toggleVisible() {
hide.value = !hide.value;
if (!hide.value && patternHide.value) {
firstFrame.value = true;
patternHide.value = false;
if (!hide.value) {
lastPattern = -1;
lastDrawnRow = -1;
}
nextTick(() => { stop(hide.value); });
}
function togglePattern() {
patternHide.value = !patternHide.value;
if (!patternHide.value) {
if (player.value.getRow() === 0) {
try {
player.value.play(buffer);
display();
} catch (err) {
console.warn(err);
}
player.value.stop();
handleScrollBarEnable();
if (player.value.getRow() === 0 && player.value.getPattern() === 0) {
try {
player.value.play(buffer);
display(true);
} catch (err) {
console.warn(err);
}
player.value.stop();
} else {
display(true);
}
}
function display() {
if (!displayCanvas.value) {
stop();
return;
}
if (patternHide.value) return;
if (firstFrame.value) {
firstFrame.value = false;
patternHide.value = true;
}
function drawPattern() {
if (!displayCanvas.value) return;
const canvas = displayCanvas.value;
const startTime = performance.now();
const pattern = player.value.getPattern();
const nbRows = player.value.getPatternNumRows(pattern);
const row = player.value.getRow();
const halfbuf = rowBuffer / 2;
const minRow = row - halfbuf;
const maxRow = row + halfbuf;
let rowDif = 0;
let nbChannels = 0;
if (player.value.currentPlayingNode) {
nbChannels = player.value.currentPlayingNode.nbChannels;
}
if (canvas.width !== 12 + 84 * nbChannels + 2) {
canvas.width = 12 + 84 * nbChannels + 2;
canvas.height = 12 * rowBuffer;
}
const nbRows = player.value.getPatternNumRows(pattern);
const ctx = canvas.getContext('2d') as CanvasRenderingContext2D;
ctx.font = '10px monospace';
ctx.fillStyle = colours.background;
ctx.fillRect(0, 0, canvas.width, canvas.height);
ctx.fillStyle = colours.default.inactive;
for (let rowOffset = 0; rowOffset < rowBuffer; rowOffset++) {
const rowToDraw = row - rowBuffer / 2 + rowOffset;
if (rowToDraw >= 0 && rowToDraw < nbRows) {
const active = (rowToDraw === row) ? 'active' : 'inactive';
let rowText = parseInt(rowToDraw).toString(16);
if (rowText.length === 1) {
rowText = '0' + rowText;
}
ctx.fillStyle = colours.default[active];
if (rowToDraw % 4 === 0) {
ctx.fillStyle = colours.quarter[active];
}
ctx.fillText(rowText, 0, 10 + rowOffset * 12);
for (let channel = 0; channel < nbChannels; channel++) {
const part = player.value.getPatternRowChannel(pattern, rowToDraw, channel);
const baseOffset = (2 + (part.length + 1) * channel) * CHAR_WIDTH;
const baseRowOffset = ROW_OFFSET_Y + rowOffset * CHAR_HEIGHT;
if (pattern === lastPattern) {
rowDif = row - lastDrawnRow;
} else {
if (patternTime.initial !== 0 && !alreadyHiddenOnce) {
const trackerTime = player.value.currentPlayingNode.getProcessTime();
ctx.fillStyle = colours.default[active];
ctx.fillText('|', baseOffset, baseRowOffset);
const note = part.substring(0, 3);
ctx.fillStyle = colours.default[active];
ctx.fillText(note, baseOffset + CHAR_WIDTH, baseRowOffset);
const instr = part.substring(4, 6);
ctx.fillStyle = colours.instr[active];
ctx.fillText(instr, baseOffset + CHAR_WIDTH * 5, baseRowOffset);
const volume = part.substring(6, 9);
ctx.fillStyle = colours.volume[active];
ctx.fillText(volume, baseOffset + CHAR_WIDTH * 7, baseRowOffset);
const fx = part.substring(10, 11);
ctx.fillStyle = colours.fx[active];
ctx.fillText(fx, baseOffset + CHAR_WIDTH * 11, baseRowOffset);
const op = part.substring(11, 13);
ctx.fillStyle = colours.operant[active];
ctx.fillText(op, baseOffset + CHAR_WIDTH * 12, baseRowOffset);
if (patternTime.initial + trackerTime.max > MAX_TIME_SPENT && trackerTime.max + patternTime.max > MAX_TIME_PER_ROW) {
alreadyHiddenOnce = true;
togglePattern();
return;
}
}
patternTime = { 'current': 0, 'max': 0, 'initial': 0 };
alreadyDrawn = [];
if (canvas.width !== (12 + 84 * nbChannels + 2)) canvas.width = 12 + 84 * nbChannels + 2;
if (canvas.height !== (12 * nbRows)) canvas.height = 12 * nbRows;
}
const ctx = canvas.getContext('2d', { alpha: false, desynchronized: true }) as CanvasRenderingContext2D;
if (ctx.font !== '10px monospace') ctx.font = '10px monospace';
ctx.imageSmoothingEnabled = false;
if (pattern !== lastPattern) {
ctx.fillStyle = colours.background;
ctx.fillRect(0, 0, canvas.width, canvas.height);
ctx.drawImage( numberRowCanvas, 0, 0 );
}
ctx.fillStyle = colours.foreground.default;
for (let rowOffset = minRow + rowDif; rowOffset < maxRow + rowDif; rowOffset++) {
const rowToDraw = rowOffset - rowDif;
if (alreadyDrawn[rowToDraw] === true) continue;
if (rowToDraw >= 0 && rowToDraw < nbRows) {
const baseOffset = 2 * CHAR_WIDTH;
const baseRowOffset = ROW_OFFSET_Y + rowToDraw * CHAR_HEIGHT;
let done = drawRow(ctx, rowToDraw, nbChannels, pattern, baseOffset, baseRowOffset);
alreadyDrawn[rowToDraw] = done;
}
}
lastDrawnRow = row;
lastPattern = pattern;
patternTime.current = performance.now() - startTime;
if (patternTime.initial !== 0 && patternTime.current > patternTime.max) patternTime.max = patternTime.current;
else if (patternTime.initial === 0) patternTime.initial = patternTime.current;
}
function drawPetternPreview() {
if (!displayCanvas.value) return;
const canvas = displayCanvas.value;
const pattern = player.value.getPattern();
const nbRows = player.value.getPatternNumRows(pattern);
const row = player.value.getRow();
const halfbuf = rowBuffer / 2;
alreadyDrawn = [];
let nbChannels = 0;
if (player.value.currentPlayingNode) {
nbChannels = player.value.currentPlayingNode.nbChannels;
}
if (canvas.width !== (12 + 84 * nbChannels + 2)) canvas.width = 12 + 84 * nbChannels + 2;
if (canvas.height !== (12 * rowBuffer)) canvas.height = 12 * rowBuffer;
const ctx = canvas.getContext('2d', { alpha: false }) as CanvasRenderingContext2D;
ctx.font = '10px monospace';
ctx.imageSmoothingEnabled = false;
ctx.fillStyle = colours.background;
ctx.fillRect(0, 0, canvas.width, canvas.height);
ctx.drawImage( numberRowCanvas, 0, (halfbuf - row) * CHAR_HEIGHT );
for (let rowOffset = 0; rowOffset < rowBuffer; rowOffset++) {
const rowToDraw = rowOffset + row - halfbuf;
if (rowToDraw >= 0 && rowToDraw < nbRows) {
const baseOffset = 2 * CHAR_WIDTH;
const baseRowOffset = ROW_OFFSET_Y + rowOffset * CHAR_HEIGHT;
drawRow(ctx, rowToDraw, nbChannels, pattern, baseOffset, baseRowOffset);
} else if (rowToDraw >= 0) {
const baseRowOffset = ROW_OFFSET_Y + rowOffset * CHAR_HEIGHT;
ctx.fillStyle = colours.background;
ctx.fillRect(0, baseRowOffset - CHAR_HEIGHT, CHAR_WIDTH * 2, baseRowOffset);
}
}
lastPattern = -1;
lastDrawnRow = -1;
}
function drawRow(ctx: CanvasRenderingContext2D, row: number, channels: number, pattern: number, drawX = (2 * CHAR_WIDTH), drawY = ROW_OFFSET_Y) {
if (!player.value.currentPlayingNode) return false;
if (alreadyDrawn[row]) return true;
const spacer = 11;
const space = ' ';
let seperators = '';
let note = '';
let instr = '';
let volume = '';
let fx = '';
let op = '';
for (let channel = 0; channel < channels; channel++) {
const part = player.value.getPatternRowChannel(pattern, row, channel);
seperators += '|' + space.repeat( spacer + 2 );
note += part.substring(0, 3) + space.repeat( spacer );
instr += part.substring(4, 6) + space.repeat( spacer + 1 );
volume += part.substring(6, 9) + space.repeat( spacer );
fx += part.substring(10, 11) + space.repeat( spacer + 2 );
op += part.substring(11, 13) + space.repeat( spacer + 1 );
}
ctx.fillStyle = colours.foreground.default;
ctx.fillText(seperators, drawX, drawY);
ctx.fillStyle = colours.foreground.default;
ctx.fillText(note, drawX + CHAR_WIDTH, drawY);
ctx.fillStyle = colours.foreground.instr;
ctx.fillText(instr, drawX + CHAR_WIDTH * 5, drawY);
ctx.fillStyle = colours.foreground.volume;
ctx.fillText(volume, drawX + CHAR_WIDTH * 7, drawY);
ctx.fillStyle = colours.foreground.fx;
ctx.fillText(fx, drawX + CHAR_WIDTH * 11, drawY);
ctx.fillStyle = colours.foreground.operant;
ctx.fillText(op, drawX + CHAR_WIDTH * 12, drawY);
return true;
}
function display(skipOptimizationChecks = false) {
if (!displayCanvas.value || !displayCanvas.value.parentElement) {
stop();
return;
}
if (patternHide.value && !skipOptimizationChecks) return;
if (firstFrame) {
// Changing it to false should enable pattern display by default.
patternHide.value = true;
handleScrollBarEnable();
firstFrame = false;
}
const row = player.value.getRow();
const pattern = player.value.getPattern();
if ( row === lastDrawnRow && pattern === lastPattern && !skipOptimizationChecks) return;
// Size vs speed
if (patternHide.value) drawPetternPreview();
else drawPattern();
displayCanvas.value.style.top = !patternHide.value ? 'calc( 50% - ' + (row * CHAR_HEIGHT) + 'px )' : '0%';
}
let suppressScrollSliderWatcher = false;
function scrollHandler() {
suppressScrollSliderWatcher = true;
if (!patternScrollSlider.value) return;
if (!displayCanvas.value) return;
if (!displayCanvas.value.parentElement) return;
patternScrollSliderPos.value = (displayCanvas.value.parentElement.scrollLeft) / (displayCanvas.value.width - displayCanvas.value.parentElement.offsetWidth) * 100;
patternScrollSlider.value.style.opacity = '1';
}
function scrollEndHandle() {
suppressScrollSliderWatcher = false;
if (!patternScrollSlider.value) return;
patternScrollSlider.value.style.opacity = '';
}
function handleScrollBarEnable() {
patternScrollSliderShow.value = (!patternHide.value && !isTouchUsing);
if (patternScrollSliderShow.value !== true) return;
if (!displayCanvas.value) return;
if (!displayCanvas.value.parentElement) return;
if (firstFrame) {
patternScrollSliderShow.value = (12 + 84 * player.value.getPatternNumRows(player.value.getPattern()) + 2 > displayCanvas.value.parentElement.offsetWidth);
} else {
patternScrollSliderShow.value = (displayCanvas.value.width > displayCanvas.value.parentElement.offsetWidth);
}
}
watch(patternScrollSliderPos, () => {
if (suppressScrollSliderWatcher) return;
if (!displayCanvas.value) return;
if (!displayCanvas.value.parentElement) return;
displayCanvas.value.parentElement.scrollLeft = (displayCanvas.value.width - displayCanvas.value.parentElement.offsetWidth) * patternScrollSliderPos.value / 100;
});
onDeactivated(() => {
stop();
});
</script>
<style lang="scss" scoped>
@ -290,6 +460,7 @@ function display() {
cursor: pointer;
top: 12px;
right: 12px;
z-index: 4;
}
> .pattern-display {
@ -299,22 +470,55 @@ function display() {
overflow-y: hidden;
background-color: black;
text-align: center;
.pattern-canvas {
background-color: black;
height: 100%;
max-height: 312px; /* magic_number = CHAR_HEIGHT * rowBuffer, needs to be in px */
scrollbar-width: none;
&::-webkit-scrollbar {
display: none;
}
.pattern-canvas {
position: relative;
background-color: black;
image-rendering: pixelated;
pointer-events: none;
z-index: 0;
}
.patternShadowTop {
background: #00000080;
width: 100%;
height: calc( 50% - 14px );
translate: 0 -100%;
top: calc( 50% - 14px );
position: absolute;
pointer-events: none;
z-index: 2;
}
.patternShadowBottom {
background: #00000080;
width: 100%;
height: calc( 50% - 12px );
top: calc( 50% - 1px );
position: absolute;
pointer-events: none;
z-index: 2;
}
.pattern-hide {
display: flex;
flex-direction: column;
justify-content: center;
align-items: center;
background: rgba(64, 64, 64, 0.3);
backdrop-filter: blur(2em);
backdrop-filter: var(--modalBgFilter);
color: #fff;
font-size: 12px;
position: absolute;
z-index: 0;
z-index: 4;
width: 100%;
height: 100%;
@ -328,7 +532,7 @@ function display() {
display: flex;
width: 100%;
background-color: var(--bg);
z-index: 1;
z-index: 5;
> * {
padding: 4px 8px;
@ -353,6 +557,18 @@ function display() {
margin: 4px 8px;
overflow-x: hidden;
&.pattern-slider {
position: absolute;
width: calc( 100% - 8px * 2 );
top: calc( 100% - 21px * 3 );
opacity: 0%;
transition: opacity 0.2s;
&:hover {
opacity: 100%;
}
}
&:focus {
outline: none;

View File

@ -150,6 +150,27 @@ ChiptuneJsPlayer.prototype.getRow = function () {
return 0;
};
ChiptuneJsPlayer.prototype.getNumPatterns = function () {
if (this.currentPlayingNode && this.currentPlayingNode.modulePtr) {
return libopenmpt._openmpt_module_get_num_patterns(this.currentPlayingNode.modulePtr);
}
return 0;
};
ChiptuneJsPlayer.prototype.getCurrentSpeed = function () {
if (this.currentPlayingNode && this.currentPlayingNode.modulePtr) {
return libopenmpt._openmpt_module_get_current_speed(this.currentPlayingNode.modulePtr);
}
return 0;
};
ChiptuneJsPlayer.prototype.getCurrentTempo = function () {
if (this.currentPlayingNode && this.currentPlayingNode.modulePtr) {
return libopenmpt._openmpt_module_get_current_tempo(this.currentPlayingNode.modulePtr);
}
return 0;
};
ChiptuneJsPlayer.prototype.getPatternNumRows = function (pattern: number) {
if (this.currentPlayingNode && this.currentPlayingNode.modulePtr) {
return libopenmpt._openmpt_module_get_pattern_num_rows(this.currentPlayingNode.modulePtr, pattern);
@ -164,6 +185,20 @@ ChiptuneJsPlayer.prototype.getPatternRowChannel = function (pattern: number, row
return '';
};
ChiptuneJsPlayer.prototype.getCtls = function () {
if (this.currentPlayingNode && this.currentPlayingNode.modulePtr) {
return libopenmpt._openmpt_module_get_ctls(this.currentPlayingNode.modulePtr);
}
return 0;
};
ChiptuneJsPlayer.prototype.version = function () {
if (this.currentPlayingNode && this.currentPlayingNode.modulePtr) {
return libopenmpt._openmpt_get_library_version();
}
return 0;
};
ChiptuneJsPlayer.prototype.createLibopenmptNode = function (buffer, config: object) {
const maxFramesPerChunk = 4096;
const processNode = this.audioContext.createScriptProcessor(2048, 0, 2);
@ -178,6 +213,7 @@ ChiptuneJsPlayer.prototype.createLibopenmptNode = function (buffer, config: obje
processNode.paused = false;
processNode.leftBufferPtr = libopenmpt._malloc(4 * maxFramesPerChunk);
processNode.rightBufferPtr = libopenmpt._malloc(4 * maxFramesPerChunk);
processNode.perf = { 'current': 0, 'max': 0 };
processNode.cleanup = function () {
if (this.modulePtr !== 0) {
libopenmpt._openmpt_module_destroy(this.modulePtr);
@ -205,7 +241,13 @@ ChiptuneJsPlayer.prototype.createLibopenmptNode = function (buffer, config: obje
processNode.togglePause = function () {
this.paused = !this.paused;
};
processNode.getProcessTime = function() {
const max = this.perf.max;
this.perf.max = 0;
return { 'current': this.perf.current, 'max': max };
};
processNode.onaudioprocess = function (e) {
let startTimeP1 = performance.now();
const outputL = e.outputBuffer.getChannelData(0);
const outputR = e.outputBuffer.getChannelData(1);
let framesToRender = outputL.length;
@ -231,11 +273,13 @@ ChiptuneJsPlayer.prototype.createLibopenmptNode = function (buffer, config: obje
const currentPattern = libopenmpt._openmpt_module_get_current_pattern(this.modulePtr);
const currentRow = libopenmpt._openmpt_module_get_current_row(this.modulePtr);
startTimeP1 = startTimeP1 - performance.now();
if (currentPattern !== this.patternIndex) {
processNode.player.fireEvent('onPatternChange');
}
processNode.player.fireEvent('onRowChange', { index: currentRow });
const startTimeP2 = performance.now();
while (framesToRender > 0) {
const framesPerChunk = Math.min(framesToRender, maxFramesPerChunk);
const actualFramesPerChunk = libopenmpt._openmpt_module_read_float_stereo(this.modulePtr, this.context.sampleRate, framesPerChunk, this.leftBufferPtr, this.rightBufferPtr);
@ -262,6 +306,8 @@ ChiptuneJsPlayer.prototype.createLibopenmptNode = function (buffer, config: obje
this.cleanup();
error ? processNode.player.fireEvent('onError', { type: 'openmpt' }) : processNode.player.fireEvent('onEnded');
}
this.perf.current = performance.now() - startTimeP2 + startTimeP1;
if (this.perf.current > this.perf.max) this.perf.max = this.perf.current;
};
return processNode;
};