import { useRpc, useSubscription, type PluginWorkspacePanelProps } from "@getpaseo/plugin/client";
import type { RpcOutput } from "@getpaseo/plugin";
import { type UseMutationResult, useMutation } from "@tanstack/react-query";
import { useCallback, useMemo, useState } from "react";
import { Pressable, StyleSheet, Text, View } from "react-native";
import { checkFrame } from "../shared/frame-check";
import {
decodeMediaFrame,
MediaOpcode,
mediaStream,
VIDEO,
type MediaFrame,
} from "../shared/media-frames";
interface Playback {
picture: Uint8Array | null;
level: number;
videoFrames: number;
keyframes: number;
audioChunks: number;
bytes: number;
}
const EMPTY: Playback = {
picture: null,
level: 0,
videoFrames: 0,
keyframes: 0,
audioChunks: 0,
bytes: 0,
};
export function MediaPanel({ theme, layout }: PluginWorkspacePanelProps) {
const styles = useStyles(theme, layout.compact);
const [playback, setPlayback] = useState(EMPTY);
const subscription = useSubscription(mediaStream, { source: "test-pattern" }, (bytes) => {
const frame = decodeMediaFrame(bytes);
if (frame) setPlayback((current) => play(current, frame, bytes.byteLength));
});
return (
Test pattern
{subscription.status === "error" ? subscription.error.message : subscription.status}
{playback.videoFrames} video frames ({playback.keyframes} keyframes), {playback.audioChunks}{" "}
audio chunks, {(playback.bytes / 1024).toFixed(0)} KiB
);
}
/** Uploads the picture on screen and shows what the daemon received. */
function SendFrameBack({
picture,
styles,
}: {
picture: Uint8Array | null;
styles: ReturnType;
}) {
const sendFrame = useRpc(checkFrame);
const check = useMutation({ mutationFn: sendFrame });
const { mutate } = check;
const send = useCallback(() => {
if (picture) mutate(picture);
}, [mutate, picture]);
return (
Send this frame back
{describeCheck(check)}
);
}
function describeCheck(
check: UseMutationResult, Error, Uint8Array>,
): string {
if (check.status === "error") return check.error.message;
if (check.status === "success")
return `The daemon received ${check.data.bytes} bytes, sha256 ${check.data.sha256.slice(0, 12)}`;
return check.status === "pending" ? "Sending…" : "";
}
function play(current: Playback, frame: MediaFrame, bytes: number): Playback {
if (frame.opcode === MediaOpcode.Audio)
return {
...current,
level: audioLevel(frame.payload),
audioChunks: current.audioChunks + 1,
bytes: current.bytes + bytes,
};
return {
...current,
picture: frame.payload,
videoFrames: current.videoFrames + 1,
keyframes: current.keyframes + (frame.opcode === MediaOpcode.Keyframe ? 1 : 0),
bytes: current.bytes + bytes,
};
}
function audioLevel(samples: Uint8Array): number {
const view = new DataView(samples.buffer, samples.byteOffset, samples.byteLength);
let sum = 0;
const count = samples.byteLength / 2;
for (let index = 0; index < count; index += 1) sum += view.getInt16(index * 2, true) ** 2;
return Math.min(1, Math.sqrt(sum / Math.max(1, count)) / 16384);
}
function LevelMeter({ level, styles }: { level: number; styles: ReturnType }) {
const fill = useMemo(
() => [styles.meter, { width: `${Math.round(level * 100)}%` as const }],
[level, styles],
);
return (
);
}
function Picture({ pixels, cellSize }: { pixels: Uint8Array | null; cellSize: number }) {
const rows = useMemo(() => pictureRows(pixels, cellSize), [pixels, cellSize]);
return (
{rows.map((row) => (
{row.cells.map((cell) => (
))}
))}
);
}
/** One style per pixel, row by row. */
function pictureRows(pixels: Uint8Array | null, cellSize: number) {
const rows = [];
for (let y = 0; y < VIDEO.height; y += 1) {
const cells = [];
for (let x = 0; x < VIDEO.width; x += 1) {
const offset = (y * VIDEO.width + x) * 3;
const backgroundColor = pixels
? `rgb(${pixels[offset]}, ${pixels[offset + 1]}, ${pixels[offset + 2]})`
: "transparent";
cells.push({ key: `${x}`, style: { width: cellSize, height: cellSize, backgroundColor } });
}
rows.push({ key: `${y}`, cells });
}
return rows;
}
const pictureStyles = StyleSheet.create({ row: { flexDirection: "row" } });
function useStyles(theme: PluginWorkspacePanelProps["theme"], compact: boolean) {
return useMemo(
() =>
StyleSheet.create({
screen: {
flex: 1,
padding: compact ? 16 : 24,
gap: 12,
backgroundColor: theme.colors.surface0,
},
title: { color: theme.colors.foreground, fontSize: 18, fontWeight: "600" },
muted: { color: theme.colors.foregroundMuted },
meterTrack: {
height: 8,
borderRadius: 4,
overflow: "hidden",
backgroundColor: theme.colors.surface2,
},
meter: { height: 8, backgroundColor: theme.colors.accent },
upload: { gap: 8, alignItems: "flex-start" },
button: {
paddingVertical: 8,
paddingHorizontal: 14,
borderRadius: 8,
backgroundColor: theme.colors.accent,
},
buttonText: { color: theme.colors.accentForeground },
}),
[theme, compact],
);
}