~ knotless Search/
↑↓ move↵ open⇧↵ all results

cameron.stream/prism

Audio-reactive WebGL visualizer for OBS streams

0
0
main
192 lines · 9.6 KB Raw History
// prism-audio: read-only audio analyser for Prism.
// Captures system audio (or one app) via ScreenCaptureKit, runs a 2048-point FFT
// every 800 samples (60 Hz at 48 kHz) and prints one JSON line per hop:
//   {"rms","peak","bands":[64 × 0–255],"bass","mid","high","flux","kick"}
// Bands are log-spaced 30 Hz–16 kHz on a FIXED dB scale (no auto-gain), with a
// +3 dB/octave tilt so pink noise reads flat. Nothing is played or recorded.
//
//   prism-audio                      all system audio (except this process)
//   prism-audio --app <bundle id>    one application, e.g. com.ableton.live
//   prism-audio --stdin [rate]       raw float32 mono PCM on stdin (tests)
//
// Build: swiftc AudioCapture.swift -o prism-audio -O \
//          -framework ScreenCaptureKit -framework AVFoundation -framework CoreMedia -framework Accelerate
import Foundation
import ScreenCaptureKit
import AVFoundation
import CoreMedia
import Accelerate

func emit(_ value: [String: Any]) {
    if let data = try? JSONSerialization.data(withJSONObject: value), let s = String(data: data, encoding: .utf8) { print(s); fflush(stdout) }
}
func round4(_ v: Float) -> Double { (Double(v) * 10000).rounded() / 10000 }

final class Analyzer {
    static let size = 2048, bandCount = 64, minHz: Float = 30, maxHz: Float = 16000
    let sampleRate: Float, hop: Int
    private var ring = [Float](repeating: 0, count: Analyzer.size)
    private var write = 0, filled = 0, sinceHop = 0
    private var hopSquares: Float = 0, hopPeak: Float = 0
    private let window: [Float]
    private let setup: vDSP_DFT_Setup
    private var reference: Float = 1
    private var previousBands = [Float](repeating: 0, count: Analyzer.bandCount)
    private let edges: [Float]
    var output: ([String: Any]) -> Void = emit

    init(sampleRate: Float) {
        self.sampleRate = sampleRate
        hop = max(256, Int((sampleRate / 60).rounded()))
        window = vDSP.window(ofType: Float.self, usingSequence: .hanningDenormalized, count: Analyzer.size, isHalfWindow: false)
        setup = vDSP_DFT_zrop_CreateSetup(nil, vDSP_Length(Analyzer.size), .FORWARD)!
        edges = (0...Analyzer.bandCount).map { Analyzer.minHz * pow(Analyzer.maxHz / Analyzer.minHz, Float($0) / Float(Analyzer.bandCount)) }
        // Self-calibrate: a full-scale 1 kHz sine defines 0 dBFS for one bin.
        let tone = (0..<Analyzer.size).map { sin(2 * Float.pi * 1000 * Float($0) / sampleRate) }
        reference = magnitudes(tone).max() ?? 1
    }

    private func magnitudes(_ frame: [Float]) -> [Float] {
        let half = Analyzer.size / 2
        var windowed = [Float](repeating: 0, count: Analyzer.size)
        vDSP.multiply(frame, window, result: &windowed)
        var inR = [Float](repeating: 0, count: half), inI = inR, outR = inR, outI = inR
        for i in 0..<half { inR[i] = windowed[2 * i]; inI[i] = windowed[2 * i + 1] }
        vDSP_DFT_Execute(setup, inR, inI, &outR, &outI)
        var mags = [Float](repeating: 0, count: half)
        for k in 1..<half { mags[k] = sqrt(outR[k] * outR[k] + outI[k] * outI[k]) }
        mags[0] = 0
        return mags
    }

    func push(_ samples: UnsafePointer<Float>, count: Int, stride: Int = 1) {
        for i in 0..<count {
            let v = samples[i * stride]
            ring[write] = v; write = (write + 1) % Analyzer.size
            filled = min(Analyzer.size, filled + 1)
            hopSquares += v * v; hopPeak = max(hopPeak, abs(v))
            sinceHop += 1
            if sinceHop >= hop { sinceHop = 0; if filled == Analyzer.size { analyse() }; hopSquares = 0; hopPeak = 0 }
        }
    }

    private func analyse() {
        let frame = Array(ring[write...] + ring[..<write])
        let mags = magnitudes(frame)
        let binHz = sampleRate / Float(Analyzer.size)
        var bands = [Float](repeating: 0, count: Analyzer.bandCount)
        for b in 0..<Analyzer.bandCount {
            let lo = edges[b], hi = edges[b + 1]
            let first = Int(ceil(lo / binHz)), last = Int(floor(hi / binHz))
            var power: Float
            if last >= first {
                power = 0
                for k in first...min(last, mags.count - 1) { power += mags[k] * mags[k] }
                power /= Float(min(last, mags.count - 1) - first + 1)
            } else {
                // Band narrower than one bin (deep bass): interpolate at its centre.
                let x = sqrt(lo * hi) / binHz, k = min(Int(x), mags.count - 2), t = x - Float(k)
                let m = mags[k] * (1 - t) + mags[k + 1] * t
                power = m * m
            }
            let centre = sqrt(lo * hi)
            let db = 10 * log10(max(power, 1e-14)) - 20 * log10(reference) + 3 * log2(centre / 1000)
            bands[b] = pow(min(1, max(0, (db + 72) / 62)), 1.15)
        }
        func average(_ from: Float, _ to: Float) -> Float {
            var sum: Float = 0, n: Float = 0
            for b in 0..<Analyzer.bandCount where sqrt(edges[b] * edges[b + 1]) >= from && sqrt(edges[b] * edges[b + 1]) < to { sum += bands[b]; n += 1 }
            return n > 0 ? sum / n : 0
        }
        var flux: Float = 0, kick: Float = 0, kickBands: Float = 0
        for b in 0..<Analyzer.bandCount {
            let rise = max(0, bands[b] - previousBands[b])
            flux += rise
            if sqrt(edges[b] * edges[b + 1]) < 150 { kick += rise; kickBands += 1 }
        }
        previousBands = bands
        output([
            "rms": round4(sqrt(hopSquares / Float(hop))), "peak": round4(hopPeak),
            "bands": bands.map { Int(($0 * 255).rounded()) },
            "bass": round4(average(30, 160)), "mid": round4(average(160, 2000)), "high": round4(average(2000, 16001)),
            "flux": round4(min(1, flux / Float(Analyzer.bandCount) * 6)), "kick": round4(min(1, kick / max(1, kickBands) * 4)),
        ])
    }
}

final class Capture: NSObject, SCStreamOutput, SCStreamDelegate {
    var stream: SCStream?
    var analyzer: Analyzer?
    var mono = [Float]()
    let bundleID: String?
    init(bundleID: String?) { self.bundleID = bundleID }

    func stream(_ stream: SCStream, didStopWithError error: Error) { emit(["error": error.localizedDescription]); exit(1) }
    func stream(_ stream: SCStream, didOutputSampleBuffer sample: CMSampleBuffer, of type: SCStreamOutputType) {
        guard type == .audio, sample.isValid, let desc = sample.formatDescription else { return }
        let format = AVAudioFormat(cmAudioFormatDescription: desc)
        if analyzer == nil { analyzer = Analyzer(sampleRate: Float(format.sampleRate)) }
        let frames = CMSampleBufferGetNumSamples(sample)
        guard frames > 0 else { return }
        do {
            try sample.withAudioBufferList { list, _ in
                guard let pcm = AVAudioPCMBuffer(pcmFormat: format, bufferListNoCopy: list.unsafePointer), let channels = pcm.floatChannelData else { return }
                pcm.frameLength = AVAudioFrameCount(frames)
                let count = Int(pcm.frameLength), stride = Int(pcm.stride), n = Int(format.channelCount)
                if mono.count < count { mono = [Float](repeating: 0, count: count) }
                for i in 0..<count {
                    var v: Float = 0
                    for c in 0..<n { v += channels[c][i * stride] }
                    mono[i] = v / Float(max(1, n))
                }
                mono.withUnsafeBufferPointer { analyzer?.push($0.baseAddress!, count: count) }
            }
        } catch { emit(["error": "audio buffer: \(error.localizedDescription)"]) }
    }

    func start() async {
        do {
            let content = try await SCShareableContent.excludingDesktopWindows(false, onScreenWindowsOnly: false)
            guard let display = content.displays.first else { emit(["error": "No display found"]); exit(1) }
            let filter: SCContentFilter
            if let id = bundleID {
                guard let app = content.applications.first(where: { $0.bundleIdentifier == id }) else { emit(["error": "Application \(id) is not running"]); exit(2) }
                filter = SCContentFilter(display: display, including: [app], exceptingWindows: [])
            } else {
                filter = SCContentFilter(display: display, excludingWindows: [])
            }
            let config = SCStreamConfiguration()
            config.width = 2; config.height = 2
            config.minimumFrameInterval = CMTime(value: 1, timescale: 1)
            config.capturesAudio = true
            config.excludesCurrentProcessAudio = true
            config.sampleRate = 48000; config.channelCount = 2
            let s = SCStream(filter: filter, configuration: config, delegate: self)
            try s.addStreamOutput(self, type: .audio, sampleHandlerQueue: DispatchQueue(label: "prism.audio"))
            stream = s
            try await s.startCapture()
            emit(["status": "connected", "capture": bundleID ?? "system"])
        } catch { emit(["error": error.localizedDescription]); exit(1) }
    }
}

let args = CommandLine.arguments
if let i = args.firstIndex(of: "--stdin") {
    let rate = (i + 1 < args.count ? Float(args[i + 1]) : nil) ?? 48000
    let analyzer = Analyzer(sampleRate: rate)
    let input = FileHandle.standardInput
    var carry = Data()
    while true {
        let chunk = input.readData(ofLength: 4096 * 4)
        if chunk.isEmpty { break }
        carry.append(chunk)
        let usable = carry.count / 4 * 4
        carry.prefix(usable).withUnsafeBytes { raw in analyzer.push(raw.bindMemory(to: Float.self).baseAddress!, count: usable / 4) }
        carry = carry.suffix(from: carry.startIndex + usable)
    }
    exit(0)
}
let appIndex = args.firstIndex(of: "--app")
let capture = Capture(bundleID: appIndex.flatMap { $0 + 1 < args.count ? args[$0 + 1] : nil })
Task { await capture.start() }
RunLoop.main.run()