🤗 HuggingFace Strategy

Stem Separation

Music Generation

Speech/Voice

Run models directly in the browser with WebGPU acceleration. Zero server cost, instant inference.

Supported Tasks

Transformers.js Example

// client-side inference
import { pipeline } from '@xenova/transformers';

// Initialize on first use (downloads model)
const classifier = await pipeline(
  'zero-shot-classification',
  'Xenova/bart-large-mnli'
);

// Classify sample descriptions
const result = await classifier(
  'punchy kick drum with 808 sub bass',
  ['drums', 'bass', 'melody', 'vocals', 'effects']
);

// result.labels = ['bass', 'drums', 'effects', 'melody', 'vocals']
// result.scores = [0.82, 0.74, 0.15, 0.08, 0.02]

Replicate provides one-click deployment of HuggingFace models with pay-per-second billing.

Recommended Models

Replicate API Example

// replicate.ts
async function separateStems(audioUrl: string): Promise<StemResult> {
  const response = await fetch('https://api.replicate.com/v1/predictions', {
    method: 'POST',
    headers: {
      'Authorization': `Token ${env.REPLICATE_API_TOKEN}`,
      'Content-Type': 'application/json'
    },
    body: JSON.stringify({
      version: 'cjwbw/htdemucs:...',
      input: {
        audio: audioUrl,
        stems: 4  // vocals, drums, bass, other
      }
    })
  });

  const prediction = await response.json();

  // Poll for completion
  while (prediction.status !== 'succeeded') {
    await new Promise(r => setTimeout(r, 1000));
    const status = await fetch(prediction.urls.get).then(r => r.json());
    if (status.status === 'succeeded') {
      return status.output;
    }
  }
}

For maximum control and lowest latency, self-host models on Fly.io GPU instances.

SHUSH-Style Deployment

# fly.toml
app = "flowstate-ai"
primary_region = "sjc"  # San Jose (GPU available)

[build]
  dockerfile = "Dockerfile.gpu"

[http_service]
  internal_port = 8000
  force_https = true

[[vm]]
  size = "a100-40gb"  # GPU instance
  memory = "32gb"

[env]
  MODEL_PATH = "/models/htdemucs"
  BATCH_SIZE = "4"

GPU Container

# Dockerfile.gpu
FROM pytorch/pytorch:2.0.1-cuda11.7-cudnn8-runtime

WORKDIR /app

# Install dependencies
COPY requirements.txt .
RUN pip install -r requirements.txt

# Download model
RUN python -c "import demucs; demucs.pretrained.get_model('htdemucs')"

# Copy API server
COPY server.py .

EXPOSE 8000
CMD ["uvicorn", "server:app", "--host", "0.0.0.0", "--port", "8000"]

// model-router.ts
interface ModelRequest {
  task: 'stems' | 'generate' | 'transcribe' | 'tts' | 'classify';
  priority: 'realtime' | 'background';
  input: any;
}

async function routeModel(request: ModelRequest): Promise<any> {
  const { task, priority } = request;

  // TIER 1: Cloudflare Workers AI (realtime, free tier)
  if (task === 'transcribe' && priority === 'realtime') {
    return workersAI.whisper(request.input);
  }

  if (task === 'classify') {
    // Client-side with Transformers.js
    return clientSideClassify(request.input);
  }

  // TIER 2: Self-hosted (realtime, quality)
  if (task === 'tts') {
    return flyIO.chatterbox(request.input);
  }

  // TIER 3: Replicate (background, heavy compute)
  if (task === 'stems') {
    return replicate.htdemucs(request.input);
  }

  if (task === 'generate') {
    return replicate.musicgen(request.input);
  }

  throw new Error(`Unknown task: ${task}`);
}