mirror of
https://github.com/hrydgard/ppsspp.git
synced 2026-10-01 14:58:14 +00:00
Henrik RydgårdandClaude Opus 5.5
9d33e828ab
The PSP camera keeps JPEG frames within the framesize from the video setup. Go!Edit stores frames in 15KB slots and only takes frames that fit, so our uncompressed-quality webcam frames were dropped and one frame got repeated for the whole clip. Lower the JPEG quality until a frame fits. Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
1065 lines
30 KiB
C++
1065 lines
30 KiB
C++
// Copyright (c) 2020- PPSSPP Project.
|
|
|
|
// This program is free software: you can redistribute it and/or modify
|
|
// it under the terms of the GNU General Public License as published by
|
|
// the Free Software Foundation, version 2.0 or later versions.
|
|
|
|
// This program is distributed in the hope that it will be useful,
|
|
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
// GNU General Public License 2.0 for more details.
|
|
|
|
// A copy of the GPL 2.0 should have been included with the program.
|
|
// If not, see http://www.gnu.org/licenses/
|
|
|
|
// Official git repository and contact information can be found at
|
|
// https://github.com/hrydgard/ppsspp and http://www.ppsspp.org/.
|
|
|
|
#include <shlwapi.h>
|
|
#include <wrl/client.h>
|
|
|
|
#include "mfapi.h"
|
|
|
|
#include "Common/Thread/ThreadUtil.h"
|
|
#include "CaptureDevice.h"
|
|
#include "BufferLock.h"
|
|
#include "ext/jpge/jpge.h"
|
|
#include "CommonTypes.h"
|
|
#include "Core/HLE/sceUsbCam.h"
|
|
#include "Core/Config.h"
|
|
|
|
using Microsoft::WRL::ComPtr;
|
|
|
|
namespace MFAPI {
|
|
HINSTANCE Mflib;
|
|
HINSTANCE Mfplatlib;
|
|
HINSTANCE Mfreadwritelib;
|
|
|
|
// NOTE: MFGetAttributeSize is an inline function, so we don't need to load it dynamically.
|
|
|
|
typedef HRESULT(WINAPI *MFEnumDeviceSourcesFunc)(IMFAttributes *, IMFActivate ***, UINT32 *);
|
|
typedef HRESULT(WINAPI *MFStartupFunc)(ULONG, DWORD);
|
|
typedef HRESULT(WINAPI *MFShutdownFunc)();
|
|
typedef HRESULT(WINAPI *MFCreateAttributesFunc)(IMFAttributes **, UINT32);
|
|
typedef HRESULT(WINAPI *MFGetStrideForBitmapInfoHeaderFunc)(DWORD, DWORD, LONG *);
|
|
typedef HRESULT(WINAPI *MFCreateSourceReaderFromMediaSourceFunc)(IMFMediaSource *, IMFAttributes *, IMFSourceReader **);
|
|
typedef HRESULT(WINAPI *MFCopyImageFunc)(BYTE *, LONG, const BYTE *, LONG, DWORD, DWORD);
|
|
|
|
MFEnumDeviceSourcesFunc EnumDeviceSources;
|
|
MFStartupFunc Startup;
|
|
MFShutdownFunc ShutdownFn;
|
|
MFCreateAttributesFunc CreateAttributes;
|
|
MFGetStrideForBitmapInfoHeaderFunc GetStrideForBitmapInfoHeader;
|
|
MFCreateSourceReaderFromMediaSourceFunc CreateSourceReaderFromMediaSource;
|
|
MFCopyImageFunc CopyImage;
|
|
}
|
|
|
|
|
|
bool RegisterCMPTMFApis() {
|
|
// We dynamically load these, as they are not always available (de-bloated Windows, Windows Vista, etc).
|
|
|
|
MFAPI::Mflib = LoadLibrary(L"Mf.dll");
|
|
if (!MFAPI::Mflib) {
|
|
ERROR_LOG(Log::System, "Failed to load Mflib.");
|
|
return false;
|
|
}
|
|
MFAPI::Mfplatlib = LoadLibrary(L"Mfplat.dll");
|
|
if (!MFAPI::Mfplatlib) {
|
|
ERROR_LOG(Log::System, "Failed to load Mfplat.");
|
|
return false;
|
|
}
|
|
MFAPI::Mfreadwritelib = LoadLibrary(L"Mfreadwrite.dll");
|
|
if (!MFAPI::Mfreadwritelib) {
|
|
ERROR_LOG(Log::System, "Failed to load Mfreadwrite.");
|
|
return false;
|
|
}
|
|
|
|
MFAPI::EnumDeviceSources = (MFAPI::MFEnumDeviceSourcesFunc)GetProcAddress(MFAPI::Mflib, "MFEnumDeviceSources");
|
|
MFAPI::Startup = (MFAPI::MFStartupFunc)GetProcAddress(MFAPI::Mfplatlib, "MFStartup");
|
|
MFAPI::ShutdownFn = (MFAPI::MFShutdownFunc)GetProcAddress(MFAPI::Mfplatlib, "MFShutdown");
|
|
MFAPI::CreateAttributes = (MFAPI::MFCreateAttributesFunc)GetProcAddress(MFAPI::Mfplatlib, "MFCreateAttributes");
|
|
MFAPI::GetStrideForBitmapInfoHeader = (MFAPI::MFGetStrideForBitmapInfoHeaderFunc)GetProcAddress(MFAPI::Mfplatlib, "MFGetStrideForBitmapInfoHeader");
|
|
MFAPI::CopyImage = (MFAPI::MFCopyImageFunc)GetProcAddress(MFAPI::Mfplatlib, "MFCopyImage");
|
|
MFAPI::CreateSourceReaderFromMediaSource = (MFAPI::MFCreateSourceReaderFromMediaSourceFunc)GetProcAddress(MFAPI::Mfreadwritelib, "MFCreateSourceReaderFromMediaSource");
|
|
if (!MFAPI::EnumDeviceSources || !MFAPI::Startup || !MFAPI::ShutdownFn || !MFAPI::CreateAttributes || !MFAPI::GetStrideForBitmapInfoHeader || !MFAPI::CreateSourceReaderFromMediaSource || !MFAPI::CopyImage) {
|
|
return false;
|
|
}
|
|
|
|
return true;
|
|
}
|
|
|
|
bool UnRegisterCMPTMFApis() {
|
|
if (MFAPI::Mflib) {
|
|
FreeLibrary(MFAPI::Mflib);
|
|
MFAPI::Mflib = nullptr;
|
|
}
|
|
|
|
if (MFAPI::Mfplatlib) {
|
|
FreeLibrary(MFAPI::Mfplatlib);
|
|
MFAPI::Mfplatlib = nullptr;
|
|
}
|
|
|
|
if (MFAPI::Mfreadwritelib) {
|
|
FreeLibrary(MFAPI::Mfreadwritelib);
|
|
MFAPI::Mfreadwritelib = nullptr;
|
|
}
|
|
|
|
MFAPI::EnumDeviceSources = nullptr;
|
|
MFAPI::Startup = nullptr;
|
|
MFAPI::ShutdownFn = nullptr;
|
|
MFAPI::CreateAttributes = nullptr;
|
|
MFAPI::GetStrideForBitmapInfoHeader = nullptr;
|
|
MFAPI::CreateSourceReaderFromMediaSource = nullptr;
|
|
MFAPI::CopyImage = nullptr;
|
|
return true;
|
|
}
|
|
|
|
WindowsCaptureDevice *winCamera;
|
|
WindowsCaptureDevice *winMic;
|
|
// TODO: Add more formats, but need some tests.
|
|
VideoFormatTransform g_VideoFormats[] =
|
|
{
|
|
{ MFVideoFormat_RGB32, AV_PIX_FMT_RGBA },
|
|
{ MFVideoFormat_RGB24, AV_PIX_FMT_RGB24 },
|
|
{ MFVideoFormat_YUY2, AV_PIX_FMT_YUYV422 },
|
|
{ MFVideoFormat_NV12, AV_PIX_FMT_NV12 }
|
|
};
|
|
|
|
AudioFormatTransform g_AudioFormats[] = {
|
|
{ MFAudioFormat_PCM, 8, AV_SAMPLE_FMT_U8 },
|
|
{ MFAudioFormat_PCM, 16, AV_SAMPLE_FMT_S16 },
|
|
{ MFAudioFormat_PCM, 32, AV_SAMPLE_FMT_S32 },
|
|
{ MFAudioFormat_Float, 32, AV_SAMPLE_FMT_FLT }
|
|
};
|
|
|
|
const int g_cVideoFormats = ARRAYSIZE(g_VideoFormats);
|
|
const int g_cAudioFormats = ARRAYSIZE(g_AudioFormats);
|
|
|
|
MediaParam defaultVideoParam = { { 640, 480, 0, MFVideoFormat_RGB24 } };
|
|
MediaParam defaultAudioParam = { { 44100, 2, 16, MFAudioFormat_PCM } };
|
|
|
|
HRESULT GetDefaultStride(IMFMediaType *pType, LONG *plStride);
|
|
|
|
ReaderCallback::ReaderCallback(WindowsCaptureDevice *_device) : device(_device) {}
|
|
|
|
ReaderCallback::~ReaderCallback() {
|
|
#ifdef USE_FFMPEG
|
|
if (img_convert_ctx) {
|
|
sws_freeContext(img_convert_ctx);
|
|
}
|
|
if (resample_ctx) {
|
|
swr_free(&resample_ctx);
|
|
}
|
|
#endif
|
|
}
|
|
|
|
HRESULT ReaderCallback::QueryInterface(REFIID riid, void** ppv)
|
|
{
|
|
static const QITAB qit[] =
|
|
{
|
|
QITABENT(ReaderCallback, IMFSourceReaderCallback),
|
|
{ 0 },
|
|
};
|
|
return QISearch(this, qit, riid, ppv);
|
|
}
|
|
|
|
HRESULT ReaderCallback::OnReadSample(
|
|
HRESULT hrStatus,
|
|
DWORD dwStreamIndex,
|
|
DWORD dwStreamFlags,
|
|
LONGLONG llTimestamp,
|
|
IMFSample *pSample) {
|
|
HRESULT hr = S_OK;
|
|
ComPtr<IMFMediaBuffer> pBuffer;
|
|
std::lock_guard<std::mutex> lock(device->sdMutex);
|
|
if (device->isShutDown())
|
|
return hr;
|
|
|
|
if (FAILED(hrStatus))
|
|
hr = hrStatus;
|
|
|
|
if (SUCCEEDED(hr)) {
|
|
if (pSample) {
|
|
hr = pSample->GetBufferByIndex(0, &pBuffer);
|
|
}
|
|
}
|
|
if (SUCCEEDED(hr)) {
|
|
switch (device->type) {
|
|
case CAPTUREDEVICE_TYPE::VIDEO: {
|
|
BYTE *pbScanline0 = nullptr;
|
|
VideoBufferLock *videoBuffer = nullptr;
|
|
int imgJpegSize = device->imgJpegSize;
|
|
unsigned char* invertedSrcImg = nullptr;
|
|
LONG srcPadding = 0;
|
|
LONG lStride = 0;
|
|
|
|
UINT32 srcW = device->deviceParam.width;
|
|
UINT32 srcH = device->deviceParam.height;
|
|
UINT32 dstW = device->targetMediaParam.width;
|
|
UINT32 dstH = device->targetMediaParam.height;
|
|
GUID srcMFVideoFormat = device->deviceParam.videoFormat;
|
|
|
|
// pSample can be null, in this case ReadSample still should be called to request next frame.
|
|
if (pSample) {
|
|
videoBuffer = new VideoBufferLock(pBuffer.Get());
|
|
hr = videoBuffer->LockBuffer(device->deviceParam.default_stride, device->deviceParam.height, &pbScanline0, &lStride);
|
|
|
|
if (lStride > 0)
|
|
srcPadding = lStride - device->deviceParam.default_stride;
|
|
else
|
|
srcPadding = device->deviceParam.default_stride - lStride;
|
|
|
|
#ifdef USE_FFMPEG
|
|
if (SUCCEEDED(hr)) {
|
|
// Convert image to RGB24
|
|
if (lStride > 0) {
|
|
imgConvert(
|
|
device->imageRGB, dstW, dstH, device->imgRGBLineSizes,
|
|
pbScanline0, srcW, srcH, srcMFVideoFormat, srcPadding);
|
|
} else {
|
|
// If stride < 0, the pointer to the first row of source image is the last row in memory,should invert it in memory.
|
|
invertedSrcImg = (unsigned char*)av_malloc(av_image_get_buffer_size(getAVVideoFormatbyMFVideoFormat(srcMFVideoFormat), srcW, srcH, 1));
|
|
imgInvert(invertedSrcImg, pbScanline0, srcW, srcH, device->deviceParam.videoFormat, lStride);
|
|
// We alloc a inverted image with no padding, set padding to zero.
|
|
srcPadding = 0;
|
|
imgConvert(
|
|
device->imageRGB, dstW, dstH, device->imgRGBLineSizes,
|
|
invertedSrcImg, srcW, srcH, srcMFVideoFormat, srcPadding);
|
|
av_free(invertedSrcImg);
|
|
}
|
|
|
|
// Mirror the image in-place if needed.
|
|
if (g_Config.bCameraMirrorHorizontal) {
|
|
for (int y = 0; y < (int)dstH; y++) {
|
|
uint8_t *line = device->imageRGB + y * device->imgRGBLineSizes[0];
|
|
for (int x = 0; x < (int)dstW / 2; x++) {
|
|
const int invX = dstW - 1 - x;
|
|
const uint8_t r = line[x * 3 + 0];
|
|
const uint8_t g = line[x * 3 + 1];
|
|
const uint8_t b = line[x * 3 + 2];
|
|
line[x * 3 + 0] = line[invX * 3 + 0];
|
|
line[x * 3 + 1] = line[invX * 3 + 1];
|
|
line[x * 3 + 2] = line[invX * 3 + 2];
|
|
line[invX * 3 + 0] = r;
|
|
line[invX * 3 + 1] = g;
|
|
line[invX * 3 + 2] = b;
|
|
}
|
|
}
|
|
}
|
|
|
|
// Compress image to jpeg from RGB24. Like the PSP camera, lower the quality until the frame
|
|
// fits the size the game asked for: Go!Edit stores frames in fixed 15KB slots.
|
|
const int maxFrameSize = Camera::getMaxFrameSize();
|
|
jpge::params params;
|
|
for (int quality = 85; ; quality -= 10) {
|
|
imgJpegSize = device->imgJpegSize;
|
|
params.m_quality = quality;
|
|
jpge::compress_image_to_jpeg_file_in_memory(
|
|
device->imageJpeg, imgJpegSize,
|
|
dstW,
|
|
dstH,
|
|
3,
|
|
device->imageRGB, params);
|
|
if (imgJpegSize <= maxFrameSize || quality <= 15) {
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
#endif
|
|
Camera::pushCameraImage(imgJpegSize, device->imageJpeg);
|
|
}
|
|
// Request the next frame.
|
|
if (SUCCEEDED(hr)) {
|
|
hr = device->m_pReader->ReadSample(
|
|
(DWORD)MF_SOURCE_READER_FIRST_VIDEO_STREAM,
|
|
0,
|
|
nullptr,
|
|
nullptr,
|
|
nullptr,
|
|
nullptr
|
|
);
|
|
}
|
|
delete videoBuffer;
|
|
break;
|
|
}
|
|
case CAPTUREDEVICE_TYPE::AUDIO: {
|
|
BYTE *sampleBuf = nullptr;
|
|
DWORD length = 0;
|
|
u32 sizeAfterResample = 0;
|
|
// pSample can be null, in this case ReadSample still should be called to request next frame.
|
|
if (pSample) {
|
|
pBuffer->Lock(&sampleBuf, nullptr, &length);
|
|
if (device->needResample()) {
|
|
sizeAfterResample = doResample(
|
|
&device->resampleBuf, device->targetMediaParam.sampleRate, device->targetMediaParam.channels, &device->resampleBufSize,
|
|
sampleBuf, device->deviceParam.sampleRate, device->deviceParam.channels, device->deviceParam.audioFormat, length, device->deviceParam.bitsPerSample);
|
|
if (device->resampleBuf)
|
|
Microphone::addAudioData(device->resampleBuf, sizeAfterResample);
|
|
} else {
|
|
Microphone::addAudioData(sampleBuf, length);
|
|
}
|
|
pBuffer->Unlock();
|
|
}
|
|
// Request the next frame.
|
|
if (SUCCEEDED(hr)) {
|
|
hr = device->m_pReader->ReadSample(
|
|
(DWORD)MF_SOURCE_READER_FIRST_AUDIO_STREAM,
|
|
0,
|
|
nullptr,
|
|
nullptr,
|
|
nullptr,
|
|
nullptr
|
|
);
|
|
}
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
|
|
return hr;
|
|
}
|
|
|
|
AVPixelFormat ReaderCallback::getAVVideoFormatbyMFVideoFormat(const GUID &MFVideoFormat) {
|
|
for (int i = 0; i < g_cVideoFormats; i++) {
|
|
if (MFVideoFormat == g_VideoFormats[i].MFVideoFormat)
|
|
return g_VideoFormats[i].AVVideoFormat;
|
|
}
|
|
return AV_PIX_FMT_RGB24;
|
|
}
|
|
|
|
AVSampleFormat ReaderCallback::getAVAudioFormatbyMFAudioFormat(const GUID &MFAudioFormat, const u32 &bitsPerSample) {
|
|
for (int i = 0; i < g_cAudioFormats; i++) {
|
|
if (MFAudioFormat == g_AudioFormats[i].MFAudioFormat && bitsPerSample == g_AudioFormats[i].bitsPerSample)
|
|
return g_AudioFormats[i].AVAudioFormat;
|
|
}
|
|
return AV_SAMPLE_FMT_S16;
|
|
}
|
|
|
|
void ReaderCallback::imgConvert(
|
|
unsigned char *dst, unsigned int &dstW, unsigned int &dstH, int dstLineSizes[4],
|
|
unsigned char *src, const unsigned int &srcW, const unsigned int &srcH, const GUID &srcFormat,
|
|
const int &srcPadding) {
|
|
#ifdef USE_FFMPEG
|
|
int srcLineSizes[4] = { 0, 0, 0, 0 };
|
|
unsigned char *pSrc[4];
|
|
unsigned char *pDst[4];
|
|
|
|
AVPixelFormat srcAVFormat = getAVVideoFormatbyMFVideoFormat(srcFormat);
|
|
|
|
|
|
av_image_fill_linesizes(srcLineSizes, srcAVFormat, srcW);
|
|
|
|
// Is this correct?
|
|
if (srcPadding != 0) {
|
|
for (int i = 0; i < 4; i++) {
|
|
if (srcLineSizes[i] != 0)
|
|
srcLineSizes[i] += srcPadding;
|
|
}
|
|
}
|
|
|
|
av_image_fill_pointers(pSrc, srcAVFormat, srcH, src, srcLineSizes);
|
|
av_image_fill_pointers(pDst, AV_PIX_FMT_RGB24, dstH, dst, dstLineSizes);
|
|
|
|
if (img_convert_ctx == nullptr) {
|
|
img_convert_ctx = sws_getContext(
|
|
srcW,
|
|
srcH,
|
|
srcAVFormat,
|
|
dstW,
|
|
dstH,
|
|
AV_PIX_FMT_RGB24,
|
|
SWS_BICUBIC,
|
|
nullptr,
|
|
nullptr,
|
|
nullptr
|
|
);
|
|
}
|
|
|
|
if (img_convert_ctx) {
|
|
sws_scale(img_convert_ctx,
|
|
(const uint8_t *const *)pSrc,
|
|
srcLineSizes,
|
|
0,
|
|
srcH,
|
|
(uint8_t *const *)pDst,
|
|
dstLineSizes
|
|
);
|
|
}
|
|
#endif
|
|
}
|
|
|
|
void ReaderCallback::imgInvert(unsigned char *dst, unsigned char *src, const int &srcW, const int &srcH, const GUID &srcFormat, const int &srcStride) {
|
|
#ifdef USE_FFMPEG
|
|
AVPixelFormat srcAvFormat = getAVVideoFormatbyMFVideoFormat(srcFormat);
|
|
int dstLineSizes[4] = { 0, 0, 0, 0 };
|
|
|
|
av_image_fill_linesizes(dstLineSizes, srcAvFormat, srcW);
|
|
|
|
if(srcFormat == MFVideoFormat_RGB32)
|
|
imgInvertRGBA(dst, dstLineSizes[0], src, srcStride, srcH);
|
|
else if(srcFormat == MFVideoFormat_RGB24)
|
|
imgInvertRGB(dst, dstLineSizes[0], src, srcStride, srcH);
|
|
else if (srcFormat == MFVideoFormat_YUY2)
|
|
imgInvertYUY2(dst, dstLineSizes[0], src, srcStride, srcH);
|
|
else if (srcFormat == MFVideoFormat_NV12)
|
|
imgInvertNV12(dst, dstLineSizes[0], src, srcStride, srcH);;
|
|
#endif
|
|
}
|
|
|
|
void ReaderCallback::imgInvertRGBA(unsigned char *dst, int &dstStride, unsigned char *src, const int &srcStride, const int &h) {
|
|
MFAPI::CopyImage(dst, dstStride, src, srcStride, dstStride, h);
|
|
}
|
|
|
|
void ReaderCallback::imgInvertRGB(unsigned char *dst, int &dstStride, unsigned char *src, const int &srcStride, const int &h) {
|
|
for (int y = 0; y < h; y++) {
|
|
for (int srcx = dstStride - 1, dstx = 0; dstx < dstStride; srcx--, dstx++) {
|
|
dst[dstx] = src[srcx];
|
|
}
|
|
dst += dstStride;
|
|
src += srcStride;
|
|
}
|
|
}
|
|
|
|
void ReaderCallback::imgInvertYUY2(unsigned char *dst, int &dstStride, unsigned char *src, const int &srcStride, const int &h) {
|
|
for (int y = 0; y < h; y++) {
|
|
for (int srcx = dstStride - 1, dstx = 0; dstx < dstStride; srcx--, dstx++) {
|
|
dst[dstx] = src[srcx];
|
|
}
|
|
dst += dstStride;
|
|
src += srcStride;
|
|
}
|
|
}
|
|
|
|
void ReaderCallback::imgInvertNV12(unsigned char *dst, int &dstStride, unsigned char *src, const int &srcStride, const int &h) {
|
|
unsigned char *dstY = dst;
|
|
unsigned char *dstU = dst + dstStride * h;
|
|
unsigned char *srcY = src;
|
|
unsigned char *srcV = src + srcStride * h;
|
|
|
|
unsigned char *srcY1 = srcY;
|
|
unsigned char *srcY2 = srcY1 + srcStride;
|
|
unsigned char *dstY1 = dstY;
|
|
unsigned char *dstY2 = dstY1 + dstStride;
|
|
|
|
bool isodd = h % 2 != 0;
|
|
|
|
for (int y = 0; y < (isodd ? h - 1 : h); y += 2) {
|
|
for (int srcx = dstStride - 1, dstx = 0; dstx < dstStride; srcx--, dstx++) {
|
|
dstY1[dstx] = srcY1[srcx];
|
|
dstY2[dstx] = srcY2[srcx];
|
|
dstU[dstx] = srcV[srcx];
|
|
}
|
|
dstY += dstStride * 2;
|
|
srcY1 += srcStride * 2;
|
|
srcY2 += srcStride *2;
|
|
srcV += srcStride;
|
|
dstU += dstStride;
|
|
}
|
|
}
|
|
|
|
u32 ReaderCallback::doResample(u8 **dst, u32 &dstSampleRate, u32 &dstChannels, u32 *dstSize, u8 *src, const u32 &srcSampleRate, const u32 &srcChannels, const GUID &srcFormat, const u32& srcSize, const u32& srcBitsPerSample) {
|
|
#ifdef USE_FFMPEG
|
|
AVSampleFormat srcAVFormat = getAVAudioFormatbyMFAudioFormat(srcFormat, srcBitsPerSample);
|
|
int outSamplesCount = 0;
|
|
if (resample_ctx == nullptr) {
|
|
resample_ctx = swr_alloc_set_opts(nullptr,
|
|
av_get_default_channel_layout(dstChannels),
|
|
AV_SAMPLE_FMT_S16,
|
|
dstSampleRate,
|
|
av_get_default_channel_layout(srcChannels),
|
|
srcAVFormat,
|
|
srcSampleRate,
|
|
0,
|
|
nullptr);
|
|
if (resample_ctx == nullptr || swr_init(resample_ctx) < 0) {
|
|
swr_free(&resample_ctx);
|
|
return 0;
|
|
}
|
|
}
|
|
int srcSamplesCount = srcSize / srcChannels / av_get_bytes_per_sample(srcAVFormat); // per channel.
|
|
int outCount = srcSamplesCount * dstSampleRate / srcSampleRate + 256;
|
|
unsigned int outSize = av_samples_get_buffer_size(nullptr, dstChannels, outCount, AV_SAMPLE_FMT_S16, 0);
|
|
|
|
if (!*dst) {
|
|
*dst = (u8 *)av_malloc(outSize);
|
|
*dstSize = outSize;
|
|
}
|
|
if (!*dst)
|
|
return 0;
|
|
|
|
if(*dstSize < outSize)
|
|
av_fast_malloc(dst, dstSize, outSize);
|
|
|
|
outSamplesCount = swr_convert(resample_ctx, dst, outCount, (const uint8_t **)&src, srcSamplesCount);
|
|
if (outSamplesCount < 0)
|
|
return 0;
|
|
return av_samples_get_buffer_size(nullptr, dstChannels, outSamplesCount, AV_SAMPLE_FMT_S16, 0);
|
|
#else
|
|
return 0;
|
|
#endif
|
|
}
|
|
|
|
WindowsCaptureDevice::WindowsCaptureDevice(CAPTUREDEVICE_TYPE _type) :
|
|
type(_type),
|
|
error(CAPTUREDEVIDE_ERROR_NO_ERROR),
|
|
state(CAPTUREDEVICE_STATE::UNINITIALIZED) {
|
|
param = { 0 };
|
|
deviceParam = { { 0 } };
|
|
|
|
switch (type) {
|
|
case CAPTUREDEVICE_TYPE::VIDEO:
|
|
targetMediaParam = defaultVideoParam;
|
|
break;
|
|
case CAPTUREDEVICE_TYPE::AUDIO:
|
|
targetMediaParam = defaultAudioParam;
|
|
break;
|
|
}
|
|
|
|
thread_ = std::thread(&WindowsCaptureDevice::messageHandler, this);
|
|
}
|
|
|
|
WindowsCaptureDevice::~WindowsCaptureDevice() {
|
|
#ifdef USE_FFMPEG
|
|
switch (type) {
|
|
case CAPTUREDEVICE_TYPE::VIDEO:
|
|
av_freep(&imageRGB);
|
|
av_freep(&imageJpeg);
|
|
break;
|
|
case CAPTUREDEVICE_TYPE::AUDIO:
|
|
av_freep(&resampleBuf);
|
|
break;
|
|
}
|
|
#endif
|
|
}
|
|
|
|
void WindowsCaptureDevice::CheckDevices() {
|
|
isDeviceChanged = true;
|
|
}
|
|
|
|
bool WindowsCaptureDevice::start(void *startParam) {
|
|
HRESULT hr = S_OK;
|
|
ComPtr<IMFAttributes> pAttributes;
|
|
ComPtr<IMFMediaType> pType;
|
|
UINT32 selection = 0;
|
|
UINT32 count = 0;
|
|
|
|
// Release old sources first(if any).
|
|
m_pSource = nullptr;
|
|
m_pReader = nullptr;
|
|
m_pCallback = nullptr;
|
|
// Need to re-enumerate the list,because old sources were released.
|
|
std::vector<std::string> deviceList = getDeviceList(true);
|
|
|
|
if (deviceList.size() < 1) {
|
|
setError(CAPTUREDEVIDE_ERROR_START_FAILED, "Has no device");
|
|
return false;
|
|
}
|
|
|
|
m_pCallback = new ReaderCallback(this);
|
|
|
|
std::string selectedDeviceName = type == CAPTUREDEVICE_TYPE::VIDEO ? g_Config.sCameraDevice : g_Config.sMicDevice;
|
|
|
|
switch (state) {
|
|
case CAPTUREDEVICE_STATE::STOPPED:
|
|
for (auto &name : deviceList) {
|
|
if (name == selectedDeviceName) {
|
|
selection = count;
|
|
break;
|
|
}
|
|
++count;
|
|
}
|
|
setSelection(selection);
|
|
hr = param.ppDevices[param.selection]->ActivateObject(
|
|
IID_PPV_ARGS(&m_pSource));
|
|
|
|
if (SUCCEEDED(hr))
|
|
hr = MFAPI::CreateAttributes(&pAttributes, 2);
|
|
|
|
// Use async mode
|
|
if (SUCCEEDED(hr))
|
|
hr = pAttributes->SetUnknown(MF_SOURCE_READER_ASYNC_CALLBACK, m_pCallback.Get());
|
|
|
|
if (SUCCEEDED(hr))
|
|
hr = pAttributes->SetUINT32(MF_READWRITE_DISABLE_CONVERTERS, TRUE);
|
|
|
|
if (SUCCEEDED(hr)) {
|
|
hr = MFAPI::CreateSourceReaderFromMediaSource(
|
|
m_pSource.Get(),
|
|
pAttributes.Get(),
|
|
&m_pReader
|
|
);
|
|
}
|
|
|
|
if (!m_pReader)
|
|
hr = E_FAIL;
|
|
|
|
if (SUCCEEDED(hr)) {
|
|
switch (type) {
|
|
case CAPTUREDEVICE_TYPE::VIDEO: {
|
|
if (startParam) {
|
|
std::vector<int> *resolution = static_cast<std::vector<int>*>(startParam);
|
|
targetMediaParam.width = resolution->at(0);
|
|
targetMediaParam.height = resolution->at(1);
|
|
delete resolution;
|
|
}
|
|
#ifdef USE_FFMPEG
|
|
|
|
av_freep(&imageRGB);
|
|
av_freep(&imageJpeg);
|
|
imageRGB = (unsigned char*)av_malloc(av_image_get_buffer_size(AV_PIX_FMT_RGB24, targetMediaParam.width, targetMediaParam.height, 1));
|
|
imgJpegSize = av_image_get_buffer_size(AV_PIX_FMT_YUVJ411P, targetMediaParam.width, targetMediaParam.height, 1);
|
|
imageJpeg = (unsigned char*)av_malloc(imgJpegSize);
|
|
av_image_fill_linesizes(imgRGBLineSizes, AV_PIX_FMT_RGB24, targetMediaParam.width);
|
|
#endif
|
|
|
|
for (DWORD i = 0; ; i++) {
|
|
hr = m_pReader->GetNativeMediaType(
|
|
(DWORD)MF_SOURCE_READER_FIRST_VIDEO_STREAM,
|
|
i,
|
|
&pType
|
|
);
|
|
|
|
if (FAILED(hr)) { break; }
|
|
|
|
hr = setDeviceParam(pType.Get());
|
|
|
|
if (SUCCEEDED(hr))
|
|
break;
|
|
}
|
|
/*
|
|
hr = m_pReader->GetNativeMediaType(
|
|
(DWORD)MF_SOURCE_READER_FIRST_VIDEO_STREAM,
|
|
(DWORD)0xFFFFFFFF,//MF_SOURCE_READER_CURRENT_TYPE_INDEX
|
|
&pType
|
|
);
|
|
if (SUCCEEDED(hr))
|
|
hr = setDeviceParam(pType);*/ // Don't support on Win7
|
|
|
|
// Request the first frame, in async mode, OnReadSample will be called when ReadSample completed.
|
|
if (SUCCEEDED(hr)) {
|
|
hr = m_pReader->ReadSample(
|
|
(DWORD)MF_SOURCE_READER_FIRST_VIDEO_STREAM,
|
|
0,
|
|
nullptr,
|
|
nullptr,
|
|
nullptr,
|
|
nullptr
|
|
);
|
|
}
|
|
break;
|
|
}
|
|
|
|
case CAPTUREDEVICE_TYPE::AUDIO: {
|
|
if (startParam) {
|
|
std::vector<u32> *micParam = static_cast<std::vector<u32>*>(startParam);
|
|
targetMediaParam.sampleRate = micParam->at(0);
|
|
targetMediaParam.channels = micParam->at(1);
|
|
delete micParam;
|
|
}
|
|
|
|
for (DWORD i = 0; ; i++) {
|
|
hr = m_pReader->GetNativeMediaType(
|
|
(DWORD)MF_SOURCE_READER_FIRST_AUDIO_STREAM,
|
|
i,
|
|
&pType
|
|
);
|
|
|
|
if (FAILED(hr)) { break; }
|
|
|
|
hr = setDeviceParam(pType.Get());
|
|
|
|
if (SUCCEEDED(hr))
|
|
break;
|
|
}
|
|
|
|
if (SUCCEEDED(hr)) {
|
|
hr = m_pReader->ReadSample(
|
|
(DWORD)MF_SOURCE_READER_FIRST_AUDIO_STREAM,
|
|
0,
|
|
nullptr,
|
|
nullptr,
|
|
nullptr,
|
|
nullptr
|
|
);
|
|
}
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
|
|
if (FAILED(hr)) {
|
|
setError(CAPTUREDEVIDE_ERROR_START_FAILED, "Cannot start");
|
|
if(m_pSource)
|
|
m_pSource->Shutdown();
|
|
m_pSource = nullptr;
|
|
m_pReader = nullptr;
|
|
return false;
|
|
}
|
|
|
|
updateState(CAPTUREDEVICE_STATE::STARTED);
|
|
break;
|
|
case CAPTUREDEVICE_STATE::LOST:
|
|
setError(CAPTUREDEVIDE_ERROR_START_FAILED, "Device has lost");
|
|
return false;
|
|
case CAPTUREDEVICE_STATE::STARTED:
|
|
setError(CAPTUREDEVIDE_ERROR_START_FAILED, "Device has started");
|
|
return false;
|
|
case CAPTUREDEVICE_STATE::UNINITIALIZED:
|
|
setError(CAPTUREDEVIDE_ERROR_START_FAILED, "Device doesn't initialize");
|
|
return false;
|
|
default:
|
|
break;
|
|
}
|
|
return true;
|
|
}
|
|
|
|
bool WindowsCaptureDevice::stop() {
|
|
if (state == CAPTUREDEVICE_STATE::STOPPED)
|
|
return true;
|
|
if (m_pSource)
|
|
m_pSource->Stop();
|
|
|
|
updateState(CAPTUREDEVICE_STATE::STOPPED);
|
|
|
|
return true;
|
|
};
|
|
|
|
std::vector<std::string> WindowsCaptureDevice::getDeviceList(bool forceEnum, int *pActuallCount) {
|
|
HRESULT hr = S_OK;
|
|
UINT32 count = 0;
|
|
LPWSTR pwstrName = nullptr;
|
|
char *cstrName = nullptr;
|
|
std::string strName;
|
|
DWORD dwMinSize = 0;
|
|
std::vector<std::string> deviceList;
|
|
|
|
if (isDeviceChanged || forceEnum) {
|
|
std::unique_lock<std::mutex> lock(paramMutex);
|
|
for (DWORD i = 0; i < param.count; i++) {
|
|
SafeRelease(¶m.ppDevices[i]);
|
|
}
|
|
CoTaskMemFree(param.ppDevices); // Null pointer is okay.
|
|
|
|
hr = enumDevices();
|
|
|
|
lock.unlock();
|
|
|
|
if (SUCCEEDED(hr))
|
|
isDeviceChanged = false;
|
|
else
|
|
return deviceList;
|
|
}
|
|
|
|
for (; count < param.count; count++) {
|
|
hr = param.ppDevices[count]->GetAllocatedString(
|
|
MF_DEVSOURCE_ATTRIBUTE_FRIENDLY_NAME,
|
|
&pwstrName,
|
|
nullptr
|
|
);
|
|
|
|
if (SUCCEEDED(hr)) {
|
|
// Get the size needed first
|
|
dwMinSize = WideCharToMultiByte(CP_UTF8, 0, pwstrName, -1, nullptr, 0, nullptr, FALSE);
|
|
if (dwMinSize == 0)
|
|
hr = E_FAIL;
|
|
}
|
|
if (SUCCEEDED(hr)) {
|
|
cstrName = new char[dwMinSize];
|
|
WideCharToMultiByte(CP_UTF8, 0, pwstrName, -1, cstrName, dwMinSize, NULL, FALSE);
|
|
strName = cstrName;
|
|
delete[] cstrName;
|
|
|
|
deviceList.push_back(strName);
|
|
}
|
|
|
|
CoTaskMemFree(pwstrName);
|
|
|
|
if (FAILED(hr)) {
|
|
setError(CAPTUREDEVIDE_ERROR_GETNAMES_FAILED, "Error occurred,gotten " + std::to_string((int)count) + " device names");
|
|
if(pActuallCount)
|
|
*pActuallCount = count;
|
|
return deviceList;
|
|
}
|
|
}
|
|
if (pActuallCount)
|
|
*pActuallCount = count + 1;
|
|
return deviceList;
|
|
}
|
|
|
|
HRESULT WindowsCaptureDevice::setDeviceParam(IMFMediaType *pType) {
|
|
HRESULT hr = S_OK;
|
|
GUID subtype = { 0 };
|
|
bool getFormat = false;
|
|
|
|
switch (type) {
|
|
case CAPTUREDEVICE_TYPE::VIDEO:
|
|
hr = pType->GetGUID(MF_MT_SUBTYPE, &subtype);
|
|
if (FAILED(hr))
|
|
break;
|
|
|
|
for (int i = 0; i < g_cVideoFormats; i++) {
|
|
if (subtype == g_VideoFormats[i].MFVideoFormat) {
|
|
deviceParam.videoFormat = subtype;
|
|
getFormat = true;
|
|
break;
|
|
}
|
|
}
|
|
|
|
if (!getFormat) {
|
|
for (int i = 0; i < g_cVideoFormats; i++) {
|
|
hr = pType->SetGUID(MF_MT_SUBTYPE, g_VideoFormats[i].MFVideoFormat);
|
|
if (FAILED(hr))
|
|
continue;
|
|
|
|
hr = m_pReader->SetCurrentMediaType(
|
|
(DWORD)MF_SOURCE_READER_FIRST_VIDEO_STREAM,
|
|
NULL,
|
|
pType
|
|
);
|
|
|
|
if (SUCCEEDED(hr)) {
|
|
deviceParam.videoFormat = g_VideoFormats[i].MFVideoFormat;
|
|
getFormat = true;
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
if (SUCCEEDED(hr))
|
|
hr = MFGetAttributeSize(pType, MF_MT_FRAME_SIZE, &deviceParam.width, &deviceParam.height);
|
|
|
|
if (SUCCEEDED(hr))
|
|
hr = GetDefaultStride(pType, &deviceParam.default_stride);
|
|
|
|
break;
|
|
case CAPTUREDEVICE_TYPE::AUDIO:
|
|
hr = pType->GetGUID(MF_MT_SUBTYPE, &subtype);
|
|
if (FAILED(hr))
|
|
break;
|
|
|
|
for (int i = 0; i < g_cAudioFormats; i++) {
|
|
if (subtype == g_AudioFormats[i].MFAudioFormat) {
|
|
deviceParam.audioFormat = subtype;
|
|
getFormat = true;
|
|
break;
|
|
}
|
|
}
|
|
|
|
if (!getFormat) {
|
|
for (int i = 0; i < g_cAudioFormats; i++) {
|
|
hr = pType->SetGUID(MF_MT_SUBTYPE, g_AudioFormats[i].MFAudioFormat);
|
|
if (FAILED(hr))
|
|
continue;
|
|
|
|
hr = m_pReader->SetCurrentMediaType(
|
|
(DWORD)MF_SOURCE_READER_FIRST_AUDIO_STREAM,
|
|
NULL,
|
|
pType
|
|
);
|
|
|
|
if (SUCCEEDED(hr)) {
|
|
deviceParam.audioFormat = g_AudioFormats[i].MFAudioFormat;
|
|
getFormat = true;
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
if (SUCCEEDED(hr))
|
|
hr = pType->GetUINT32(MF_MT_AUDIO_SAMPLES_PER_SECOND, &deviceParam.sampleRate);
|
|
|
|
if (SUCCEEDED(hr))
|
|
hr = pType->GetUINT32(MF_MT_AUDIO_NUM_CHANNELS, &deviceParam.channels);
|
|
|
|
if (SUCCEEDED(hr))
|
|
hr = pType->GetUINT32(MF_MT_AUDIO_BITS_PER_SAMPLE, (UINT32 *)&deviceParam.bitsPerSample);
|
|
|
|
break;
|
|
}
|
|
|
|
return hr;
|
|
}
|
|
|
|
void WindowsCaptureDevice::sendMessage(CAPTUREDEVICE_MESSAGE message) {
|
|
// Must be unique lock
|
|
std::unique_lock<std::mutex> lock(mutex);
|
|
messageQueue.push(message);
|
|
cond.notify_one();
|
|
}
|
|
|
|
CAPTUREDEVICE_MESSAGE WindowsCaptureDevice::getMessage() {
|
|
// Must be unique lock
|
|
std::unique_lock<std::mutex> lock(mutex);
|
|
CAPTUREDEVICE_MESSAGE message;
|
|
cond.wait(lock, [this]() { return !messageQueue.empty(); });
|
|
message = messageQueue.front();
|
|
messageQueue.pop();
|
|
|
|
return message;
|
|
}
|
|
|
|
void WindowsCaptureDevice::updateState(const CAPTUREDEVICE_STATE &newState) {
|
|
state = newState;
|
|
if (isShutDown()) {
|
|
std::unique_lock<std::mutex> guard(stateMutex_);
|
|
stateCond_.notify_all();
|
|
}
|
|
}
|
|
|
|
void WindowsCaptureDevice::waitShutDown() {
|
|
_dbg_assert_(thread_.joinable());
|
|
if (!thread_.joinable()) {
|
|
ERROR_LOG(Log::System, "Tried to wait for capture device thread, but not joinable.");
|
|
return;
|
|
}
|
|
|
|
sendMessage({ CAPTUREDEVICE_COMMAND::SHUTDOWN, nullptr });
|
|
thread_.join();
|
|
}
|
|
|
|
// This is the main thread for the capture device.
|
|
void WindowsCaptureDevice::messageHandler() {
|
|
SetCurrentThreadName(type == CAPTUREDEVICE_TYPE::AUDIO ? "AudioCapture" : "VideoCapture");
|
|
|
|
if (!MFAPI::Startup) {
|
|
setError(CAPTUREDEVIDE_ERROR_INIT_FAILED, "Cannot register devices");
|
|
return;
|
|
}
|
|
|
|
HRESULT hr = ERROR_SUCCESS;
|
|
{
|
|
std::lock_guard<std::mutex> lock(paramMutex);
|
|
hr = enumDevices();
|
|
}
|
|
if (FAILED(hr)) {
|
|
setError(CAPTUREDEVIDE_ERROR_INIT_FAILED, "Cannot enumerate devices");
|
|
return;
|
|
}
|
|
|
|
updateState(CAPTUREDEVICE_STATE::STOPPED);
|
|
CAPTUREDEVICE_MESSAGE message;
|
|
|
|
if (type == CAPTUREDEVICE_TYPE::VIDEO) {
|
|
SetCurrentThreadName("Camera");
|
|
} else if (type == CAPTUREDEVICE_TYPE::AUDIO) {
|
|
SetCurrentThreadName("Microphone");
|
|
}
|
|
|
|
while ((message = getMessage()).command != CAPTUREDEVICE_COMMAND::SHUTDOWN) {
|
|
switch (message.command) {
|
|
case CAPTUREDEVICE_COMMAND::START:
|
|
start(message.opacity);
|
|
break;
|
|
case CAPTUREDEVICE_COMMAND::STOP:
|
|
stop();
|
|
break;
|
|
case CAPTUREDEVICE_COMMAND::UPDATE_STATE:
|
|
updateState((*(CAPTUREDEVICE_STATE *)message.opacity));
|
|
break;
|
|
case CAPTUREDEVICE_COMMAND::SHUTDOWN:
|
|
break;
|
|
}
|
|
}
|
|
|
|
if (state != CAPTUREDEVICE_STATE::STOPPED)
|
|
stop();
|
|
|
|
std::lock_guard<std::mutex> lock(sdMutex);
|
|
m_pSource = nullptr;
|
|
m_pReader = nullptr;
|
|
m_pCallback = nullptr;
|
|
|
|
std::unique_lock<std::mutex> lock2(paramMutex);
|
|
for (DWORD i = 0; i < param.count; i++) {
|
|
SafeRelease(¶m.ppDevices[i]);
|
|
}
|
|
CoTaskMemFree(param.ppDevices); // Null pointer is okay.
|
|
lock2.unlock();
|
|
|
|
updateState(CAPTUREDEVICE_STATE::SHUTDOWN);
|
|
}
|
|
|
|
HRESULT WindowsCaptureDevice::enumDevices() {
|
|
HRESULT hr = S_OK;
|
|
ComPtr<IMFAttributes> pAttributes;
|
|
|
|
hr = MFAPI::CreateAttributes(&pAttributes, 1);
|
|
if (SUCCEEDED(hr)) {
|
|
switch (type) {
|
|
case CAPTUREDEVICE_TYPE::VIDEO:
|
|
hr = pAttributes->SetGUID(
|
|
MF_DEVSOURCE_ATTRIBUTE_SOURCE_TYPE,
|
|
MF_DEVSOURCE_ATTRIBUTE_SOURCE_TYPE_VIDCAP_GUID
|
|
);
|
|
|
|
break;
|
|
case CAPTUREDEVICE_TYPE::AUDIO:
|
|
hr = pAttributes->SetGUID(
|
|
MF_DEVSOURCE_ATTRIBUTE_SOURCE_TYPE,
|
|
MF_DEVSOURCE_ATTRIBUTE_SOURCE_TYPE_AUDCAP_GUID
|
|
);
|
|
|
|
break;
|
|
default:
|
|
setError(CAPTUREDEVIDE_ERROR_UNKNOWN_TYPE, "Unknown device type");
|
|
return E_FAIL;
|
|
}
|
|
}
|
|
if (SUCCEEDED(hr)) {
|
|
hr = MFAPI::EnumDeviceSources(pAttributes.Get(), ¶m.ppDevices, ¶m.count);
|
|
}
|
|
|
|
return hr;
|
|
}
|
|
|
|
bool WindowsCaptureDevice::needResample() {
|
|
return deviceParam.sampleRate != targetMediaParam.sampleRate ||
|
|
deviceParam.channels != targetMediaParam.channels ||
|
|
deviceParam.audioFormat != targetMediaParam.audioFormat ||
|
|
deviceParam.bitsPerSample != targetMediaParam.bitsPerSample;
|
|
}
|
|
|
|
//-----------------------------------------------------------------------------
|
|
// GetDefaultStride
|
|
//
|
|
// Gets the default stride for a video frame, assuming no extra padding bytes.
|
|
//
|
|
//-----------------------------------------------------------------------------
|
|
|
|
HRESULT GetDefaultStride(IMFMediaType *pType, LONG *plStride)
|
|
{
|
|
LONG lStride = 0;
|
|
|
|
// Try to get the default stride from the media type.
|
|
HRESULT hr = pType->GetUINT32(MF_MT_DEFAULT_STRIDE, (UINT32*)&lStride);
|
|
if (FAILED(hr))
|
|
{
|
|
// Attribute not set. Try to calculate the default stride.
|
|
GUID subtype = GUID_NULL;
|
|
|
|
UINT32 width = 0;
|
|
UINT32 height = 0;
|
|
|
|
// Get the subtype and the image size.
|
|
hr = pType->GetGUID(MF_MT_SUBTYPE, &subtype);
|
|
if (SUCCEEDED(hr))
|
|
{
|
|
hr = MFGetAttributeSize(pType, MF_MT_FRAME_SIZE, &width, &height);
|
|
}
|
|
if (SUCCEEDED(hr))
|
|
{
|
|
hr = MFAPI::GetStrideForBitmapInfoHeader(subtype.Data1, width, &lStride);
|
|
}
|
|
|
|
// Set the attribute for later reference.
|
|
if (SUCCEEDED(hr))
|
|
{
|
|
(void)pType->SetUINT32(MF_MT_DEFAULT_STRIDE, UINT32(lStride));
|
|
}
|
|
}
|
|
|
|
if (SUCCEEDED(hr))
|
|
{
|
|
*plStride = lStride;
|
|
}
|
|
return hr;
|
|
}
|