Unverified Commit b6a32d4d authored by Jake Stine's avatar Jake Stine Committed by GitHub
Browse files

audio: add support for static sources, add lua_ref logic to playing sounds (#173)

* audio: Overhauled. New Mixer, new decoders, and added proper support for soundData as a static source.

 - avoids cutting sounds before they finish playing
 - cleans up memory for sources
 - even closes file handles!!

* Rename ogg vorbis decoder functions to decOgg_* to match with decWav_ API

* audio: fix WAVE audio corruption at end of sample

Main issue is that some RIFF WAV files have additional non-audio chunks at the end of the file, for meta data. We were mixing this into the audio buffer, causing noise.
parent 2f87c731
......@@ -12,8 +12,15 @@
/* TODO/FIXME - no sound on big-endian */
static unsigned num_sources = 0;
static audio_Source** sources = NULL;
// any source which is playing must maintain a ref in lua, to avoid __gc.
typedef struct {
audio_Source* source;
int lua_ref;
} audioSourceWithRef;
static int num_sources = 0;
static audioSourceWithRef* sources_playing = NULL;
static float volume = 1.0;
#define CHANNELS 2
......@@ -34,110 +41,158 @@ int audio_sources_nullify_refs(const audio_Source* source)
for(int i=0; i<num_sources; ++i)
{
if (sources[i] == source)
audioSourceWithRef* srcref = &sources_playing[i];
if (srcref->source == source)
{
if (sources[i]->state != AUDIO_STOPPED)
if (srcref->source->state != AUDIO_STOPPED)
++counted;
// do not free - the pointers in sources are lua user data
sources[i] = NULL;
srcref->source = NULL;
srcref->lua_ref = LUA_NOREF;
}
}
return counted;
}
// unrefs stopped sounds.
// this is done periodically by lutro to avoid adding lua dependencies to the mixer.
void mixer_unref_stopped_sounds(lua_State* L)
{
// Loop over audio sources
for (int i = 0; i < num_sources; i++)
{
if (sources_playing[i].lua_ref >= 0)
{
audio_Source* source = sources_playing[i].source;
if (!source || source->state == AUDIO_STOPPED)
{
lua_getglobal(L, "refs_audio_playing");
luaL_unref(L, -1, sources_playing[i].lua_ref);
sources_playing[i].lua_ref = LUA_REFNIL;
}
}
if (sources_playing[i].lua_ref < 0)
sources_playing[i].source = NULL;
}
}
void lutro_audio_stop_all(void)
{
// Loop over audio sources
// no cleanup needed, __gc will handle it later after a call to mixer_unref_stopped_sounds()
for (int i = 0; i < num_sources; i++)
{
if (sources_playing[i].source)
sources_playing[i].source->state = AUDIO_STOPPED;
}
}
void mixer_render(int16_t *buffer)
{
static mixer_presaturate_t presaturateBuffer[AUDIO_FRAMES * CHANNELS];
memset(presaturateBuffer, 0, AUDIO_FRAMES * CHANNELS * sizeof(mixer_presaturate_t));
presaturate_buffer_desc bufdesc;
bufdesc.data = presaturateBuffer;
bufdesc.channels = CHANNELS;
bufdesc.samplelen = AUDIO_FRAMES;
// Loop over audio sources
for (int i = 0; i < num_sources; i++)
{
if (!sources[i])
audio_Source* source = sources_playing[i].source;
if (!source)
continue;
if (sources[i]->state == AUDIO_STOPPED)
if (source->state == AUDIO_STOPPED)
continue;
// options here are to premultiply source volumes with master volume, or apply master volume at the end of mixing
// during the saturation step. Each approach has its strengths and weaknesses and overall neither differs much when
// using float or double for presaturation buffer (see final saturation step below)
float srcvol = sources[i]->volume;
float srcvol = source->volume;
if (sources[i]->oggData)
// Decoder Seek Position Note:
// It's unclear if a decoder can be shared by multiple sources, so always seek the decoder for each chunk.
// Our decoder APIs internally optimize away redundant seeks.
if (source->oggData)
{
bool finished = decoder_decodeOgg(sources[i]->oggData, presaturateBuffer, srcvol, sources[i]->loop);
decOgg_seek(source->oggData, source->sndpos);
bool finished = decOgg_decode(source->oggData, &bufdesc, srcvol, source->loop);
if (finished)
{
decoder_seek(sources[i]->oggData, 0);
sources[i]->state = AUDIO_STOPPED;
decOgg_seek(source->oggData, 0); // see notes above
source->state = AUDIO_STOPPED;
source->sndpos = 0;
}
source->sndpos = decOgg_sampleTell(source->oggData);
continue;
}
void* rawsamples_alloc = calloc(
AUDIO_FRAMES * sources[i]->bps, sizeof(uint8_t));
fseek(sources[i]->sndta.fp, WAV_HEADER_SIZE + sources[i]->pos, SEEK_SET);
bool end = ! fread(rawsamples_alloc,
sizeof(uint8_t),
AUDIO_FRAMES * sources[i]->bps,
sources[i]->sndta.fp);
// ogg outputs float values range 1.0 to -1.0
// 16-bit wav outputs values range 32767 to -32768
// 8-bit wav is scaled up to 16 bit and then normalized using 16-bit divisor.
float srcvol_and_scale_to_one = srcvol / 32767;
if (sources[i]->sndta.head.BitsPerSample == 8)
if (source->wavData)
{
const int8_t* rawsamples8 = (int8_t*)rawsamples_alloc;
for (int j = 0; j < AUDIO_FRAMES; j++)
decWav_seek(source->wavData, source->sndpos); // see notes above
bool finished = decWav_decode(source->wavData, &bufdesc, srcvol, source->loop);
if (finished)
{
// note this is currently *64 because the mixer is mixing 8-bit smaples as 0->255 instead of normalizing to -128 to 127.
// This would have caused the more appropriate *128 multiplier to cause saturation along the top of the waveform.
// We should be able to change this to *128 now, but it will affect volume behavior of any games that use 8 bit samples and
// were authored with the current *64 behavior. So need to verify how we want to go about handling this --jstine
mixer_presaturate_t left = (sources[i]->sndta.head.NumChannels == 2) ? rawsamples8[j*2+0] : rawsamples8[j] * 64;
mixer_presaturate_t right = (sources[i]->sndta.head.NumChannels == 2) ? rawsamples8[j*2+1] : rawsamples8[j] * 64;
if (sources[i]->sndta.head.NumChannels == 2) { right = rawsamples8[j*2+1]*64; }
presaturateBuffer[j*2+0] += (left * srcvol_and_scale_to_one);
presaturateBuffer[j*2+1] += (right * srcvol_and_scale_to_one);
sources[i]->pos += sources[i]->bps;
decWav_seek(source->wavData, 0);
source->state = AUDIO_STOPPED;
}
source->sndpos = decWav_sampleTell(source->wavData);
continue;
}
if (sources[i]->sndta.head.BitsPerSample == 16)
if (source->sndta)
{
const int16_t* rawsamples16 = (int16_t*)rawsamples_alloc;
for (int j = 0; j < AUDIO_FRAMES; j++)
{
mixer_presaturate_t left = (sources[i]->sndta.head.NumChannels == 2) ? rawsamples16[j*2+0] : rawsamples16[j];
mixer_presaturate_t right = (sources[i]->sndta.head.NumChannels == 2) ? rawsamples16[j*2+1] : rawsamples16[j];
snd_SoundData* sndta = source->sndta;
if (sources[i]->sndta.head.NumChannels == 2) { right = rawsamples16[j*2+1]; }
int total_mixed = 0;
presaturateBuffer[j*2+0] += (left * srcvol_and_scale_to_one);
presaturateBuffer[j*2+1] += (right * srcvol_and_scale_to_one);
sources[i]->pos += sources[i]->bps;
while (total_mixed < AUDIO_FRAMES)
{
int mixchunksz = AUDIO_FRAMES - total_mixed;
int remaining = sndta->numSamples - source->sndpos;
if (mixchunksz > remaining)
{
mixchunksz = remaining;
}
if (sndta->numChannels == 1)
{
for (int j = 0; j < mixchunksz; ++j, ++total_mixed, ++source->sndpos)
{
presaturateBuffer[(total_mixed*2) + 0] += sndta->data[source->sndpos] * srcvol;
presaturateBuffer[(total_mixed*2) + 1] += sndta->data[source->sndpos] * srcvol;
}
}
if (sndta->numChannels == 2)
{
for (int j = 0; j < mixchunksz; ++j, ++total_mixed, ++source->sndpos)
{
presaturateBuffer[(total_mixed*2) + 0] += sndta->data[(source->sndpos*2) + 0] * srcvol;
presaturateBuffer[(total_mixed*2) + 1] += sndta->data[(source->sndpos*2) + 1] * srcvol;
}
}
assert(source->sndpos <= sndta->numSamples);
assert(total_mixed <= AUDIO_FRAMES);
if (source->sndpos == sndta->numSamples)
{
source->sndpos = 0;
if (!source->loop)
{
source->state = AUDIO_STOPPED;
break;
}
}
}
}
if (end)
{
if (!sources[i]->loop)
sources[i]->state = AUDIO_STOPPED;
sources[i]->pos = 0;
}
free(rawsamples_alloc);
}
// final saturation step - downsample.
......@@ -168,43 +223,56 @@ int lutro_audio_preload(lua_State *L)
return 1;
}
void lutro_audio_init()
void lutro_audio_init(lua_State* L)
{
num_sources = 0;
sources = NULL;
sources_playing = NULL;
volume = 1.0;
lua_newtable(L);
lua_setglobal(L, "refs_audio_playing");
}
void lutro_audio_deinit()
{
if (sources)
if (!sources_playing) return;
// lua owns most of our objects so there's only two proper ways to deinit audio:
// 1. assume luaState has been forcibly destroyed without its own cleanup.
// 2. run lua_close() and let it clean most of this up first.
lutro_audio_stop_all();
int counted = 0;
for (int i = 0; i < num_sources; i++)
{
for (unsigned i = 0; i < num_sources; i++)
{
if (sources[i] && sources[i]->oggData)
{
ov_clear(&sources[i]->oggData->vf);
free(sources[i]->oggData);
}
}
if (sources_playing[i].source)
++counted;
}
free(sources);
sources = NULL;
num_sources = 0;
if (counted)
{
fprintf(stderr, "Found %d leaked audio source references. Was lua_close() called first?\n", counted);
fflush(stderr);
//assert(false);
return;
}
free(sources_playing);
sources_playing = NULL;
num_sources = 0;
}
static int assign_to_existing_source_slot(audio_Source* self)
static int find_empty_source_slot(audio_Source* self)
{
for(int i=0; i<num_sources; ++i)
{
if (!sources[i])
{
sources[i] = self;
return 1;
}
// it's possible a stopped source is still in the sources_playing list, since cleanup
// operations are deferred. This is OK and expected, just use the slot that's still assigned...
if (!sources_playing[i].source || sources_playing[i].source == self)
return i;
}
return 0;
return -1;
}
int audio_newSource(lua_State *L)
......@@ -216,70 +284,46 @@ int audio_newSource(lua_State *L)
audio_Source* self = (audio_Source*)lua_newuserdata(L, sizeof(audio_Source));
self->oggData = NULL;
self->sndta.fp = NULL;
//if (lua_isstring(L,1)) // (lua_type(L, 1) == LUA_TSTRING)
//{
//
//}
self->wavData = NULL;
void *p = lua_touserdata(L, 1);
if (p == NULL)
{
const char* path = luaL_checkstring(L, 1);
char fullpath[PATH_MAX_LENGTH];
strlcpy(fullpath, settings.gamedir, sizeof(fullpath));
strlcat(fullpath, path, sizeof(fullpath));
//get file extension
char ext[PATH_MAX_LENGTH];
strcpy(ext, path_get_extension(path));
for(int i = 0; ext[i]; i++)
ext[i] = tolower((uint8_t)ext[i]);
//ogg
if (strstr(ext, "ogg"))
// when matching file paths, only lua string types are OK.
// lua_tostring will convert numbers into strings implicitly, which is not what we want.
luaL_checktype(L, 1, LUA_TSTRING); // explicit non-converted string type test
const char* path = lua_tostring(L, 1);
AssetPathInfo asset;
lutro_assetPath_init(&asset, path);
if (strstr(asset.ext, "ogg"))
{
self->oggData = malloc(sizeof(OggData));
decoder_initOgg(self->oggData, fullpath);
self->oggData = malloc(sizeof(dec_OggData));
decOgg_init(self->oggData, asset.fullpath);
}
//default: WAV file
else
{
FILE *fp = fopen(fullpath, "rb");
if (!fp)
return -1;
fread(&self->sndta.head, sizeof(uint8_t), WAV_HEADER_SIZE, fp);
self->sndta.fp = fp;
if (strstr(asset.ext, "wav"))
{
self->wavData = malloc(sizeof(dec_WavData));
decWav_init(self->wavData, asset.fullpath);
}
}
else
{
snd_SoundData* sndta = (snd_SoundData*)luaL_checkudata(L, 1, "SoundData");
self->sndta = *sndta;
self->sndta = sndta;
lua_pushvalue(L, 1); // push ref to SoundData parameter
self->lua_ref_sndta = luaL_ref(L, LUA_REGISTRYINDEX);
}
self->loop = false;
self->volume = 1.0;
self->pos = 0;
self->sndpos = 0;
self->state = AUDIO_STOPPED;
//WAV file
if (self->sndta.fp)
{
self->bps = self->sndta.head.NumChannels * self->sndta.head.BitsPerSample / 8;
fseek(self->sndta.fp, 0, SEEK_END);
}
if (!assign_to_existing_source_slot(self))
{
num_sources++;
sources = (audio_Source**)realloc(sources, num_sources * sizeof(audio_Source));
sources[num_sources-1] = self;
}
if (luaL_newmetatable(L, "Source") != 0)
{
static luaL_Reg audio_funcs[] = {
......@@ -402,18 +446,8 @@ int source_tell(lua_State *L)
//currently assuming samples vs seconds
//TODO: check if 2nd param is "seconds" or "samples"
//WAV file
if (self->sndta.fp)
{
lua_pushnumber(L, self->pos / self->bps);
}
//OGG file
else if (self->oggData)
{
uint32_t pos = 0;
decoder_sampleTell(self->oggData, &pos);
lua_pushnumber(L, pos);
}
// sndpos should always be accurate for any given source or stream.
lua_pushnumber(L, self->sndpos);
return 1;
}
......@@ -430,16 +464,13 @@ int source_seek(lua_State *L)
//currently assuming samples vs seconds
//TODO: check if 3rd param is "seconds" or "samples"
//WAV file
if (self->sndta.fp)
{
self->pos = self->bps * (unsigned)luaL_checknumber(L, 2);
}
//OGG file
else if (self->oggData)
{
decoder_seek(self->oggData, (unsigned)luaL_checknumber(L, 2));
}
self->sndpos = luaL_checkinteger(L, 2);
if (self->wavData)
decWav_seek(self->wavData, self->sndpos);
if (self->oggData)
decOgg_seek(self->oggData, self->sndpos);
return 0;
}
......@@ -476,45 +507,101 @@ int source_gc(lua_State *L)
if (leaks)
{
fprintf(stderr, "source_gc: playing audio references were nullified.\n");
fflush(stderr);
//assert(false);
}
luaL_unref(L, LUA_REGISTRYINDEX, self->lua_ref_sndta);
self->lua_ref_sndta = LUA_REFNIL;
if (self->wavData)
{
if (self->wavData->fp)
fclose(self->wavData->fp);
free(self->wavData);
}
if (self->oggData)
{
ov_clear(&self->oggData->vf);
free(self->oggData);
}
(void)self;
return 0;
}
// audio.play returns nothing, but source:play returns a boolean.
// it's OK enough for us to always return a boolean.
int audio_play(lua_State *L)
{
audio_Source* self = (audio_Source*)luaL_checkudata(L, 1, "Source");
//play pos should not reset if play called again before finished
//as in Love2D, game code should explicitly call stop or seek(0, nil) before play to reset pos if already playing
if (self->state == AUDIO_PLAYING)
return 0; // nothing to do.
if (self->state == AUDIO_PAUSED)
{
self->state = AUDIO_PLAYING;
return 0;
}
assert(self->state == AUDIO_STOPPED);
self->state = AUDIO_PLAYING;
lua_pushboolean(L, true);
// add a ref to our playing audio registry. this blocks __gc until the ref(s) are removed.
int slot = find_empty_source_slot(self);
if (slot < 0)
{
slot = num_sources++;
sources_playing = (audioSourceWithRef*)realloc(sources_playing, num_sources * sizeof(audioSourceWithRef));
sources_playing[slot].lua_ref = LUA_REFNIL;
sources_playing[slot].source = NULL;
}
// assume that find_empty_source_slot with either return the current slot, or an empty one.
if (sources_playing[slot].lua_ref < 0)
{
lua_getglobal(L, "refs_audio_playing");
lua_pushvalue(L, 1); // push ref to Source parameter
sources_playing[slot].source = self;
sources_playing[slot].lua_ref = luaL_ref(L, -2);
}
else
{
// existing ref means it should be our same source already.
assert(sources_playing[slot].source == self);
}
// for now sources always succeed in lutro.
// the only reason for a source to fail in Love2D is because it has a limited number of mixer
// voices internally that it allows. Lutro has no hard limit on mixer voices.
lua_pushboolean(L, 1);
return 1;
}
// returns nothing.
int audio_stop(lua_State *L)
{
audio_Source* self = (audio_Source*)luaL_checkudata(L, 1, "Source");
bool success = false;
//play pos is reset on stop
if (self->state == AUDIO_STOPPED)
return 0;
//WAV file
if (self->sndta.fp)
{
self->pos = 0;
self->state = AUDIO_STOPPED;
}
//OGG file
else if (self->oggData)
{
success = decoder_seek(self->oggData, 0);
self->state = AUDIO_STOPPED;
}
self->sndpos = 0;
self->state = AUDIO_STOPPED;
lua_pushboolean(L, success);
return 1;
// unref will be handled via periodic invocation of mixer_unref_stopped_sounds()
//lua_getglobal(L, "refs_audio_playing");
//luaL_unref(L, -1, self->lua_ref);
//self->lua_ref = LUA_REFNIL;
return 0;
}
......@@ -8,26 +8,7 @@
#include "runtime.h"
#include "sound.h"
#include "decoder.h"
#define AUDIO_FRAMES (44100 / 60)
// The following types are acceptable for pre-saturated mixing, as they meet the requirement for
// having a larger range than the saturated mixer result type of int16_t. double precision should
// be preferred on x86/amd64, and single precision on ARM. float16 could also work as an input
// but care must be taken to saturate at INT16_MAX-1 and INT16_MIN+1 due to float16 not having a
// 1:1 representation of whole numbers in the in16 range.
//
// TODO: set up appropriate compiler defs for mixer presaturate type.
typedef float mixer_presaturate_t;
#define cvt_presaturate_to_int16(in) ((int16_t)roundf(in))
//typedef double mixer_presaturate_t;
//#define cvt_presaturate_to_int16(in) (round(in))
//typedef int32_t mixer_presaturate_t;
//#define cvt_presaturate_to_int16(in) ((int16_t)in)
#include "audio_mixer.h"
typedef enum
{
......@@ -38,24 +19,29 @@ typedef enum
typedef struct
{
//currently for WAV
snd_SoundData sndta;
unsigned bps; // bytes per sample
//currently for Ogg Vorbis
OggData *oggData;
// only one of these should be non-null for a given source.
dec_WavData* wavData; // streaming from wav
dec_OggData* oggData; // streaming from ogg
snd_SoundData* sndta; // pre-decoded sound
int lua_ref_sndta; // (REGISTRY) ref to sndta is held as long as this object isn't disposed/__gc'd
intmax_t sndpos; // readpos in samples for pre-decoded sound only
bool loop;
float volume;
float pitch;
unsigned pos;
audio_source_state state;
} audio_Source;
void lutro_audio_init();
void lutro_audio_init(lua_State* L);
void lutro_audio_deinit();
void lutro_audio_stop_all();