Skip to content

Commit

Permalink
Improve lookup, and reduce copying when finalizing mix presentation O…
Browse files Browse the repository at this point in the history 
…BUs.

  - Backfill a test of existing behavior that the finalizing does not re-order the mix presentations.
  - Change private implementation:
    - Hold rendering metadata in a hash map, for efficient lookup during `GetPostProcessedSpan`, instead of iterating over a list.
    - Remove unneeded `std::optional` wrapper. Instead omit the slot from the hash map when rendering is disabled.
    - N.b: Iterating over `absl::flat_hash_map` is non-determistic - ensure the output OBUs are not re-ordered by holding them in a `std::list`.
    - Reduce copying by finalizing that list in place and moving it out.
  - Behavior change:
    - Forbid multiple calls to `GetFinalizedMixPresentationObus` - which allows directly moving the OBUs out instead of finalizing.

PiperOrigin-RevId: 725238140
  • Loading branch information
@jwcullen
jwcullen committed Feb 11, 2025
1 parent f6e7d64 commit 7b6cf43
Show file tree
Hide file tree
Showing 3 changed files with 165 additions and 88 deletions.
126 changes: 62 additions & 64 deletions iamf/cli/rendering_mix_presentation_finalizer.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,6 @@
#include <functional>
#include <list>
#include <memory>
#include <optional>
#include <string>
#include <utility>
#include <vector>
Expand Down Expand Up @@ -736,47 +735,37 @@ absl::Status RenderWriteAndCalculateLoudnessForTemporalUnit(

absl::StatusOr<const LayoutRenderingMetadata*>
GetRenderedSamplesAndPostProcessor(
const auto& mix_presentation_id_to_rendering_metadata,
const absl::flat_hash_map<DecodedUleb128,
std::vector<SubmixRenderingMetadata>>&
mix_presentation_id_to_sub_mix_rendering_metadata,
DecodedUleb128 mix_presentation_id, size_t sub_mix_index,
size_t layout_index) {
// TODO(b/394072450): Store the mix presentations in an associative map, to
// avoid repeatedly searching for them. Be careful to avoid
// re-ordering same when the order of the mix presentations
// when `GetFinalizedMixPresentationObus` is called.
// Lookup the requested layout in the requested mix presentation.
auto mix_presentation_it = std::find_if(
mix_presentation_id_to_rendering_metadata.begin(),
mix_presentation_id_to_rendering_metadata.end(),
[mix_presentation_id](const auto& mix_presentation_rendering_metadata) {
return mix_presentation_rendering_metadata.mix_presentation_obu
.GetMixPresentationId() == mix_presentation_id;
});

const auto sub_mix_rendering_metadata_it =
mix_presentation_id_to_sub_mix_rendering_metadata.find(
mix_presentation_id);
const auto mix_presentation_id_error_message =
absl::StrCat(" Mix Presentation ID ", mix_presentation_id);
if (mix_presentation_it == mix_presentation_id_to_rendering_metadata.end()) {
if (sub_mix_rendering_metadata_it ==
mix_presentation_id_to_sub_mix_rendering_metadata.end()) {
return absl::NotFoundError(
absl::StrCat(mix_presentation_id_error_message,
" not found in rendering metadata."));
}

// Validate rendering is enabled, and the layout is in bounds.
RETURN_IF_NOT_OK(
ValidateHasValue(mix_presentation_it->submix_rendering_metadata,
absl::StrCat("because `submix_rendering_metadata` "
"implies rendering is disabled.",
mix_presentation_id_error_message)));
// Validate the sub mix and layout are in bounds, then retrieve it.
const auto& [unused_mix_presentation_id, sub_mix_rendering_metadatas] =
*sub_mix_rendering_metadata_it;
RETURN_IF_NOT_OK(Validate(
sub_mix_index, std::less<size_t>(),
mix_presentation_it->submix_rendering_metadata->size(),
sub_mix_index, std::less<size_t>(), sub_mix_rendering_metadatas.size(),
absl::StrCat(mix_presentation_id_error_message, " sub_mix_index <")));
const auto& submix_rendering_metadata =
(*mix_presentation_it->submix_rendering_metadata)[sub_mix_index];
RETURN_IF_NOT_OK(Validate(
layout_index, std::less<size_t>(),
submix_rendering_metadata.layout_rendering_metadata.size(),
sub_mix_rendering_metadatas[sub_mix_index]
.layout_rendering_metadata.size(),
absl::StrCat(mix_presentation_id_error_message, " layout_index <")));
return &submix_rendering_metadata.layout_rendering_metadata[layout_index];
return &sub_mix_rendering_metadatas[sub_mix_index]
.layout_rendering_metadata[layout_index];
}

} // namespace
Expand All @@ -797,29 +786,33 @@ RenderingMixPresentationFinalizer::Create(
LOG(INFO) << "Loudness calculator factory is null so loudness will not be "
"calculated.";
}
std::list<MixPresentationRenderingMetadata> rendering_metadata;
absl::flat_hash_map<DecodedUleb128, std::vector<SubmixRenderingMetadata>>
mix_presentation_id_to_rendering_metadata;
std::list<MixPresentationObu> mix_presentation_obus_to_render;
for (const auto& mix_presentation_obu : mix_presentation_obus) {
std::optional<std::vector<SubmixRenderingMetadata>>
sub_mix_rendering_metadata;
// Copy all mix presentation OBUs, so they can be echoed back, even when
// rendering is disabled.
mix_presentation_obus_to_render.emplace_back(mix_presentation_obu);

// Fill in rendering metadata if rendering is enabled, and at least one
// layout can be rendered.
if (rendering_enabled) {
std::vector<SubmixRenderingMetadata> temp_sub_mix_rendering_metadata;
RETURN_IF_NOT_OK(GenerateRenderingMetadataForSubmixes(
*renderer_factory, loudness_calculator_factory,
sample_processor_factory, audio_elements, mix_presentation_obu,
temp_sub_mix_rendering_metadata));
if (CanRenderAnyLayout(temp_sub_mix_rendering_metadata)) {
sub_mix_rendering_metadata = std::move(temp_sub_mix_rendering_metadata);
mix_presentation_id_to_rendering_metadata.emplace(
mix_presentation_obu.GetMixPresentationId(),
std::move(temp_sub_mix_rendering_metadata), );
}
}

rendering_metadata.emplace_back(MixPresentationRenderingMetadata{
.mix_presentation_obu = mix_presentation_obu,
.submix_rendering_metadata = std::move(sub_mix_rendering_metadata),
});
}

return RenderingMixPresentationFinalizer(std::move(rendering_metadata));
return RenderingMixPresentationFinalizer(
std::move(mix_presentation_id_to_rendering_metadata),
std::move(mix_presentation_obus_to_render));
}

absl::Status RenderingMixPresentationFinalizer::PushTemporalUnit(
Expand All @@ -834,17 +827,16 @@ absl::Status RenderingMixPresentationFinalizer::PushTemporalUnit(
return absl::FailedPreconditionError(
"PushTemporalUnit() should not be called after "
"FinalizePushingTemporalUnits() has been called.");
case kFlushedFinalizedMixPresentationObus:
return absl::FailedPreconditionError(
"PushTemporalUnit() should not be called after "
"GetFinalizedMixPresentationOBUs() has been called.");
}
for (auto& [mix_presentation_obu_for_logging, submix_rendering_metadata] :
rendering_metadata_) {
if (!submix_rendering_metadata.has_value()) {
LOG(INFO) << "No layouts can be rendered for Mix Presentation ID= "
<< mix_presentation_obu_for_logging.GetMixPresentationId();
continue;
}
for (auto& [mix_presentation_ids, sub_mix_rendering_metadata] :
mix_presentation_id_to_sub_mix_rendering_metadata_) {
RETURN_IF_NOT_OK(RenderWriteAndCalculateLoudnessForTemporalUnit(
id_to_labeled_frame, start_timestamp, end_timestamp, parameter_blocks,
*submix_rendering_metadata));
sub_mix_rendering_metadata));
}
return absl::OkStatus();
}
Expand All @@ -854,7 +846,8 @@ RenderingMixPresentationFinalizer::GetPostProcessedSamplesAsSpan(
DecodedUleb128 mix_presentation_id, size_t sub_mix_index,
size_t layout_index) const {
const auto layout_rendering_metadata = GetRenderedSamplesAndPostProcessor(
rendering_metadata_, mix_presentation_id, sub_mix_index, layout_index);
mix_presentation_id_to_sub_mix_rendering_metadata_, mix_presentation_id,
sub_mix_index, layout_index);
if (!layout_rendering_metadata.ok()) {
return layout_rendering_metadata.status();
}
Expand All @@ -875,15 +868,14 @@ absl::Status RenderingMixPresentationFinalizer::FinalizePushingTemporalUnits() {
state_ = kFinalizePushTemporalUnitCalled;
break;
case kFinalizePushTemporalUnitCalled:
case kFlushedFinalizedMixPresentationObus:
return absl::FailedPreconditionError(
"FinalizePushingTemporalUnits() should not be called twice.");
}

for (auto& [mix_presentation_obu, submix_rendering_metadata] :
rendering_metadata_) {
if (submix_rendering_metadata.has_value()) {
RETURN_IF_NOT_OK(FlushPostProcessors(*submix_rendering_metadata));
}
for (auto& [mix_presentation_id, sub_mix_rendering_metadata] :
mix_presentation_id_to_sub_mix_rendering_metadata_) {
RETURN_IF_NOT_OK(FlushPostProcessors(sub_mix_rendering_metadata));
}
return absl::OkStatus();
}
Expand All @@ -899,27 +891,33 @@ RenderingMixPresentationFinalizer::GetFinalizedMixPresentationObus(
case kFinalizePushTemporalUnitCalled:
// Ok to finalize.
break;
case kFlushedFinalizedMixPresentationObus:
return absl::FailedPreconditionError(
"GetFinalizedMixPresentationOBUs() should not be called twice.");
}

std::list<MixPresentationObu> finalized_obus;
for (auto& [original_presentation_obu, submix_rendering_metadata] :
rendering_metadata_) {
MixPresentationObu finalized_mix_presentation_obu =
original_presentation_obu;
if (submix_rendering_metadata.has_value()) {
RETURN_IF_NOT_OK(FillLoudnessForMixPresentation(
validate_loudness, *submix_rendering_metadata,
finalized_mix_presentation_obu));
} else {
// Finalize the OBUs in place.
for (auto& mix_presentation_obu : mix_presentation_obus_) {
const auto sub_mix_rendering_metadata_it =
mix_presentation_id_to_sub_mix_rendering_metadata_.find(
mix_presentation_obu.GetMixPresentationId());
if (sub_mix_rendering_metadata_it ==
mix_presentation_id_to_sub_mix_rendering_metadata_.end()) {
LOG(INFO) << "Rendering was disabled for Mix Presentation ID= "
<< finalized_mix_presentation_obu.GetMixPresentationId()
<< mix_presentation_obu.GetMixPresentationId()
<< " echoing the input OBU.";
continue;
}

finalized_obus.emplace_back(std::move(finalized_mix_presentation_obu));
RETURN_IF_NOT_OK(FillLoudnessForMixPresentation(
validate_loudness, sub_mix_rendering_metadata_it->second,
mix_presentation_obu));
}

return finalized_obus;
// Flush the finalized OBUs and mark that this class should not use them
// again.
state_ = kFlushedFinalizedMixPresentationObus;
return std::move(mix_presentation_obus_);
}

} // namespace iamf_tools
39 changes: 22 additions & 17 deletions iamf/cli/rendering_mix_presentation_finalizer.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,6 @@
#include <cstdint>
#include <list>
#include <memory>
#include <optional>
#include <utility>
#include <vector>

Expand Down Expand Up @@ -262,33 +261,39 @@ class RenderingMixPresentationFinalizer {
bool validate_loudness);

private:
enum State { kAcceptingTemporalUnits, kFinalizePushTemporalUnitCalled };

/*!\brief Metadata for all sub mixes within a single mix presentation. */
struct MixPresentationRenderingMetadata {
// Mix presentation OBU to render and to be updated with the final measured
// loudness.
MixPresentationObu mix_presentation_obu;

// Metadata for rendering all sub mixes within the mix presentation. If
// `nullopt`, rendering is disabled for this mix presentation.
std::optional<std::vector<SubmixRenderingMetadata>>
submix_rendering_metadata;
enum State {
kAcceptingTemporalUnits,
kFinalizePushTemporalUnitCalled,
kFlushedFinalizedMixPresentationObus
};

/*!\brief Private constructor.
*
* Used only by the factory method.
*
* \param rendering_metadata Mix presentation metadata.
* \param mix_presentation_id_to_sub_mix_rendering_metadata Mix presentation
* ID to rendering metadata for each sub mix.
* \param mix_presentation_obus Mix presentation OBUs to render and measure
* the loudness of.
*/
RenderingMixPresentationFinalizer(
std::list<MixPresentationRenderingMetadata>&& rendering_metadata)
: rendering_metadata_(std::move(rendering_metadata)) {}
absl::flat_hash_map<DecodedUleb128,
std::vector<SubmixRenderingMetadata>>&&
mix_presentation_id_to_sub_mix_rendering_metadata,
std::list<MixPresentationObu>&& mix_presentation_obus)
: mix_presentation_id_to_sub_mix_rendering_metadata_(
std::move(mix_presentation_id_to_sub_mix_rendering_metadata)),
mix_presentation_obus_(std::move(mix_presentation_obus)) {}

State state_ = kAcceptingTemporalUnits;

std::list<MixPresentationRenderingMetadata> rendering_metadata_;
// Mapping from Mix Presentation ID to rendering metadata. Slots are absent
// for Mix Presentations that have no layouts which can be rendered.
absl::flat_hash_map<DecodedUleb128, std::vector<SubmixRenderingMetadata>>
mix_presentation_id_to_sub_mix_rendering_metadata_;

// Mix Presentation OBUs to render and measure the loudness of.
std::list<MixPresentationObu> mix_presentation_obus_;
};

} // namespace iamf_tools
Expand Down
88 changes: 81 additions & 7 deletions iamf/cli/tests/rendering_mix_presentation_finalizer_test.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,7 @@ using ::absl_testing::IsOk;
using ::absl_testing::IsOkAndHolds;
using ::absl_testing::StatusIs;
using ::testing::_;
using ::testing::Eq;
using ::testing::IsEmpty;
using ::testing::Not;
using testing::Return;
Expand Down Expand Up @@ -942,18 +943,91 @@ TEST_F(FinalizerTest,
finalizer.GetFinalizedMixPresentationObus(kDontValidateLoudness).ok());
}

TEST_F(FinalizerTest, GetFinalizedMixPresentationObusMayBeCalledMultipleTimes) {
TEST_F(FinalizerTest,
PushingIsNotAllowedAnyTimeAfterFinalizePushingTemporalUnits) {
InitPrerequisiteObusForMonoInput(kAudioElementId);
AddMixPresentationObuForMonoOutput(kMixPresentationId);
const LabelSamplesMap kLabelToSamples = {
{kMono, Int32ToInternalSampleType({100, 900})}};
const std::vector<std::vector<int32_t>> kExpectedSamples = {{100}, {900}};
AddLabeledFrame(kAudioElementId, kLabelToSamples, kEndTime);
auto finalizer = CreateFinalizerExpectOk();
EXPECT_THAT(
finalizer.PushTemporalUnit(ordered_labeled_frames_[0],
/*start_timestamp=*/0,
/*end_timestamp=*/10, parameter_blocks_),
IsOk());
EXPECT_THAT(finalizer.FinalizePushingTemporalUnits(), IsOk());

// Any time after `FinalizePushingTemporalUnits()` has been called pushing is
// not allowed. Even if later functions such as
// `GetFinalizedMixPresentationObus` are called.
EXPECT_THAT(
finalizer.PushTemporalUnit(ordered_labeled_frames_[0],
/*start_timestamp=*/10,
/*end_timestamp=*/20, parameter_blocks_),
Not(IsOk()));
EXPECT_THAT(finalizer.GetFinalizedMixPresentationObus(kDontValidateLoudness),
IsOk());
EXPECT_THAT(
finalizer.PushTemporalUnit(ordered_labeled_frames_[0],
/*start_timestamp=*/10,
/*end_timestamp=*/20, parameter_blocks_),
Not(IsOk()));
}

TEST_F(
FinalizerTest,
GetPostProcessedSamplesAsSpanCanBeUsedBeforeOrAfterGetFinalizedMixPresentationObus) {
InitPrerequisiteObusForStereoInput(kAudioElementId);
AddMixPresentationObuForStereoOutput(kMixPresentationId);
renderer_factory_ = std::make_unique<RendererFactory>();
auto finalizer = CreateFinalizerExpectOk();
EXPECT_THAT(finalizer.FinalizePushingTemporalUnits(), IsOk());

const auto finalized_obus =
finalizer.GetFinalizedMixPresentationObus(kDontValidateLoudness);
EXPECT_THAT(finalized_obus, IsOk());
// Subsequent calls are permitted, but they should not change the result.
EXPECT_EQ(finalized_obus,
finalizer.GetFinalizedMixPresentationObus(kDontValidateLoudness));
// It's acceptable to retrieve the post-processed samples either before or
// after retrieving the finalized mix presentation OBUs.
EXPECT_THAT(finalizer.GetPostProcessedSamplesAsSpan(
kMixPresentationId, kFirstLayoutIndex, kFirstSubmixIndex),
IsOk());
EXPECT_THAT(finalizer.GetFinalizedMixPresentationObus(kDontValidateLoudness),
IsOk());
EXPECT_THAT(finalizer.GetPostProcessedSamplesAsSpan(
kMixPresentationId, kFirstLayoutIndex, kFirstSubmixIndex),
IsOk());
}

TEST_F(FinalizerTest, GetFinalizedMixPresentationObusMayNotBeCalledTwice) {
InitPrerequisiteObusForStereoInput(kAudioElementId);
AddMixPresentationObuForStereoOutput(kMixPresentationId);
auto finalizer = CreateFinalizerExpectOk();
EXPECT_THAT(finalizer.FinalizePushingTemporalUnits(), IsOk());

EXPECT_THAT(finalizer.GetFinalizedMixPresentationObus(kDontValidateLoudness),
IsOk());

// The finalized OBUs have already been flushed out and cannot be retrieved
// again.
EXPECT_THAT(finalizer.GetFinalizedMixPresentationObus(kDontValidateLoudness),
Not(IsOk()));
}

TEST_F(FinalizerTest, GetFinalizedMixPresentationObusReturnsObusInSameorder) {
InitPrerequisiteObusForStereoInput(kAudioElementId);
constexpr DecodedUleb128 kFirstMixPresentationId = 100;
constexpr DecodedUleb128 kSecondMixPresentationId = 99;
constexpr DecodedUleb128 kThirdMixPresentationId = 101;
AddMixPresentationObuForStereoOutput(kFirstMixPresentationId);
AddMixPresentationObuForStereoOutput(kSecondMixPresentationId);
AddMixPresentationObuForStereoOutput(kThirdMixPresentationId);
auto finalizer = CreateFinalizerExpectOk();
EXPECT_THAT(finalizer.FinalizePushingTemporalUnits(), IsOk());

// We expect the entire list to come back in the same order. List order
// may affect the priority downstream when selecting the default mix to
// playback.
ASSERT_THAT(finalizer.GetFinalizedMixPresentationObus(kDontValidateLoudness),
IsOkAndHolds(Eq(obus_to_finalize_)));
}

// =========== Tests for PushTemporalUnit ===========
Expand Down

0 comments on commit 7b6cf43

Please sign in to comment.