39 reader(reader), target(target), pulldown(target_pulldown), is_dirty(true), avr(NULL), parent_position(0.0)
80 throw ReaderClosed(
"No Reader has been initialized for FrameMapper. Call Reader(*reader) before calling this method.");
83 void FrameMapper::AddField(int64_t frame)
86 AddField(
Field(frame, field_toggle));
89 void FrameMapper::AddField(
Field field)
95 field_toggle = (field_toggle ? false :
true);
102 void FrameMapper::Init()
118 parent_position = parent->
Position();
119 parent_start = parent->
Start();
121 parent_position = 0.0;
133 if ((fabs(original.
ToFloat() - 24.0) < 1e-7 || fabs(original.
ToFloat() - 25.0) < 1e-7 || fabs(original.
ToFloat() - 30.0) < 1e-7) &&
134 (fabs(target.
ToFloat() - 24.0) < 1e-7 || fabs(target.
ToFloat() - 25.0) < 1e-7 || fabs(target.
ToFloat() - 30.0) < 1e-7)) {
137 float difference = target.
ToInt() - original.
ToInt();
140 int field_interval = 0;
141 int frame_interval = 0;
145 field_interval = round(fabs(original.
ToInt() / difference));
148 frame_interval = field_interval * 2.0f;
157 for (int64_t field = 1; field <= number_of_fields; field++)
165 else if (difference > 0)
175 else if (pulldown ==
PULLDOWN_ADVANCED && field % field_interval == 0 && field % frame_interval != 0)
182 AddField(
Field(frame + 1, field_toggle));
184 else if (pulldown ==
PULLDOWN_NONE && field % frame_interval == 0)
191 else if (difference < 0)
197 field_toggle = (field_toggle ? false :
true);
199 else if (pulldown ==
PULLDOWN_ADVANCED && field % field_interval == 0 && field % frame_interval != 0)
204 else if (pulldown ==
PULLDOWN_NONE && frame % field_interval == 0)
217 if (field % 2 == 0 && field > 0)
227 double value_increment = (reader->
info.
video_length + 1) / (
double) (new_length);
230 double original_frame_num = 1.0f;
231 for (int64_t frame_num = 1; frame_num <= new_length; frame_num++)
234 AddField(round(original_frame_num));
235 AddField(round(original_frame_num));
238 original_frame_num += value_increment;
247 int64_t start_samples_frame = 1;
248 int start_samples_position = 0;
250 for (std::vector<Field>::size_type field = 1; field <=
fields.size(); field++)
256 if (field % 2 == 0 && field > 0)
259 int64_t frame_number = field / 2;
270 int64_t end_samples_frame = start_samples_frame;
271 int end_samples_position = start_samples_position;
274 while (remaining_samples > 0)
281 if (original_samples >= remaining_samples)
284 end_samples_position += remaining_samples - 1;
285 remaining_samples = 0;
289 end_samples_frame += 1;
290 end_samples_position = 0;
291 remaining_samples -= original_samples;
301 start_samples_frame = end_samples_frame;
302 start_samples_position = end_samples_position + 1;
305 start_samples_frame += 1;
306 start_samples_position = 0;
339 frame.
Odd.
Frame = TargetFrameNumber;
349 if(TargetFrameNumber < 1 ||
frames.size() == 0)
353 else if (TargetFrameNumber > (int64_t)
frames.size())
355 TargetFrameNumber =
frames.size();
358 ZmqLogger::Instance()->
AppendDebugMethod(
"FrameMapper::GetMappedFrame",
"TargetFrameNumber", TargetFrameNumber,
"frames.size()",
frames.size(),
"frames[...].Odd",
frames[TargetFrameNumber - 1].Odd.Frame,
"frames[...].Even",
frames[TargetFrameNumber - 1].Even.Frame);
361 return frames[TargetFrameNumber - 1];
365 std::shared_ptr<Frame> FrameMapper::GetOrCreateFrame(int64_t number)
367 std::shared_ptr<Frame> new_frame;
377 new_frame = reader->
GetFrame(number);
395 new_frame->AddAudioSilence(samples_in_frame);
403 std::shared_ptr<Frame> final_frame = final_cache.
GetFrame(requested_frame);
404 if (final_frame)
return final_frame;
412 float position = parent->
Position();
413 float start = parent->
Start();
414 if (parent_position != position || parent_start != start) {
426 final_frame = final_cache.
GetFrame(requested_frame);
427 if (final_frame)
return final_frame;
431 int minimum_frames = 1;
437 for (int64_t frame_number = requested_frame; frame_number < requested_frame + minimum_frames; frame_number++)
441 ZmqLogger::Instance()->
AppendDebugMethod(
"FrameMapper::GetFrame (inside omp for loop)",
"frame_number", frame_number,
"minimum_frames", minimum_frames,
"requested_frame", requested_frame);
445 std::shared_ptr<Frame> mapped_frame;
448 mapped_frame = GetOrCreateFrame(mapped.
Odd.
Frame);
451 int channels_in_frame = mapped_frame->GetAudioChannelsCount();
452 int samples_in_frame =
Frame::GetSamplesPerFrame(AdjustFrameNumber(frame_number), target, mapped_frame->SampleRate(), channels_in_frame);
461 info.
channels == mapped_frame->GetAudioChannelsCount() &&
463 mapped.
Samples.
total == mapped_frame->GetAudioSamplesCount() &&
466 mapped_frame->number == frame_number &&
470 final_cache.
Add(mapped_frame);
475 auto frame = std::make_shared<Frame>(
476 frame_number, 1, 1,
"#000000", samples_in_frame, channels_in_frame);
477 frame->SampleRate(mapped_frame->SampleRate());
478 frame->ChannelsLayout(mapped_frame->ChannelsLayout());
482 std::shared_ptr<Frame> odd_frame;
483 odd_frame = GetOrCreateFrame(mapped.
Odd.
Frame);
486 frame->AddImage(std::make_shared<QImage>(*odd_frame->GetImage()),
true);
489 std::shared_ptr<Frame> even_frame;
490 even_frame = GetOrCreateFrame(mapped.
Even.
Frame);
493 std::make_shared<QImage>(*even_frame->GetImage()),
false);
497 bool need_resampling =
false;
503 need_resampling =
true;
514 const int EXTRA_INPUT_SAMPLES = 100;
517 copy_samples.
sample_end += EXTRA_INPUT_SAMPLES;
518 int samples_per_end_frame =
521 if (copy_samples.
sample_end >= samples_per_end_frame)
525 copy_samples.
sample_end -= samples_per_end_frame;
527 copy_samples.
total += EXTRA_INPUT_SAMPLES;
534 int samples_per_start_frame =
537 if (copy_samples.
sample_start >= samples_per_start_frame)
543 copy_samples.
total -= EXTRA_INPUT_SAMPLES;
548 int samples_copied = 0;
553 int remaining_samples = copy_samples.
total - samples_copied;
554 int number_to_copy = 0;
557 std::shared_ptr<Frame> original_frame = GetOrCreateFrame(starting_frame);
558 int original_samples = original_frame->GetAudioSamplesCount();
561 for (
int channel = 0; channel < channels_in_frame; channel++)
566 number_to_copy = original_samples - copy_samples.
sample_start;
567 if (number_to_copy > remaining_samples)
568 number_to_copy = remaining_samples;
571 frame->AddAudio(
true, channel, samples_copied, original_frame->GetAudioSamples(channel) + copy_samples.
sample_start, number_to_copy, 1.0);
576 number_to_copy = original_samples;
577 if (number_to_copy > remaining_samples)
578 number_to_copy = remaining_samples;
581 frame->AddAudio(
true, channel, samples_copied, original_frame->GetAudioSamples(channel), number_to_copy, 1.0);
587 if (number_to_copy > remaining_samples)
588 number_to_copy = remaining_samples;
591 frame->AddAudio(
false, channel, samples_copied, original_frame->GetAudioSamples(channel), number_to_copy, 1.0);
596 samples_copied += number_to_copy;
606 final_cache.
Add(frame);
611 return final_cache.
GetFrame(requested_frame);
622 for (
float map = 1; map <=
frames.size(); map++)
625 cout <<
"Target frame #: " << map <<
" mapped to original frame #:\t(" << frame.
Odd.
Frame <<
" odd, " << frame.
Even.
Frame <<
" even)" << endl;
696 root[
"type"] =
"FrameMapper";
712 catch (
const std::exception& e)
715 throw InvalidJSON(
"JSON is invalid (missing keys or invalid data types)");
736 ZmqLogger::Instance()->
AppendDebugMethod(
"FrameMapper::ChangeMapping",
"target_fps.num", target_fps.
num,
"target_fps.den", target_fps.
den,
"target_pulldown", target_pulldown,
"target_sample_rate", target_sample_rate,
"target_channels", target_channels,
"target_channel_layout", target_channel_layout);
742 target.
num = target_fps.
num;
743 target.
den = target_fps.
den;
748 pulldown = target_pulldown;
776 int total_frame_samples = 0;
777 int channels_in_frame = frame->GetAudioChannelsCount();
778 int sample_rate_in_frame = frame->SampleRate();
779 int samples_in_frame = frame->GetAudioSamplesCount();
780 ChannelLayout channel_layout_in_frame = frame->ChannelsLayout();
782 ZmqLogger::Instance()->
AppendDebugMethod(
"FrameMapper::ResampleMappedAudio",
"frame->number", frame->number,
"original_frame_number", original_frame_number,
"channels_in_frame", channels_in_frame,
"samples_in_frame", samples_in_frame,
"sample_rate_in_frame", sample_rate_in_frame);
785 float* frame_samples_float = NULL;
787 frame_samples_float = frame->GetInterleavedAudioSamples(sample_rate_in_frame, NULL, &samples_in_frame);
790 total_frame_samples = samples_in_frame * channels_in_frame;
793 int16_t* frame_samples = (int16_t*) av_malloc(
sizeof(int16_t)*total_frame_samples);
798 const int16_t max16 = 32767;
799 const int16_t min16 = -32768;
800 for (
int s = 0; s < total_frame_samples; s++) {
801 valF = frame_samples_float[s] * (1 << 15);
804 else if (valF < min16)
807 conv = int(valF + 32768.5) - 32768;
810 frame_samples[s] = conv;
815 delete[] frame_samples_float;
816 frame_samples_float = NULL;
818 ZmqLogger::Instance()->
AppendDebugMethod(
"FrameMapper::ResampleMappedAudio (got sample data from frame)",
"frame->number", frame->number,
"total_frame_samples", total_frame_samples,
"target channels",
info.
channels,
"channels_in_frame", channels_in_frame,
"target sample_rate",
info.
sample_rate,
"samples_in_frame", samples_in_frame);
824 audio_frame->nb_samples = total_frame_samples / channels_in_frame;
826 int error_code = avcodec_fill_audio_frame(audio_frame, channels_in_frame, AV_SAMPLE_FMT_S16, (uint8_t *) frame_samples,
827 audio_frame->nb_samples * av_get_bytes_per_sample(AV_SAMPLE_FMT_S16) * channels_in_frame, 1);
832 "FrameMapper::ResampleMappedAudio ERROR [" + av_err2string(error_code) +
"]",
833 "error_code", error_code);
834 throw ErrorEncodingVideo(
"Error while resampling audio in frame mapper", frame->number);
840 ZmqLogger::Instance()->
AppendDebugMethod(
"FrameMapper::ResampleMappedAudio (adjust # of samples)",
"total_frame_samples", total_frame_samples,
"info.sample_rate",
info.
sample_rate,
"sample_rate_in_frame", sample_rate_in_frame,
"info.channels",
info.
channels,
"channels_in_frame", channels_in_frame,
"original_frame_number", original_frame_number);
845 audio_converted->nb_samples = total_frame_samples;
846 av_samples_alloc(audio_converted->data, audio_converted->linesize,
info.
channels, total_frame_samples, AV_SAMPLE_FMT_S16, 0);
848 ZmqLogger::Instance()->
AppendDebugMethod(
"FrameMapper::ResampleMappedAudio (preparing for resample)",
"in_sample_fmt", AV_SAMPLE_FMT_S16,
"out_sample_fmt", AV_SAMPLE_FMT_S16,
"in_sample_rate", sample_rate_in_frame,
"out_sample_rate",
info.
sample_rate,
"in_channels", channels_in_frame,
"out_channels",
info.
channels);
855 av_opt_set_int(avr,
"in_channel_layout", channel_layout_in_frame, 0);
857 av_opt_set_int(avr,
"in_sample_fmt", AV_SAMPLE_FMT_S16, 0);
858 av_opt_set_int(avr,
"out_sample_fmt", AV_SAMPLE_FMT_S16, 0);
859 av_opt_set_int(avr,
"in_sample_rate", sample_rate_in_frame, 0);
861 av_opt_set_int(avr,
"in_channels", channels_in_frame, 0);
868 audio_converted->data,
869 audio_converted->linesize[0],
870 audio_converted->nb_samples,
872 audio_frame->linesize[0],
873 audio_frame->nb_samples);
876 int16_t* resampled_samples =
new int16_t[(nb_samples *
info.
channels)];
879 memcpy(resampled_samples, audio_converted->data[0], (nb_samples * av_get_bytes_per_sample(AV_SAMPLE_FMT_S16) *
info.
channels));
882 av_freep(&audio_frame->data[0]);
884 av_freep(&audio_converted->data[0]);
886 frame_samples = NULL;
889 int channel_buffer_size = nb_samples;
892 ZmqLogger::Instance()->
AppendDebugMethod(
"FrameMapper::ResampleMappedAudio (Audio successfully resampled)",
"nb_samples", nb_samples,
"total_frame_samples", total_frame_samples,
"info.sample_rate",
info.
sample_rate,
"channels_in_frame", channels_in_frame,
"info.channels",
info.
channels,
"info.channel_layout",
info.
channel_layout);
895 float *channel_buffer =
new float[channel_buffer_size];
898 for (
int channel_filter = 0; channel_filter <
info.
channels; channel_filter++)
901 for (
int z = 0; z < channel_buffer_size; z++)
902 channel_buffer[z] = 0.0f;
908 for (
int sample = 0; sample < (nb_samples *
info.
channels); sample++)
911 if (channel_filter == channel)
914 channel_buffer[position] = resampled_samples[sample] * (1.0f / (1 << 15));
930 frame->AddAudio(
true, channel_filter, 0, channel_buffer, position, 1.0f);
940 delete[] channel_buffer;
941 channel_buffer = NULL;
944 delete[] resampled_samples;
945 resampled_samples = NULL;
949 int64_t FrameMapper::AdjustFrameNumber(int64_t clip_frame_number) {
952 float position = 0.0;
957 start = parent->
Start();
964 int64_t clip_start_position = round(position *
info.
fps.
ToDouble()) + 1;
965 int64_t frame_number = clip_frame_number + clip_start_position - clip_start_frame;
Header file for Clip class.
Header file for all Exception classes.
#define AV_FREE_FRAME(av_frame)
#define SWR_CONVERT(ctx, out, linesize, out_count, in, linesize2, in_count)
#define AV_ALLOCATE_FRAME()
#define AV_RESET_FRAME(av_frame)
Header file for the FrameMapper class.
#define OPEN_MP_NUM_PROCESSORS
void SetMaxBytesFromInfo(int64_t number_of_frames, int width, int height, int sample_rate, int channels)
Set maximum bytes to a different amount based on a ReaderInfo struct.
void Add(std::shared_ptr< openshot::Frame > frame)
Add a Frame to the cache.
std::shared_ptr< openshot::Frame > GetFrame(int64_t frame_number)
Get a frame from the cache.
void Clear()
Clear the cache of all frames.
float Start() const
Get start position (in seconds) of clip (trim start of video)
float Position() const
Get position on timeline (in seconds)
This class represents a clip (used to arrange readers on the timeline)
Exception when encoding audio packet.
This class represents a fraction.
int num
Numerator for the fraction.
float ToFloat()
Return this fraction as a float (i.e. 1/2 = 0.5)
double ToDouble() const
Return this fraction as a double (i.e. 1/2 = 0.5)
int ToInt()
Return a rounded integer of the fraction (for example 30000/1001 returns 30)
int den
Denominator for the fraction.
std::shared_ptr< Frame > GetFrame(int64_t requested_frame) override
This method is required for all derived classes of ReaderBase, and return the openshot::Frame object,...
void ChangeMapping(Fraction target_fps, PulldownType pulldown, int target_sample_rate, int target_channels, ChannelLayout target_channel_layout)
Change frame rate or audio mapping details.
MappedFrame GetMappedFrame(int64_t TargetFrameNumber)
Get a frame based on the target frame rate and the new frame number of a frame.
void SetJsonValue(const Json::Value root) override
Load Json::Value into this object.
bool IsOpen() override
Determine if reader is open or closed.
std::vector< Field > fields
ReaderBase * Reader()
Get the current reader.
std::vector< MappedFrame > frames
void PrintMapping()
Print all of the original frames and which new frames they map to.
void ResampleMappedAudio(std::shared_ptr< Frame > frame, int64_t original_frame_number)
Resample audio and map channels (if needed)
void Open() override
Open the internal reader.
void Close() override
Close the openshot::FrameMapper and internal reader.
std::string Json() const override
Generate JSON string of this object.
void SetJson(const std::string value) override
Load JSON string into this object.
Json::Value JsonValue() const override
Generate Json::Value for this object.
virtual ~FrameMapper()
Destructor.
int GetSamplesPerFrame(openshot::Fraction fps, int sample_rate, int channels)
Calculate the # of samples per video frame (for the current frame number)
Exception for invalid JSON.
Exception for frames that are out of bounds.
This abstract class is the base class, used by all readers in libopenshot.
virtual bool IsOpen()=0
Determine if reader is open or closed.
juce::CriticalSection getFrameCriticalSection
Section lock for multiple threads.
openshot::ReaderInfo info
Information about the current media file.
virtual void SetJsonValue(const Json::Value root)=0
Load Json::Value into this object.
virtual Json::Value JsonValue() const =0
Generate Json::Value for this object.
virtual std::shared_ptr< openshot::Frame > GetFrame(int64_t number)=0
virtual void Open()=0
Open the reader (and start consuming resources, such as images or video files)
openshot::ClipBase * ParentClip()
Parent clip object of this reader (which can be unparented and NULL)
virtual void Close()=0
Close the reader (and any resources it was consuming)
Exception when a reader is closed, and a frame is requested.
void AppendDebugMethod(std::string method_name, std::string arg1_name="", float arg1_value=-1.0, std::string arg2_name="", float arg2_value=-1.0, std::string arg3_name="", float arg3_value=-1.0, std::string arg4_name="", float arg4_value=-1.0, std::string arg5_name="", float arg5_value=-1.0, std::string arg6_name="", float arg6_value=-1.0)
Append debug information.
static ZmqLogger * Instance()
Create or get an instance of this logger singleton (invoke the class with this method)
This namespace is the default namespace for all code in the openshot library.
PulldownType
This enumeration determines how frame rates are increased or decreased.
@ PULLDOWN_CLASSIC
Classic 2:3:2:3 pull-down.
@ PULLDOWN_ADVANCED
Advanced 2:3:3:2 pull-down (minimal dirty frames)
@ PULLDOWN_NONE
Do not apply pull-down techniques, just repeat or skip entire frames.
ChannelLayout
This enumeration determines the audio channel layout (such as stereo, mono, 5 point surround,...
const Json::Value stringToJson(const std::string value)
This struct holds a single field (half a frame).
This struct holds two fields which together make up a complete video frame.
bool has_single_image
Determines if this file only contains a single image.
float duration
Length of time (in seconds)
int width
The width of the video (in pixesl)
int channels
The number of audio channels used in the audio stream.
openshot::Fraction fps
Frames per second, as a fraction (i.e. 24/1 = 24 fps)
int height
The height of the video (in pixels)
int64_t video_length
The number of frames in the video stream.
openshot::ChannelLayout channel_layout
The channel layout (mono, stereo, 5 point surround, etc...)
bool has_video
Determines if this file has a video stream.
bool has_audio
Determines if this file has an audio stream.
openshot::Fraction video_timebase
The video timebase determines how long each frame stays on the screen.
int sample_rate
The number of audio samples per second (44100 is a common sample rate)
This struct holds a the range of samples needed by this frame.