OpenShot Library | libopenshot  0.2.5
FrameMapper.cpp
Go to the documentation of this file.
1 /**
2  * @file
3  * @brief Source file for the FrameMapper class
4  * @author Jonathan Thomas <jonathan@openshot.org>
5  *
6  * @ref License
7  */
8 
9 /* LICENSE
10  *
11  * Copyright (c) 2008-2019 OpenShot Studios, LLC
12  * <http://www.openshotstudios.com/>. This file is part of
13  * OpenShot Library (libopenshot), an open-source project dedicated to
14  * delivering high quality video editing and animation solutions to the
15  * world. For more information visit <http://www.openshot.org/>.
16  *
17  * OpenShot Library (libopenshot) is free software: you can redistribute it
18  * and/or modify it under the terms of the GNU Lesser General Public License
19  * as published by the Free Software Foundation, either version 3 of the
20  * License, or (at your option) any later version.
21  *
22  * OpenShot Library (libopenshot) is distributed in the hope that it will be
23  * useful, but WITHOUT ANY WARRANTY; without even the implied warranty of
24  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
25  * GNU Lesser General Public License for more details.
26  *
27  * You should have received a copy of the GNU Lesser General Public License
28  * along with OpenShot Library. If not, see <http://www.gnu.org/licenses/>.
29  */
30 
31 #include "../include/FrameMapper.h"
32 
33 using namespace std;
34 using namespace openshot;
35 
36 FrameMapper::FrameMapper(ReaderBase *reader, Fraction target, PulldownType target_pulldown, int target_sample_rate, int target_channels, ChannelLayout target_channel_layout) :
37  reader(reader), target(target), pulldown(target_pulldown), is_dirty(true), avr(NULL)
38 {
39  // Set the original frame rate from the reader
40  original = Fraction(reader->info.fps.num, reader->info.fps.den);
41 
42  // Set all info struct members equal to the internal reader
43  info = reader->info;
44  info.fps.num = target.num;
45  info.fps.den = target.den;
46  info.video_timebase.num = target.den;
47  info.video_timebase.den = target.num;
49  info.sample_rate = target_sample_rate;
50  info.channels = target_channels;
51  info.channel_layout = target_channel_layout;
52  info.width = reader->info.width;
53  info.height = reader->info.height;
54 
55  // Used to toggle odd / even fields
56  field_toggle = true;
57 
58  // Adjust cache size based on size of frame and audio
60 }
61 
62 // Destructor
64  if (is_open)
65  // Auto Close if not already
66  Close();
67 
68  reader = NULL;
69 }
70 
71 /// Get the current reader
73 {
74  if (reader)
75  return reader;
76  else
77  // Throw error if reader not initialized
78  throw ReaderClosed("No Reader has been initialized for FrameMapper. Call Reader(*reader) before calling this method.");
79 }
80 
81 void FrameMapper::AddField(int64_t frame)
82 {
83  // Add a field, and toggle the odd / even field
84  AddField(Field(frame, field_toggle));
85 }
86 
87 void FrameMapper::AddField(Field field)
88 {
89  // Add a field to the end of the field list
90  fields.push_back(field);
91 
92  // toggle the odd / even flag
93  field_toggle = (field_toggle ? false : true);
94 }
95 
96 // Use the original and target frame rates and a pull-down technique to create
97 // a mapping between the original fields and frames or a video to a new frame rate.
98 // This might repeat or skip fields and frames of the original video, depending on
99 // whether the frame rate is increasing or decreasing.
100 void FrameMapper::Init()
101 {
102  ZmqLogger::Instance()->AppendDebugMethod("FrameMapper::Init (Calculate frame mappings)");
103 
104  // Do not initialize anything if just a picture with no audio
106  // Skip initialization
107  return;
108 
109  // Clear the fields & frames lists
110  fields.clear();
111  frames.clear();
112 
113  // Mark as not dirty
114  is_dirty = false;
115 
116  // Clear cache
117  final_cache.Clear();
118 
119  // Some framerates are handled special, and some use a generic Keyframe curve to
120  // map the framerates. These are the special framerates:
121  if ((fabs(original.ToFloat() - 24.0) < 1e-7 || fabs(original.ToFloat() - 25.0) < 1e-7 || fabs(original.ToFloat() - 30.0) < 1e-7) &&
122  (fabs(target.ToFloat() - 24.0) < 1e-7 || fabs(target.ToFloat() - 25.0) < 1e-7 || fabs(target.ToFloat() - 30.0) < 1e-7)) {
123 
124  // Get the difference (in frames) between the original and target frame rates
125  float difference = target.ToInt() - original.ToInt();
126 
127  // Find the number (i.e. interval) of fields that need to be skipped or repeated
128  int field_interval = 0;
129  int frame_interval = 0;
130 
131  if (difference != 0)
132  {
133  field_interval = round(fabs(original.ToInt() / difference));
134 
135  // Get frame interval (2 fields per frame)
136  frame_interval = field_interval * 2.0f;
137  }
138 
139 
140  // Calculate # of fields to map
141  int64_t frame = 1;
142  int64_t number_of_fields = reader->info.video_length * 2;
143 
144  // Loop through all fields in the original video file
145  for (int64_t field = 1; field <= number_of_fields; field++)
146  {
147 
148  if (difference == 0) // Same frame rate, NO pull-down or special techniques required
149  {
150  // Add fields
151  AddField(frame);
152  }
153  else if (difference > 0) // Need to ADD fake fields & frames, because original video has too few frames
154  {
155  // Add current field
156  AddField(frame);
157 
158  if (pulldown == PULLDOWN_CLASSIC && field % field_interval == 0)
159  {
160  // Add extra field for each 'field interval
161  AddField(frame);
162  }
163  else if (pulldown == PULLDOWN_ADVANCED && field % field_interval == 0 && field % frame_interval != 0)
164  {
165  // Add both extra fields in the middle 'together' (i.e. 2:3:3:2 technique)
166  AddField(frame); // add field for current frame
167 
168  if (frame + 1 <= info.video_length)
169  // add field for next frame (if the next frame exists)
170  AddField(Field(frame + 1, field_toggle));
171  }
172  else if (pulldown == PULLDOWN_NONE && field % frame_interval == 0)
173  {
174  // No pull-down technique needed, just repeat this frame
175  AddField(frame);
176  AddField(frame);
177  }
178  }
179  else if (difference < 0) // Need to SKIP fake fields & frames, because we want to return to the original film frame rate
180  {
181 
182  if (pulldown == PULLDOWN_CLASSIC && field % field_interval == 0)
183  {
184  // skip current field and toggle the odd/even flag
185  field_toggle = (field_toggle ? false : true);
186  }
187  else if (pulldown == PULLDOWN_ADVANCED && field % field_interval == 0 && field % frame_interval != 0)
188  {
189  // skip this field, plus the next field
190  field++;
191  }
192  else if (pulldown == PULLDOWN_NONE && frame % field_interval == 0)
193  {
194  // skip this field, plus the next one
195  field++;
196  }
197  else
198  {
199  // No skipping needed, so add the field
200  AddField(frame);
201  }
202  }
203 
204  // increment frame number (if field is divisible by 2)
205  if (field % 2 == 0 && field > 0)
206  frame++;
207  }
208 
209  } else {
210  // Map the remaining framerates using a linear algorithm
211  double rate_diff = target.ToDouble() / original.ToDouble();
212  int64_t new_length = reader->info.video_length * rate_diff;
213 
214  // Calculate the value difference
215  double value_increment = (reader->info.video_length + 1) / (double) (new_length);
216 
217  // Loop through curve, and build list of frames
218  double original_frame_num = 1.0f;
219  for (int64_t frame_num = 1; frame_num <= new_length; frame_num++)
220  {
221  // Add 2 fields per frame
222  AddField(round(original_frame_num));
223  AddField(round(original_frame_num));
224 
225  // Increment original frame number
226  original_frame_num += value_increment;
227  }
228  }
229 
230  // Loop through the target frames again (combining fields into frames)
231  Field Odd(0, true); // temp field used to track the ODD field
232  Field Even(0, true); // temp field used to track the EVEN field
233 
234  // Variables used to remap audio samples
235  int64_t start_samples_frame = 1;
236  int start_samples_position = 0;
237 
238  for (std::vector<Field>::size_type field = 1; field <= fields.size(); field++)
239  {
240  // Get the current field
241  Field f = fields[field - 1];
242 
243  // Is field divisible by 2?
244  if (field % 2 == 0 && field > 0)
245  {
246  // New frame number
247  int64_t frame_number = field / 2;
248 
249  // Set the bottom frame
250  if (f.isOdd)
251  Odd = f;
252  else
253  Even = f;
254 
255  // Determine the range of samples (from the original rate). Resampling happens in real-time when
256  // calling the GetFrame() method. So this method only needs to redistribute the original samples with
257  // the original sample rate.
258  int64_t end_samples_frame = start_samples_frame;
259  int end_samples_position = start_samples_position;
260  int remaining_samples = Frame::GetSamplesPerFrame(frame_number, target, reader->info.sample_rate, reader->info.channels);
261 
262  while (remaining_samples > 0)
263  {
264  // get original samples
265  int original_samples = Frame::GetSamplesPerFrame(end_samples_frame, original, reader->info.sample_rate, reader->info.channels) - end_samples_position;
266 
267  // Enough samples
268  if (original_samples >= remaining_samples)
269  {
270  // Take all that we need, and break loop
271  end_samples_position += remaining_samples - 1;
272  remaining_samples = 0;
273  } else
274  {
275  // Not enough samples (take them all, and keep looping)
276  end_samples_frame += 1; // next frame
277  end_samples_position = 0; // next frame, starting on 1st sample
278  remaining_samples -= original_samples; // reduce the remaining amount
279  }
280  }
281 
282 
283 
284  // Create the sample mapping struct
285  SampleRange Samples = {start_samples_frame, start_samples_position, end_samples_frame, end_samples_position, Frame::GetSamplesPerFrame(frame_number, target, reader->info.sample_rate, reader->info.channels)};
286 
287  // Reset the audio variables
288  start_samples_frame = end_samples_frame;
289  start_samples_position = end_samples_position + 1;
290  if (start_samples_position >= Frame::GetSamplesPerFrame(start_samples_frame, original, reader->info.sample_rate, reader->info.channels))
291  {
292  start_samples_frame += 1; // increment the frame (since we need to wrap onto the next one)
293  start_samples_position = 0; // reset to 0, since we wrapped
294  }
295 
296  // Create a frame and ADD it to the frames collection
297  MappedFrame frame = {Odd, Even, Samples};
298  frames.push_back(frame);
299  }
300  else
301  {
302  // Set the top field
303  if (f.isOdd)
304  Odd = f;
305  else
306  Even = f;
307  }
308  }
309 
310  // Clear the internal fields list (no longer needed)
311  fields.clear();
312 }
313 
314 MappedFrame FrameMapper::GetMappedFrame(int64_t TargetFrameNumber)
315 {
316  // Check if mappings are dirty (and need to be recalculated)
317  if (is_dirty)
318  // Recalculate mappings
319  Init();
320 
321  // Ignore mapping on single image readers
323  // Return the same number
324  MappedFrame frame;
325  frame.Even.Frame = TargetFrameNumber;
326  frame.Odd.Frame = TargetFrameNumber;
327  frame.Samples.frame_start = 0;
328  frame.Samples.frame_end = 0;
329  frame.Samples.sample_start = 0;
330  frame.Samples.sample_end = 0;
331  frame.Samples.total = 0;
332  return frame;
333  }
334 
335  // Check if frame number is valid
336  if(TargetFrameNumber < 1 || frames.size() == 0)
337  // frame too small, return error
338  throw OutOfBoundsFrame("An invalid frame was requested.", TargetFrameNumber, frames.size());
339 
340  else if (TargetFrameNumber > (int64_t)frames.size())
341  // frame too large, set to end frame
342  TargetFrameNumber = frames.size();
343 
344  // Debug output
345  ZmqLogger::Instance()->AppendDebugMethod("FrameMapper::GetMappedFrame", "TargetFrameNumber", TargetFrameNumber, "frames.size()", frames.size(), "frames[...].Odd", frames[TargetFrameNumber - 1].Odd.Frame, "frames[...].Even", frames[TargetFrameNumber - 1].Even.Frame);
346 
347  // Return frame
348  return frames[TargetFrameNumber - 1];
349 }
350 
351 // Get or generate a blank frame
352 std::shared_ptr<Frame> FrameMapper::GetOrCreateFrame(int64_t number)
353 {
354  std::shared_ptr<Frame> new_frame;
355 
356  // Init some basic properties about this frame (keep sample rate and # channels the same as the original reader for now)
357  int samples_in_frame = Frame::GetSamplesPerFrame(number, target, reader->info.sample_rate, reader->info.channels);
358 
359  try {
360  // Debug output
361  ZmqLogger::Instance()->AppendDebugMethod("FrameMapper::GetOrCreateFrame (from reader)", "number", number, "samples_in_frame", samples_in_frame);
362 
363  // Attempt to get a frame (but this could fail if a reader has just been closed)
364  new_frame = reader->GetFrame(number);
365 
366  // Return real frame
367  return new_frame;
368 
369  } catch (const ReaderClosed & e) {
370  // ...
371  } catch (const TooManySeeks & e) {
372  // ...
373  } catch (const OutOfBoundsFrame & e) {
374  // ...
375  }
376 
377  // Debug output
378  ZmqLogger::Instance()->AppendDebugMethod("FrameMapper::GetOrCreateFrame (create blank)", "number", number, "samples_in_frame", samples_in_frame);
379 
380  // Create blank frame
381  new_frame = std::make_shared<Frame>(number, info.width, info.height, "#000000", samples_in_frame, reader->info.channels);
382  new_frame->SampleRate(reader->info.sample_rate);
383  new_frame->ChannelsLayout(info.channel_layout);
384  new_frame->AddAudioSilence(samples_in_frame);
385  return new_frame;
386 }
387 
388 // Get an openshot::Frame object for a specific frame number of this reader.
389 std::shared_ptr<Frame> FrameMapper::GetFrame(int64_t requested_frame)
390 {
391  // Check final cache, and just return the frame (if it's available)
392  std::shared_ptr<Frame> final_frame = final_cache.GetFrame(requested_frame);
393  if (final_frame) return final_frame;
394 
395  // Create a scoped lock, allowing only a single thread to run the following code at one time
396  const GenericScopedLock<CriticalSection> lock(getFrameCriticalSection);
397 
398  // Check if mappings are dirty (and need to be recalculated)
399  if (is_dirty)
400  // Recalculate mappings
401  Init();
402 
403  // Check final cache a 2nd time (due to potential lock already generating this frame)
404  final_frame = final_cache.GetFrame(requested_frame);
405  if (final_frame) return final_frame;
406 
407  // Minimum number of frames to process (for performance reasons)
408  // Dialing this down to 1 for now, as it seems to improve performance, and reduce export crashes
409  int minimum_frames = 1;
410 
411  // Debug output
412  ZmqLogger::Instance()->AppendDebugMethod("FrameMapper::GetFrame (Loop through frames)", "requested_frame", requested_frame, "minimum_frames", minimum_frames);
413 
414  // Loop through all requested frames
415  for (int64_t frame_number = requested_frame; frame_number < requested_frame + minimum_frames; frame_number++)
416  {
417 
418  // Debug output
419  ZmqLogger::Instance()->AppendDebugMethod("FrameMapper::GetFrame (inside omp for loop)", "frame_number", frame_number, "minimum_frames", minimum_frames, "requested_frame", requested_frame);
420 
421  // Get the mapped frame
422  MappedFrame mapped = GetMappedFrame(frame_number);
423  std::shared_ptr<Frame> mapped_frame;
424 
425  // Get the mapped frame (keeping the sample rate and channels the same as the original... for the moment)
426  mapped_frame = GetOrCreateFrame(mapped.Odd.Frame);
427 
428  // Get # of channels in the actual frame
429  int channels_in_frame = mapped_frame->GetAudioChannelsCount();
430  int samples_in_frame = Frame::GetSamplesPerFrame(frame_number, target, mapped_frame->SampleRate(), channels_in_frame);
431 
432  // Determine if mapped frame is identical to source frame
433  // including audio sample distribution according to mapped.Samples,
434  // and frame_number. In some cases such as end of stream, the reader
435  // will return a frame with a different frame number. In these cases,
436  // we cannot use the frame as is, nor can we modify the frame number,
437  // otherwise the reader's cache object internals become invalid.
438  if (info.sample_rate == mapped_frame->SampleRate() &&
439  info.channels == mapped_frame->GetAudioChannelsCount() &&
440  info.channel_layout == mapped_frame->ChannelsLayout() &&
441  mapped.Samples.total == mapped_frame->GetAudioSamplesCount() &&
442  mapped.Samples.frame_start == mapped.Odd.Frame &&
443  mapped.Samples.sample_start == 0 &&
444  mapped_frame->number == frame_number &&// in some conditions (e.g. end of stream)
445  info.fps.num == reader->info.fps.num &&
446  info.fps.den == reader->info.fps.den) {
447  // Add original frame to cache, and skip the rest (for performance reasons)
448  final_cache.Add(mapped_frame);
449  continue;
450  }
451 
452  // Create a new frame
453  std::shared_ptr<Frame> frame = std::make_shared<Frame>(frame_number, 1, 1, "#000000", samples_in_frame, channels_in_frame);
454  frame->SampleRate(mapped_frame->SampleRate());
455  frame->ChannelsLayout(mapped_frame->ChannelsLayout());
456 
457 
458  // Copy the image from the odd field
459  std::shared_ptr<Frame> odd_frame;
460  odd_frame = GetOrCreateFrame(mapped.Odd.Frame);
461 
462  if (odd_frame)
463  frame->AddImage(std::shared_ptr<QImage>(new QImage(*odd_frame->GetImage())), true);
464  if (mapped.Odd.Frame != mapped.Even.Frame) {
465  // Add even lines (if different than the previous image)
466  std::shared_ptr<Frame> even_frame;
467  even_frame = GetOrCreateFrame(mapped.Even.Frame);
468  if (even_frame)
469  frame->AddImage(std::shared_ptr<QImage>(new QImage(*even_frame->GetImage())), false);
470  }
471 
472  // Resample audio on frame (if needed)
473  bool need_resampling = false;
474  if (info.has_audio &&
475  (info.sample_rate != frame->SampleRate() ||
476  info.channels != frame->GetAudioChannelsCount() ||
477  info.channel_layout != frame->ChannelsLayout()))
478  // Resample audio and correct # of channels if needed
479  need_resampling = true;
480 
481  // create a copy of mapped.Samples that will be used by copy loop
482  SampleRange copy_samples = mapped.Samples;
483 
484  if (need_resampling)
485  {
486  // Resampling needed, modify copy of SampleRange object that
487  // includes some additional input samples on first iteration,
488  // and continues the offset to ensure that the sample rate
489  // converter isn't input limited.
490  const int EXTRA_INPUT_SAMPLES = 20;
491 
492  // Extend end sample count by an additional EXTRA_INPUT_SAMPLES samples
493  copy_samples.sample_end += EXTRA_INPUT_SAMPLES;
494  int samples_per_end_frame =
495  Frame::GetSamplesPerFrame(copy_samples.frame_end, original,
496  reader->info.sample_rate, reader->info.channels);
497  if (copy_samples.sample_end >= samples_per_end_frame)
498  {
499  // check for wrapping
500  copy_samples.frame_end++;
501  copy_samples.sample_end -= samples_per_end_frame;
502  }
503  copy_samples.total += EXTRA_INPUT_SAMPLES;
504 
505  if (avr) {
506  // Sample rate conversion has been allocated on this clip, so
507  // this is not the first iteration. Extend start position by
508  // EXTRA_INPUT_SAMPLES to keep step with previous frame
509  copy_samples.sample_start += EXTRA_INPUT_SAMPLES;
510  int samples_per_start_frame =
511  Frame::GetSamplesPerFrame(copy_samples.frame_start, original,
512  reader->info.sample_rate, reader->info.channels);
513  if (copy_samples.sample_start >= samples_per_start_frame)
514  {
515  // check for wrapping
516  copy_samples.frame_start++;
517  copy_samples.sample_start -= samples_per_start_frame;
518  }
519  copy_samples.total -= EXTRA_INPUT_SAMPLES;
520  }
521  }
522 
523  // Copy the samples
524  int samples_copied = 0;
525  int64_t starting_frame = copy_samples.frame_start;
526  while (info.has_audio && samples_copied < copy_samples.total)
527  {
528  // Init number of samples to copy this iteration
529  int remaining_samples = copy_samples.total - samples_copied;
530  int number_to_copy = 0;
531 
532  // number of original samples on this frame
533  std::shared_ptr<Frame> original_frame = GetOrCreateFrame(starting_frame);
534  int original_samples = original_frame->GetAudioSamplesCount();
535 
536  // Loop through each channel
537  for (int channel = 0; channel < channels_in_frame; channel++)
538  {
539  if (starting_frame == copy_samples.frame_start)
540  {
541  // Starting frame (take the ending samples)
542  number_to_copy = original_samples - copy_samples.sample_start;
543  if (number_to_copy > remaining_samples)
544  number_to_copy = remaining_samples;
545 
546  // Add samples to new frame
547  frame->AddAudio(true, channel, samples_copied, original_frame->GetAudioSamples(channel) + copy_samples.sample_start, number_to_copy, 1.0);
548  }
549  else if (starting_frame > copy_samples.frame_start && starting_frame < copy_samples.frame_end)
550  {
551  // Middle frame (take all samples)
552  number_to_copy = original_samples;
553  if (number_to_copy > remaining_samples)
554  number_to_copy = remaining_samples;
555 
556  // Add samples to new frame
557  frame->AddAudio(true, channel, samples_copied, original_frame->GetAudioSamples(channel), number_to_copy, 1.0);
558  }
559  else
560  {
561  // Ending frame (take the beginning samples)
562  number_to_copy = copy_samples.sample_end + 1;
563  if (number_to_copy > remaining_samples)
564  number_to_copy = remaining_samples;
565 
566  // Add samples to new frame
567  frame->AddAudio(false, channel, samples_copied, original_frame->GetAudioSamples(channel), number_to_copy, 1.0);
568  }
569  }
570 
571  // increment frame
572  samples_copied += number_to_copy;
573  starting_frame++;
574  }
575 
576  // Resample audio on frame (if needed)
577  if (need_resampling)
578  // Resample audio and correct # of channels if needed
579  ResampleMappedAudio(frame, mapped.Odd.Frame);
580 
581  // Add frame to final cache
582  final_cache.Add(frame);
583 
584  } // for loop
585 
586  // Return processed openshot::Frame
587  return final_cache.GetFrame(requested_frame);
588 }
589 
591 {
592  // Check if mappings are dirty (and need to be recalculated)
593  if (is_dirty)
594  // Recalculate mappings
595  Init();
596 
597  // Get the difference (in frames) between the original and target frame rates
598  float difference = target.ToInt() - original.ToInt();
599 
600  int field_interval = 0;
601  int frame_interval = 0;
602 
603  if (difference != 0)
604  {
605  // Find the number (i.e. interval) of fields that need to be skipped or repeated
606  field_interval = round(fabs(original.ToInt() / difference));
607 
608  // Get frame interval (2 fields per frame)
609  frame_interval = field_interval * 2.0f;
610  }
611 
612  // Loop through frame mappings
613  for (float map = 1; map <= frames.size(); map++)
614  {
615  MappedFrame frame = frames[map - 1];
616  cout << "Target frame #: " << map << " mapped to original frame #:\t(" << frame.Odd.Frame << " odd, " << frame.Even.Frame << " even)" << endl;
617  cout << " - Audio samples mapped to frame " << frame.Samples.frame_start << ":" << frame.Samples.sample_start << " to frame " << frame.Samples.frame_end << ":" << frame.Samples.sample_end << endl;
618  }
619 
620 }
621 
622 
623 // Determine if reader is open or closed
625  if (reader)
626  return reader->IsOpen();
627  else
628  return false;
629 }
630 
631 
632 // Open the internal reader
634 {
635  if (reader)
636  {
637  ZmqLogger::Instance()->AppendDebugMethod("FrameMapper::Open");
638 
639  // Open the reader
640  reader->Open();
641  }
642 }
643 
644 // Close the internal reader
646 {
647  if (reader)
648  {
649  // Create a scoped lock, allowing only a single thread to run the following code at one time
650  const GenericScopedLock<CriticalSection> lock(getFrameCriticalSection);
651 
652  ZmqLogger::Instance()->AppendDebugMethod("FrameMapper::Close");
653 
654  // Close internal reader
655  reader->Close();
656 
657  // Clear the fields & frames lists
658  fields.clear();
659  frames.clear();
660 
661  // Mark as dirty
662  is_dirty = true;
663 
664  // Clear cache
665  final_cache.Clear();
666 
667  // Deallocate resample buffer
668  if (avr) {
669  SWR_CLOSE(avr);
670  SWR_FREE(&avr);
671  avr = NULL;
672  }
673  }
674 }
675 
676 
677 // Generate JSON string of this object
678 std::string FrameMapper::Json() const {
679 
680  // Return formatted string
681  return JsonValue().toStyledString();
682 }
683 
684 // Generate Json::Value for this object
685 Json::Value FrameMapper::JsonValue() const {
686 
687  // Create root json object
688  Json::Value root = ReaderBase::JsonValue(); // get parent properties
689  root["type"] = "FrameMapper";
690 
691  // return JsonValue
692  return root;
693 }
694 
695 // Load JSON string into this object
696 void FrameMapper::SetJson(const std::string value) {
697 
698  // Parse JSON string into JSON objects
699  try
700  {
701  const Json::Value root = openshot::stringToJson(value);
702  // Set all values that match
703  SetJsonValue(root);
704  }
705  catch (const std::exception& e)
706  {
707  // Error parsing JSON (or missing keys)
708  throw InvalidJSON("JSON is invalid (missing keys or invalid data types)");
709  }
710 }
711 
712 // Load Json::Value into this object
713 void FrameMapper::SetJsonValue(const Json::Value root) {
714 
715  // Set parent data
717 
718  // Re-Open path, and re-init everything (if needed)
719  if (reader) {
720 
721  Close();
722  Open();
723  }
724 }
725 
726 // Change frame rate or audio mapping details
727 void FrameMapper::ChangeMapping(Fraction target_fps, PulldownType target_pulldown, int target_sample_rate, int target_channels, ChannelLayout target_channel_layout)
728 {
729  ZmqLogger::Instance()->AppendDebugMethod("FrameMapper::ChangeMapping", "target_fps.num", target_fps.num, "target_fps.den", target_fps.den, "target_pulldown", target_pulldown, "target_sample_rate", target_sample_rate, "target_channels", target_channels, "target_channel_layout", target_channel_layout);
730 
731  // Mark as dirty
732  is_dirty = true;
733 
734  // Update mapping details
735  target.num = target_fps.num;
736  target.den = target_fps.den;
737  info.fps.num = target_fps.num;
738  info.fps.den = target_fps.den;
739  info.video_timebase.num = target_fps.den;
740  info.video_timebase.den = target_fps.num;
741  pulldown = target_pulldown;
742  info.sample_rate = target_sample_rate;
743  info.channels = target_channels;
744  info.channel_layout = target_channel_layout;
745 
746  // Clear cache
747  final_cache.Clear();
748 
749  // Adjust cache size based on size of frame and audio
751 
752  // Deallocate resample buffer
753  if (avr) {
754  SWR_CLOSE(avr);
755  SWR_FREE(&avr);
756  avr = NULL;
757  }
758 }
759 
760 // Resample audio and map channels (if needed)
761 void FrameMapper::ResampleMappedAudio(std::shared_ptr<Frame> frame, int64_t original_frame_number)
762 {
763  // Check if mappings are dirty (and need to be recalculated)
764  if (is_dirty)
765  // Recalculate mappings
766  Init();
767 
768  // Init audio buffers / variables
769  int total_frame_samples = 0;
770  int channels_in_frame = frame->GetAudioChannelsCount();
771  int sample_rate_in_frame = frame->SampleRate();
772  int samples_in_frame = frame->GetAudioSamplesCount();
773  ChannelLayout channel_layout_in_frame = frame->ChannelsLayout();
774 
775  ZmqLogger::Instance()->AppendDebugMethod("FrameMapper::ResampleMappedAudio", "frame->number", frame->number, "original_frame_number", original_frame_number, "channels_in_frame", channels_in_frame, "samples_in_frame", samples_in_frame, "sample_rate_in_frame", sample_rate_in_frame);
776 
777  // Get audio sample array
778  float* frame_samples_float = NULL;
779  // Get samples interleaved together (c1 c2 c1 c2 c1 c2)
780  frame_samples_float = frame->GetInterleavedAudioSamples(sample_rate_in_frame, NULL, &samples_in_frame);
781 
782  // Calculate total samples
783  total_frame_samples = samples_in_frame * channels_in_frame;
784 
785  // Create a new array (to hold all S16 audio samples for the current queued frames)
786  int16_t* frame_samples = (int16_t*) av_malloc(sizeof(int16_t)*total_frame_samples);
787 
788  // Translate audio sample values back to 16 bit integers
789  for (int s = 0; s < total_frame_samples; s++)
790  // Translate sample value and copy into buffer
791  frame_samples[s] = int(frame_samples_float[s] * (1 << 15));
792 
793 
794  // Deallocate float array
795  delete[] frame_samples_float;
796  frame_samples_float = NULL;
797 
798  ZmqLogger::Instance()->AppendDebugMethod("FrameMapper::ResampleMappedAudio (got sample data from frame)", "frame->number", frame->number, "total_frame_samples", total_frame_samples, "target channels", info.channels, "channels_in_frame", channels_in_frame, "target sample_rate", info.sample_rate, "samples_in_frame", samples_in_frame);
799 
800 
801  // Create input frame (and allocate arrays)
802  AVFrame *audio_frame = AV_ALLOCATE_FRAME();
803  AV_RESET_FRAME(audio_frame);
804  audio_frame->nb_samples = total_frame_samples / channels_in_frame;
805 
806  int error_code = avcodec_fill_audio_frame(audio_frame, channels_in_frame, AV_SAMPLE_FMT_S16, (uint8_t *) frame_samples,
807  audio_frame->nb_samples * av_get_bytes_per_sample(AV_SAMPLE_FMT_S16) * channels_in_frame, 1);
808 
809  if (error_code < 0)
810  {
811  ZmqLogger::Instance()->AppendDebugMethod("FrameMapper::ResampleMappedAudio ERROR [" + (std::string)av_err2str(error_code) + "]", "error_code", error_code);
812  throw ErrorEncodingVideo("Error while resampling audio in frame mapper", frame->number);
813  }
814 
815  // Update total samples & input frame size (due to bigger or smaller data types)
816  total_frame_samples = Frame::GetSamplesPerFrame(frame->number, target, info.sample_rate, info.channels);
817 
818  ZmqLogger::Instance()->AppendDebugMethod("FrameMapper::ResampleMappedAudio (adjust # of samples)", "total_frame_samples", total_frame_samples, "info.sample_rate", info.sample_rate, "sample_rate_in_frame", sample_rate_in_frame, "info.channels", info.channels, "channels_in_frame", channels_in_frame, "original_frame_number", original_frame_number);
819 
820  // Create output frame (and allocate arrays)
821  AVFrame *audio_converted = AV_ALLOCATE_FRAME();
822  AV_RESET_FRAME(audio_converted);
823  audio_converted->nb_samples = total_frame_samples;
824  av_samples_alloc(audio_converted->data, audio_converted->linesize, info.channels, total_frame_samples, AV_SAMPLE_FMT_S16, 0);
825 
826  ZmqLogger::Instance()->AppendDebugMethod("FrameMapper::ResampleMappedAudio (preparing for resample)", "in_sample_fmt", AV_SAMPLE_FMT_S16, "out_sample_fmt", AV_SAMPLE_FMT_S16, "in_sample_rate", sample_rate_in_frame, "out_sample_rate", info.sample_rate, "in_channels", channels_in_frame, "out_channels", info.channels);
827 
828  int nb_samples = 0;
829 
830  // setup resample context
831  if (!avr) {
832  avr = SWR_ALLOC();
833  av_opt_set_int(avr, "in_channel_layout", channel_layout_in_frame, 0);
834  av_opt_set_int(avr, "out_channel_layout", info.channel_layout, 0);
835  av_opt_set_int(avr, "in_sample_fmt", AV_SAMPLE_FMT_S16, 0);
836  av_opt_set_int(avr, "out_sample_fmt", AV_SAMPLE_FMT_S16, 0);
837  av_opt_set_int(avr, "in_sample_rate", sample_rate_in_frame, 0);
838  av_opt_set_int(avr, "out_sample_rate", info.sample_rate, 0);
839  av_opt_set_int(avr, "in_channels", channels_in_frame, 0);
840  av_opt_set_int(avr, "out_channels", info.channels, 0);
841  SWR_INIT(avr);
842  }
843 
844  // Convert audio samples
845  nb_samples = SWR_CONVERT(avr, // audio resample context
846  audio_converted->data, // output data pointers
847  audio_converted->linesize[0], // output plane size, in bytes. (0 if unknown)
848  audio_converted->nb_samples, // maximum number of samples that the output buffer can hold
849  audio_frame->data, // input data pointers
850  audio_frame->linesize[0], // input plane size, in bytes (0 if unknown)
851  audio_frame->nb_samples); // number of input samples to convert
852 
853  // Create a new array (to hold all resampled S16 audio samples)
854  int16_t* resampled_samples = new int16_t[(nb_samples * info.channels)];
855 
856  // Copy audio samples over original samples
857  memcpy(resampled_samples, audio_converted->data[0], (nb_samples * av_get_bytes_per_sample(AV_SAMPLE_FMT_S16) * info.channels));
858 
859  // Free frames
860  av_freep(&audio_frame->data[0]);
861  AV_FREE_FRAME(&audio_frame);
862  av_freep(&audio_converted->data[0]);
863  AV_FREE_FRAME(&audio_converted);
864  frame_samples = NULL;
865 
866  // Resize the frame to hold the right # of channels and samples
867  int channel_buffer_size = nb_samples;
868  frame->ResizeAudio(info.channels, channel_buffer_size, info.sample_rate, info.channel_layout);
869 
870  ZmqLogger::Instance()->AppendDebugMethod("FrameMapper::ResampleMappedAudio (Audio successfully resampled)", "nb_samples", nb_samples, "total_frame_samples", total_frame_samples, "info.sample_rate", info.sample_rate, "channels_in_frame", channels_in_frame, "info.channels", info.channels, "info.channel_layout", info.channel_layout);
871 
872  // Array of floats (to hold samples for each channel)
873  float *channel_buffer = new float[channel_buffer_size];
874 
875  // Divide audio into channels. Loop through each channel
876  for (int channel_filter = 0; channel_filter < info.channels; channel_filter++)
877  {
878  // Init array
879  for (int z = 0; z < channel_buffer_size; z++)
880  channel_buffer[z] = 0.0f;
881 
882  // Loop through all samples and add them to our Frame based on channel.
883  // Toggle through each channel number, since channel data is stored like (left right left right)
884  int channel = 0;
885  int position = 0;
886  for (int sample = 0; sample < (nb_samples * info.channels); sample++)
887  {
888  // Only add samples for current channel
889  if (channel_filter == channel)
890  {
891  // Add sample (convert from (-32768 to 32768) to (-1.0 to 1.0))
892  channel_buffer[position] = resampled_samples[sample] * (1.0f / (1 << 15));
893 
894  // Increment audio position
895  position++;
896  }
897 
898  // increment channel (if needed)
899  if ((channel + 1) < info.channels)
900  // move to next channel
901  channel ++;
902  else
903  // reset channel
904  channel = 0;
905  }
906 
907  // Add samples to frame for this channel
908  frame->AddAudio(true, channel_filter, 0, channel_buffer, position, 1.0f);
909 
910  ZmqLogger::Instance()->AppendDebugMethod("FrameMapper::ResampleMappedAudio (Add audio to channel)", "number of samples", position, "channel_filter", channel_filter);
911  }
912 
913  // Update frame's audio meta data
914  frame->SampleRate(info.sample_rate);
915  frame->ChannelsLayout(info.channel_layout);
916 
917  // clear channel buffer
918  delete[] channel_buffer;
919  channel_buffer = NULL;
920 
921  // Delete arrays
922  delete[] resampled_samples;
923  resampled_samples = NULL;
924 }
#define SWR_INIT(ctx)
#define AV_FREE_FRAME(av_frame)
#define SWR_CONVERT(ctx, out, linesize, out_count, in, linesize2, in_count)
#define SWR_ALLOC()
#define SWR_CLOSE(ctx)
#define AV_ALLOCATE_FRAME()
#define SWR_FREE(ctx)
#define av_err2str(errnum)
#define AV_RESET_FRAME(av_frame)
#define OPEN_MP_NUM_PROCESSORS
void SetMaxBytesFromInfo(int64_t number_of_frames, int width, int height, int sample_rate, int channels)
Set maximum bytes to a different amount based on a ReaderInfo struct.
Definition: CacheBase.cpp:49
void Add(std::shared_ptr< openshot::Frame > frame)
Add a Frame to the cache.
std::shared_ptr< openshot::Frame > GetFrame(int64_t frame_number)
Get a frame from the cache.
void Clear()
Clear the cache of all frames.
Exception when encoding audio packet.
Definition: Exceptions.h:126
This class represents a fraction.
Definition: Fraction.h:45
int num
Numerator for the fraction.
Definition: Fraction.h:47
float ToFloat()
Return this fraction as a float (i.e. 1/2 = 0.5)
Definition: Fraction.cpp:44
double ToDouble()
Return this fraction as a double (i.e. 1/2 = 0.5)
Definition: Fraction.cpp:49
int ToInt()
Return a rounded integer of the fraction (for example 30000/1001 returns 30)
Definition: Fraction.cpp:54
int den
Denominator for the fraction.
Definition: Fraction.h:48
void ChangeMapping(Fraction target_fps, PulldownType pulldown, int target_sample_rate, int target_channels, ChannelLayout target_channel_layout)
Change frame rate or audio mapping details.
void SetJsonValue(const Json::Value root)
Load Json::Value into this object.
MappedFrame GetMappedFrame(int64_t TargetFrameNumber)
Get a frame based on the target frame rate and the new frame number of a frame.
std::vector< Field > fields
Definition: FrameMapper.h:166
ReaderBase * Reader()
Get the current reader.
Definition: FrameMapper.cpp:72
void Close()
Close the openshot::FrameMapper and internal reader.
void Open()
Open the internal reader.
std::vector< MappedFrame > frames
Definition: FrameMapper.h:167
void PrintMapping()
Print all of the original frames and which new frames they map to.
void ResampleMappedAudio(std::shared_ptr< Frame > frame, int64_t original_frame_number)
Resample audio and map channels (if needed)
std::shared_ptr< Frame > GetFrame(int64_t requested_frame)
This method is required for all derived classes of ReaderBase, and return the openshot::Frame object,...
std::string Json() const override
Get and Set JSON methods.
void SetJson(const std::string value)
Load JSON string into this object.
bool IsOpen()
Determine if reader is open or closed.
Json::Value JsonValue() const override
Generate Json::Value for this object.
virtual ~FrameMapper()
Destructor.
Definition: FrameMapper.cpp:63
int GetSamplesPerFrame(openshot::Fraction fps, int sample_rate, int channels)
Calculate the # of samples per video frame (for the current frame number)
Definition: Frame.cpp:547
Exception for invalid JSON.
Definition: Exceptions.h:206
Exception for frames that are out of bounds.
Definition: Exceptions.h:286
This abstract class is the base class, used by all readers in libopenshot.
Definition: ReaderBase.h:98
virtual bool IsOpen()=0
Determine if reader is open or closed.
juce::CriticalSection getFrameCriticalSection
Section lock for multiple threads.
Definition: ReaderBase.h:101
openshot::ReaderInfo info
Information about the current media file.
Definition: ReaderBase.h:111
virtual void SetJsonValue(const Json::Value root)=0
Load Json::Value into this object.
Definition: ReaderBase.cpp:171
virtual Json::Value JsonValue() const =0
Generate Json::Value for this object.
Definition: ReaderBase.cpp:116
virtual std::shared_ptr< openshot::Frame > GetFrame(int64_t number)=0
virtual void Open()=0
Open the reader (and start consuming resources, such as images or video files)
virtual void Close()=0
Close the reader (and any resources it was consuming)
Exception when a reader is closed, and a frame is requested.
Definition: Exceptions.h:338
Exception when too many seek attempts happen.
Definition: Exceptions.h:370
void AppendDebugMethod(std::string method_name, std::string arg1_name="", float arg1_value=-1.0, std::string arg2_name="", float arg2_value=-1.0, std::string arg3_name="", float arg3_value=-1.0, std::string arg4_name="", float arg4_value=-1.0, std::string arg5_name="", float arg5_value=-1.0, std::string arg6_name="", float arg6_value=-1.0)
Append debug information.
Definition: ZmqLogger.cpp:179
static ZmqLogger * Instance()
Create or get an instance of this logger singleton (invoke the class with this method)
Definition: ZmqLogger.cpp:45
This namespace is the default namespace for all code in the openshot library.
PulldownType
This enumeration determines how frame rates are increased or decreased.
Definition: FrameMapper.h:61
@ PULLDOWN_CLASSIC
Classic 2:3:2:3 pull-down.
Definition: FrameMapper.h:62
@ PULLDOWN_ADVANCED
Advanced 2:3:3:2 pull-down (minimal dirty frames)
Definition: FrameMapper.h:63
@ PULLDOWN_NONE
Do not apply pull-down techniques, just repeat or skip entire frames.
Definition: FrameMapper.h:64
ChannelLayout
This enumeration determines the audio channel layout (such as stereo, mono, 5 point surround,...
const Json::Value stringToJson(const std::string value)
Definition: Json.cpp:33
This struct holds a single field (half a frame).
Definition: FrameMapper.h:74
This struct holds two fields which together make up a complete video frame.
Definition: FrameMapper.h:111
bool has_single_image
Determines if this file only contains a single image.
Definition: ReaderBase.h:64
float duration
Length of time (in seconds)
Definition: ReaderBase.h:65
int width
The width of the video (in pixesl)
Definition: ReaderBase.h:68
int channels
The number of audio channels used in the audio stream.
Definition: ReaderBase.h:83
openshot::Fraction fps
Frames per second, as a fraction (i.e. 24/1 = 24 fps)
Definition: ReaderBase.h:70
int height
The height of the video (in pixels)
Definition: ReaderBase.h:67
int64_t video_length
The number of frames in the video stream.
Definition: ReaderBase.h:75
openshot::ChannelLayout channel_layout
The channel layout (mono, stereo, 5 point surround, etc...)
Definition: ReaderBase.h:84
bool has_video
Determines if this file has a video stream.
Definition: ReaderBase.h:62
bool has_audio
Determines if this file has an audio stream.
Definition: ReaderBase.h:63
openshot::Fraction video_timebase
The video timebase determines how long each frame stays on the screen.
Definition: ReaderBase.h:77
int sample_rate
The number of audio samples per second (44100 is a common sample rate)
Definition: ReaderBase.h:82
This struct holds a the range of samples needed by this frame.
Definition: FrameMapper.h:94