OpenShot Library | libopenshot  0.2.5
Frame.cpp
Go to the documentation of this file.
1 /**
2  * @file
3  * @brief Source file for Frame class
4  * @author Jonathan Thomas <jonathan@openshot.org>
5  *
6  * @ref License
7  */
8 
9 /* LICENSE
10  *
11  * Copyright (c) 2008-2019 OpenShot Studios, LLC
12  * <http://www.openshotstudios.com/>. This file is part of
13  * OpenShot Library (libopenshot), an open-source project dedicated to
14  * delivering high quality video editing and animation solutions to the
15  * world. For more information visit <http://www.openshot.org/>.
16  *
17  * OpenShot Library (libopenshot) is free software: you can redistribute it
18  * and/or modify it under the terms of the GNU Lesser General Public License
19  * as published by the Free Software Foundation, either version 3 of the
20  * License, or (at your option) any later version.
21  *
22  * OpenShot Library (libopenshot) is distributed in the hope that it will be
23  * useful, but WITHOUT ANY WARRANTY; without even the implied warranty of
24  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
25  * GNU Lesser General Public License for more details.
26  *
27  * You should have received a copy of the GNU Lesser General Public License
28  * along with OpenShot Library. If not, see <http://www.gnu.org/licenses/>.
29  */
30 
31 #include "../include/Frame.h"
32 
33 using namespace std;
34 using namespace openshot;
35 
36 // Constructor - blank frame (300x200 blank image, 48kHz audio silence)
37 Frame::Frame() : number(1), pixel_ratio(1,1), channels(2), width(1), height(1), color("#000000"),
38  channel_layout(LAYOUT_STEREO), sample_rate(44100), qbuffer(NULL), has_audio_data(false), has_image_data(false),
39  max_audio_sample(0)
40 {
41  // Init the image magic and audio buffer
42  audio = std::shared_ptr<juce::AudioSampleBuffer>(new juce::AudioSampleBuffer(channels, 0));
43 
44  // initialize the audio samples to zero (silence)
45  audio->clear();
46 }
47 
48 // Constructor - image only (48kHz audio silence)
49 Frame::Frame(int64_t number, int width, int height, std::string color)
50  : number(number), pixel_ratio(1,1), channels(2), width(width), height(height), color(color),
51  channel_layout(LAYOUT_STEREO), sample_rate(44100), qbuffer(NULL), has_audio_data(false), has_image_data(false),
52  max_audio_sample(0)
53 {
54  // Init the image magic and audio buffer
55  audio = std::shared_ptr<juce::AudioSampleBuffer>(new juce::AudioSampleBuffer(channels, 0));
56 
57  // initialize the audio samples to zero (silence)
58  audio->clear();
59 }
60 
61 // Constructor - audio only (300x200 blank image)
62 Frame::Frame(int64_t number, int samples, int channels) :
63  number(number), pixel_ratio(1,1), channels(channels), width(1), height(1), color("#000000"),
64  channel_layout(LAYOUT_STEREO), sample_rate(44100), qbuffer(NULL), has_audio_data(false), has_image_data(false),
65  max_audio_sample(0)
66 {
67  // Init the image magic and audio buffer
68  audio = std::shared_ptr<juce::AudioSampleBuffer>(new juce::AudioSampleBuffer(channels, samples));
69 
70  // initialize the audio samples to zero (silence)
71  audio->clear();
72 }
73 
74 // Constructor - image & audio
75 Frame::Frame(int64_t number, int width, int height, std::string color, int samples, int channels)
76  : number(number), pixel_ratio(1,1), channels(channels), width(width), height(height), color(color),
77  channel_layout(LAYOUT_STEREO), sample_rate(44100), qbuffer(NULL), has_audio_data(false), has_image_data(false),
78  max_audio_sample(0)
79 {
80  // Init the image magic and audio buffer
81  audio = std::shared_ptr<juce::AudioSampleBuffer>(new juce::AudioSampleBuffer(channels, samples));
82 
83  // initialize the audio samples to zero (silence)
84  audio->clear();
85 }
86 
87 
88 // Copy constructor
89 Frame::Frame ( const Frame &other )
90 {
91  // copy pointers and data
92  DeepCopy(other);
93 }
94 
95 // Assignment operator
97 {
98  // copy pointers and data
99  DeepCopy(other);
100 
101  return *this;
102 }
103 
104 // Copy data and pointers from another Frame instance
105 void Frame::DeepCopy(const Frame& other)
106 {
107  number = other.number;
108  channels = other.channels;
109  width = other.width;
110  height = other.height;
111  channel_layout = other.channel_layout;
114  sample_rate = other.sample_rate;
115  pixel_ratio = Fraction(other.pixel_ratio.num, other.pixel_ratio.den);
116  color = other.color;
117  max_audio_sample = other.max_audio_sample;
118 
119  if (other.image)
120  image = std::shared_ptr<QImage>(new QImage(*(other.image)));
121  if (other.audio)
122  audio = std::shared_ptr<juce::AudioSampleBuffer>(new juce::AudioSampleBuffer(*(other.audio)));
123  if (other.wave_image)
124  wave_image = std::shared_ptr<QImage>(new QImage(*(other.wave_image)));
125 }
126 
127 // Destructor
129  // Clear all pointers
130  image.reset();
131  audio.reset();
132 }
133 
134 // Display the frame image to the screen (primarily used for debugging reasons)
136 {
137  if (!QApplication::instance()) {
138  // Only create the QApplication once
139  static int argc = 1;
140  static char* argv[1] = {NULL};
141  previewApp = std::shared_ptr<QApplication>(new QApplication(argc, argv));
142  }
143 
144  // Get preview image
145  std::shared_ptr<QImage> previewImage = GetImage();
146 
147  // Update the image to reflect the correct pixel aspect ration (i.e. to fix non-squar pixels)
148  if (pixel_ratio.num != 1 || pixel_ratio.den != 1)
149  {
150  // Calculate correct DAR (display aspect ratio)
151  int new_width = previewImage->size().width();
152  int new_height = previewImage->size().height() * pixel_ratio.Reciprocal().ToDouble();
153 
154  // Resize to fix DAR
155  previewImage = std::shared_ptr<QImage>(new QImage(previewImage->scaled(new_width, new_height, Qt::IgnoreAspectRatio, Qt::SmoothTransformation)));
156  }
157 
158  // Create window
159  QWidget previewWindow;
160  previewWindow.setStyleSheet("background-color: #000000;");
161  QHBoxLayout layout;
162 
163  // Create label with current frame's image
164  QLabel previewLabel;
165  previewLabel.setPixmap(QPixmap::fromImage(*previewImage));
166  previewLabel.setMask(QPixmap::fromImage(*previewImage).mask());
167  layout.addWidget(&previewLabel);
168 
169  // Show the window
170  previewWindow.setLayout(&layout);
171  previewWindow.show();
172  previewApp->exec();
173 }
174 
175 // Get an audio waveform image
176 std::shared_ptr<QImage> Frame::GetWaveform(int width, int height, int Red, int Green, int Blue, int Alpha)
177 {
178  // Clear any existing waveform image
179  ClearWaveform();
180 
181  // Init a list of lines
182  QVector<QPointF> lines;
183  QVector<QPointF> labels;
184 
185  // Calculate width of an image based on the # of samples
186  int total_samples = GetAudioSamplesCount();
187  if (total_samples > 0)
188  {
189  // If samples are present...
190  int new_height = 200 * audio->getNumChannels();
191  int height_padding = 20 * (audio->getNumChannels() - 1);
192  int total_height = new_height + height_padding;
193  int total_width = 0;
194 
195  // Loop through each audio channel
196  int Y = 100;
197  for (int channel = 0; channel < audio->getNumChannels(); channel++)
198  {
199  int X = 0;
200 
201  // Get audio for this channel
202  const float *samples = audio->getReadPointer(channel);
203 
204  for (int sample = 0; sample < GetAudioSamplesCount(); sample++, X++)
205  {
206  // Sample value (scaled to -100 to 100)
207  float value = samples[sample] * 100;
208 
209  // Append a line segment for each sample
210  if (value != 0.0) {
211  // LINE
212  lines.push_back(QPointF(X,Y));
213  lines.push_back(QPointF(X,Y-value));
214  }
215  else {
216  // DOT
217  lines.push_back(QPointF(X,Y));
218  lines.push_back(QPointF(X,Y));
219  }
220  }
221 
222  // Add Channel Label Coordinate
223  labels.push_back(QPointF(5, Y - 5));
224 
225  // Increment Y
226  Y += (200 + height_padding);
227  total_width = X;
228  }
229 
230  // Create blank image
231  wave_image = std::shared_ptr<QImage>(new QImage(total_width, total_height, QImage::Format_RGBA8888));
232  wave_image->fill(QColor(0,0,0,0));
233 
234  // Load QPainter with wave_image device
235  QPainter painter(wave_image.get());
236 
237  // Set pen color
238  painter.setPen(QColor(Red, Green, Blue, Alpha));
239 
240  // Draw the waveform
241  painter.drawLines(lines);
242  painter.end();
243 
244  // Loop through the channels labels (and draw the text)
245  // TODO: Configure Fonts in Qt5 correctly, so the drawText method does not crash
246 // painter.setFont(QFont(QString("Arial"), 16, 1, false));
247 // for (int channel = 0; channel < labels.size(); channel++) {
248 // stringstream label;
249 // label << "Channel " << channel;
250 // painter.drawText(labels.at(channel), QString::fromStdString(label.str()));
251 // }
252 
253  // Resize Image (if requested)
254  if (width != total_width || height != total_height) {
255  QImage scaled_wave_image = wave_image->scaled(width, height, Qt::IgnoreAspectRatio, Qt::FastTransformation);
256  wave_image = std::shared_ptr<QImage>(new QImage(scaled_wave_image));
257  }
258  }
259  else
260  {
261  // No audio samples present
262  wave_image = std::shared_ptr<QImage>(new QImage(width, height, QImage::Format_RGBA8888));
263  wave_image->fill(QColor(QString::fromStdString("#000000")));
264  }
265 
266  // Return new image
267  return wave_image;
268 }
269 
270 // Clear the waveform image (and deallocate its memory)
272 {
273  if (wave_image)
274  wave_image.reset();
275 }
276 
277 // Get an audio waveform image pixels
278 const unsigned char* Frame::GetWaveformPixels(int width, int height, int Red, int Green, int Blue, int Alpha)
279 {
280  // Get audio wave form image
281  wave_image = GetWaveform(width, height, Red, Green, Blue, Alpha);
282 
283  // Return array of pixel packets
284  return wave_image->constBits();
285 }
286 
287 // Display the wave form
289 {
290  // Get audio wave form image
291  GetWaveform(720, 480, 0, 123, 255, 255);
292 
293  if (!QApplication::instance()) {
294  // Only create the QApplication once
295  static int argc = 1;
296  static char* argv[1] = {NULL};
297  previewApp = std::shared_ptr<QApplication>(new QApplication(argc, argv));
298  }
299 
300  // Create window
301  QWidget previewWindow;
302  previewWindow.setStyleSheet("background-color: #000000;");
303  QHBoxLayout layout;
304 
305  // Create label with current frame's waveform image
306  QLabel previewLabel;
307  previewLabel.setPixmap(QPixmap::fromImage(*wave_image));
308  previewLabel.setMask(QPixmap::fromImage(*wave_image).mask());
309  layout.addWidget(&previewLabel);
310 
311  // Show the window
312  previewWindow.setLayout(&layout);
313  previewWindow.show();
314  previewApp->exec();
315 
316  // Deallocate waveform image
317  ClearWaveform();
318 }
319 
320 // Get magnitude of range of samples (if channel is -1, return average of all channels for that sample)
321 float Frame::GetAudioSample(int channel, int sample, int magnitude_range)
322 {
323  if (channel > 0) {
324  // return average magnitude for a specific channel/sample range
325  return audio->getMagnitude(channel, sample, magnitude_range);
326 
327  } else {
328  // Return average magnitude for all channels
329  return audio->getMagnitude(sample, magnitude_range);
330  }
331 }
332 
333 // Get an array of sample data
334 float* Frame::GetAudioSamples(int channel)
335 {
336  // return JUCE audio data for this channel
337  return audio->getWritePointer(channel);
338 }
339 
340 // Get a planar array of sample data, using any sample rate
341 float* Frame::GetPlanarAudioSamples(int new_sample_rate, AudioResampler* resampler, int* sample_count)
342 {
343  float *output = NULL;
344  juce::AudioSampleBuffer *buffer(audio.get());
345  int num_of_channels = audio->getNumChannels();
346  int num_of_samples = GetAudioSamplesCount();
347 
348  // Resample to new sample rate (if needed)
349  if (new_sample_rate != sample_rate)
350  {
351  // YES, RESAMPLE AUDIO
352  resampler->SetBuffer(audio.get(), sample_rate, new_sample_rate);
353 
354  // Resample data, and return new buffer pointer
355  buffer = resampler->GetResampledBuffer();
356 
357  // Update num_of_samples
358  num_of_samples = buffer->getNumSamples();
359  }
360 
361  // INTERLEAVE all samples together (channel 1 + channel 2 + channel 1 + channel 2, etc...)
362  output = new float[num_of_channels * num_of_samples];
363  int position = 0;
364 
365  // Loop through samples in each channel (combining them)
366  for (int channel = 0; channel < num_of_channels; channel++)
367  {
368  for (int sample = 0; sample < num_of_samples; sample++)
369  {
370  // Add sample to output array
371  output[position] = buffer->getReadPointer(channel)[sample];
372 
373  // increment position
374  position++;
375  }
376  }
377 
378  // Update sample count (since it might have changed due to resampling)
379  *sample_count = num_of_samples;
380 
381  // return combined array
382  return output;
383 }
384 
385 
386 // Get an array of sample data (all channels interleaved together), using any sample rate
387 float* Frame::GetInterleavedAudioSamples(int new_sample_rate, AudioResampler* resampler, int* sample_count)
388 {
389  float *output = NULL;
390  juce::AudioSampleBuffer *buffer(audio.get());
391  int num_of_channels = audio->getNumChannels();
392  int num_of_samples = GetAudioSamplesCount();
393 
394  // Resample to new sample rate (if needed)
395  if (new_sample_rate != sample_rate && resampler)
396  {
397  // YES, RESAMPLE AUDIO
398  resampler->SetBuffer(audio.get(), sample_rate, new_sample_rate);
399 
400  // Resample data, and return new buffer pointer
401  buffer = resampler->GetResampledBuffer();
402 
403  // Update num_of_samples
404  num_of_samples = buffer->getNumSamples();
405  }
406 
407  // INTERLEAVE all samples together (channel 1 + channel 2 + channel 1 + channel 2, etc...)
408  output = new float[num_of_channels * num_of_samples];
409  int position = 0;
410 
411  // Loop through samples in each channel (combining them)
412  for (int sample = 0; sample < num_of_samples; sample++)
413  {
414  for (int channel = 0; channel < num_of_channels; channel++)
415  {
416  // Add sample to output array
417  output[position] = buffer->getReadPointer(channel)[sample];
418 
419  // increment position
420  position++;
421  }
422  }
423 
424  // Update sample count (since it might have changed due to resampling)
425  *sample_count = num_of_samples;
426 
427  // return combined array
428  return output;
429 }
430 
431 // Get number of audio channels
433 {
434  const GenericScopedLock<juce::CriticalSection> lock(addingAudioSection);
435  if (audio)
436  return audio->getNumChannels();
437  else
438  return 0;
439 }
440 
441 // Get number of audio samples
443 {
444  const GenericScopedLock<juce::CriticalSection> lock(addingAudioSection);
445  return max_audio_sample;
446 }
447 
448 juce::AudioSampleBuffer *Frame::GetAudioSampleBuffer()
449 {
450  return audio.get();
451 }
452 
453 // Get the size in bytes of this frame (rough estimate)
455 {
456  int64_t total_bytes = 0;
457  if (image)
458  total_bytes += (width * height * sizeof(char) * 4);
459  if (audio) {
460  // approximate audio size (sample rate / 24 fps)
461  total_bytes += (sample_rate / 24.0) * sizeof(float);
462  }
463 
464  // return size of this frame
465  return total_bytes;
466 }
467 
468 // Get pixel data (as packets)
469 const unsigned char* Frame::GetPixels()
470 {
471  // Check for blank image
472  if (!image)
473  // Fill with black
474  AddColor(width, height, color);
475 
476  // Return array of pixel packets
477  return image->constBits();
478 }
479 
480 // Get pixel data (for only a single scan-line)
481 const unsigned char* Frame::GetPixels(int row)
482 {
483  // Return array of pixel packets
484  return image->constScanLine(row);
485 }
486 
487 // Check a specific pixel color value (returns True/False)
488 bool Frame::CheckPixel(int row, int col, int red, int green, int blue, int alpha, int threshold) {
489  int col_pos = col * 4; // Find column array position
490  if (!image || row < 0 || row >= (height - 1) ||
491  col_pos < 0 || col_pos >= (width - 1) ) {
492  // invalid row / col
493  return false;
494  }
495  // Check pixel color
496  const unsigned char* pixels = GetPixels(row);
497  if (pixels[col_pos + 0] >= (red - threshold) && pixels[col_pos + 0] <= (red + threshold) &&
498  pixels[col_pos + 1] >= (green - threshold) && pixels[col_pos + 1] <= (green + threshold) &&
499  pixels[col_pos + 2] >= (blue - threshold) && pixels[col_pos + 2] <= (blue + threshold) &&
500  pixels[col_pos + 3] >= (alpha - threshold) && pixels[col_pos + 3] <= (alpha + threshold)) {
501  // Pixel color matches successfully
502  return true;
503  } else {
504  // Pixel color does not match
505  return false;
506  }
507 }
508 
509 // Set Pixel Aspect Ratio
510 void Frame::SetPixelRatio(int num, int den)
511 {
512  pixel_ratio.num = num;
513  pixel_ratio.den = den;
514 }
515 
516 // Set frame number
517 void Frame::SetFrameNumber(int64_t new_number)
518 {
519  number = new_number;
520 }
521 
522 // Calculate the # of samples per video frame (for a specific frame number and frame rate)
523 int Frame::GetSamplesPerFrame(int64_t number, Fraction fps, int sample_rate, int channels)
524 {
525  // Get the total # of samples for the previous frame, and the current frame (rounded)
526  double fps_rate = fps.Reciprocal().ToDouble();
527 
528  // Determine previous samples total, and make sure it's evenly divisible by the # of channels
529  double previous_samples = (sample_rate * fps_rate) * (number - 1);
530  double previous_samples_remainder = fmod(previous_samples, (double)channels); // subtract the remainder to the total (to make it evenly divisible)
531  previous_samples -= previous_samples_remainder;
532 
533  // Determine the current samples total, and make sure it's evenly divisible by the # of channels
534  double total_samples = (sample_rate * fps_rate) * number;
535  double total_samples_remainder = fmod(total_samples, (double)channels); // subtract the remainder to the total (to make it evenly divisible)
536  total_samples -= total_samples_remainder;
537 
538  // Subtract the previous frame's total samples with this frame's total samples. Not all sample rates can
539  // be evenly divided into frames, so each frame can have have different # of samples.
540  int samples_per_frame = round(total_samples - previous_samples);
541  if (samples_per_frame < 0)
542  samples_per_frame = 0;
543  return samples_per_frame;
544 }
545 
546 // Calculate the # of samples per video frame (for the current frame number)
547 int Frame::GetSamplesPerFrame(Fraction fps, int sample_rate, int channels)
548 {
549  return GetSamplesPerFrame(number, fps, sample_rate, channels);
550 }
551 
552 // Get height of image
554 {
555  return height;
556 }
557 
558 // Get height of image
560 {
561  return width;
562 }
563 
564 // Get the original sample rate of this frame's audio data
566 {
567  return sample_rate;
568 }
569 
570 // Get the original sample rate of this frame's audio data
572 {
573  return channel_layout;
574 }
575 
576 
577 // Save the frame image to the specified path. The image format is determined from the extension (i.e. image.PNG, image.JPEG)
578 void Frame::Save(std::string path, float scale, std::string format, int quality)
579 {
580  // Get preview image
581  std::shared_ptr<QImage> previewImage = GetImage();
582 
583  // scale image if needed
584  if (fabs(scale) > 1.001 || fabs(scale) < 0.999)
585  {
586  int new_width = width;
587  int new_height = height;
588 
589  // Update the image to reflect the correct pixel aspect ration (i.e. to fix non-squar pixels)
590  if (pixel_ratio.num != 1 || pixel_ratio.den != 1)
591  {
592  // Calculate correct DAR (display aspect ratio)
593  int new_width = previewImage->size().width();
594  int new_height = previewImage->size().height() * pixel_ratio.Reciprocal().ToDouble();
595 
596  // Resize to fix DAR
597  previewImage = std::shared_ptr<QImage>(new QImage(previewImage->scaled(new_width, new_height, Qt::IgnoreAspectRatio, Qt::SmoothTransformation)));
598  }
599 
600  // Resize image
601  previewImage = std::shared_ptr<QImage>(new QImage(previewImage->scaled(new_width * scale, new_height * scale, Qt::KeepAspectRatio, Qt::SmoothTransformation)));
602  }
603 
604  // Save image
605  previewImage->save(QString::fromStdString(path), format.c_str(), quality);
606 }
607 
608 // Thumbnail the frame image to the specified path. The image format is determined from the extension (i.e. image.PNG, image.JPEG)
609 void Frame::Thumbnail(std::string path, int new_width, int new_height, std::string mask_path, std::string overlay_path,
610  std::string background_color, bool ignore_aspect, std::string format, int quality, float rotate) {
611 
612  // Create blank thumbnail image & fill background color
613  std::shared_ptr<QImage> thumbnail = std::shared_ptr<QImage>(new QImage(new_width, new_height, QImage::Format_RGBA8888));
614  thumbnail->fill(QColor(QString::fromStdString(background_color)));
615 
616  // Create painter
617  QPainter painter(thumbnail.get());
618  painter.setRenderHints(QPainter::Antialiasing | QPainter::SmoothPixmapTransform | QPainter::TextAntialiasing, true);
619 
620  // Get preview image
621  std::shared_ptr<QImage> previewImage = GetImage();
622 
623  // Update the image to reflect the correct pixel aspect ration (i.e. to fix non-squar pixels)
624  if (pixel_ratio.num != 1 || pixel_ratio.den != 1)
625  {
626  // Calculate correct DAR (display aspect ratio)
627  int aspect_width = previewImage->size().width();
628  int aspect_height = previewImage->size().height() * pixel_ratio.Reciprocal().ToDouble();
629 
630  // Resize to fix DAR
631  previewImage = std::shared_ptr<QImage>(new QImage(previewImage->scaled(aspect_width, aspect_height, Qt::IgnoreAspectRatio, Qt::SmoothTransformation)));
632  }
633 
634  // Resize frame image
635  if (ignore_aspect)
636  // Ignore aspect ratio
637  previewImage = std::shared_ptr<QImage>(new QImage(previewImage->scaled(new_width, new_height, Qt::IgnoreAspectRatio, Qt::SmoothTransformation)));
638  else
639  // Maintain aspect ratio
640  previewImage = std::shared_ptr<QImage>(new QImage(previewImage->scaled(new_width, new_height, Qt::KeepAspectRatio, Qt::SmoothTransformation)));
641 
642  // Composite frame image onto background (centered)
643  int x = (new_width - previewImage->size().width()) / 2.0; // center
644  int y = (new_height - previewImage->size().height()) / 2.0; // center
645  painter.setCompositionMode(QPainter::CompositionMode_SourceOver);
646 
647 
648  // Create transform and rotate (if needed)
649  QTransform transform;
650  float origin_x = previewImage->width() / 2.0;
651  float origin_y = previewImage->height() / 2.0;
652  transform.translate(origin_x, origin_y);
653  transform.rotate(rotate);
654  transform.translate(-origin_x,-origin_y);
655  painter.setTransform(transform);
656 
657  // Draw image onto QImage
658  painter.drawImage(x, y, *previewImage);
659 
660 
661  // Overlay Image (if any)
662  if (overlay_path != "") {
663  // Open overlay
664  std::shared_ptr<QImage> overlay = std::shared_ptr<QImage>(new QImage());
665  overlay->load(QString::fromStdString(overlay_path));
666 
667  // Set pixel format
668  overlay = std::shared_ptr<QImage>(new QImage(overlay->convertToFormat(QImage::Format_RGBA8888)));
669 
670  // Resize to fit
671  overlay = std::shared_ptr<QImage>(new QImage(overlay->scaled(new_width, new_height, Qt::IgnoreAspectRatio, Qt::SmoothTransformation)));
672 
673  // Composite onto thumbnail
674  painter.setCompositionMode(QPainter::CompositionMode_SourceOver);
675  painter.drawImage(0, 0, *overlay);
676  }
677 
678 
679  // Mask Image (if any)
680  if (mask_path != "") {
681  // Open mask
682  std::shared_ptr<QImage> mask = std::shared_ptr<QImage>(new QImage());
683  mask->load(QString::fromStdString(mask_path));
684 
685  // Set pixel format
686  mask = std::shared_ptr<QImage>(new QImage(mask->convertToFormat(QImage::Format_RGBA8888)));
687 
688  // Resize to fit
689  mask = std::shared_ptr<QImage>(new QImage(mask->scaled(new_width, new_height, Qt::IgnoreAspectRatio, Qt::SmoothTransformation)));
690 
691  // Negate mask
692  mask->invertPixels();
693 
694  // Get pixels
695  unsigned char *pixels = (unsigned char *) thumbnail->bits();
696  const unsigned char *mask_pixels = (const unsigned char *) mask->constBits();
697 
698  // Convert the mask image to grayscale
699  // Loop through pixels
700  for (int pixel = 0, byte_index=0; pixel < new_width * new_height; pixel++, byte_index+=4)
701  {
702  // Get the RGB values from the pixel
703  int gray_value = qGray(mask_pixels[byte_index], mask_pixels[byte_index] + 1, mask_pixels[byte_index] + 2);
704  int Frame_Alpha = pixels[byte_index + 3];
705  int Mask_Value = constrain(Frame_Alpha - gray_value);
706 
707  // Set all alpha pixels to gray value
708  pixels[byte_index + 3] = Mask_Value;
709  }
710  }
711 
712 
713  // End painter
714  painter.end();
715 
716  // Save image
717  thumbnail->save(QString::fromStdString(path), format.c_str(), quality);
718 }
719 
720 // Constrain a color value from 0 to 255
721 int Frame::constrain(int color_value)
722 {
723  // Constrain new color from 0 to 255
724  if (color_value < 0)
725  color_value = 0;
726  else if (color_value > 255)
727  color_value = 255;
728 
729  return color_value;
730 }
731 
732 // Add (or replace) pixel data to the frame (based on a solid color)
733 void Frame::AddColor(int new_width, int new_height, std::string new_color)
734 {
735  // Set color
736  color = new_color;
737 
738  // Create new image object, and fill with pixel data
739  const GenericScopedLock<juce::CriticalSection> lock(addingImageSection);
740  #pragma omp critical (AddImage)
741  {
742  image = std::shared_ptr<QImage>(new QImage(new_width, new_height, QImage::Format_RGBA8888));
743 
744  // Fill with solid color
745  image->fill(QColor(QString::fromStdString(color)));
746  }
747  // Update height and width
748  width = image->width();
749  height = image->height();
750  has_image_data = true;
751 }
752 
753 // Add (or replace) pixel data to the frame
754 void Frame::AddImage(int new_width, int new_height, int bytes_per_pixel, QImage::Format type, const unsigned char *pixels_)
755 {
756  // Create new buffer
757  const GenericScopedLock<juce::CriticalSection> lock(addingImageSection);
758  int buffer_size = new_width * new_height * bytes_per_pixel;
759  qbuffer = new unsigned char[buffer_size]();
760 
761  // Copy buffer data
762  memcpy((unsigned char*)qbuffer, pixels_, buffer_size);
763 
764  // Create new image object, and fill with pixel data
765  #pragma omp critical (AddImage)
766  {
767  image = std::shared_ptr<QImage>(new QImage(qbuffer, new_width, new_height, new_width * bytes_per_pixel, type, (QImageCleanupFunction) &openshot::Frame::cleanUpBuffer, (void*) qbuffer));
768 
769  // Always convert to RGBA8888 (if different)
770  if (image->format() != QImage::Format_RGBA8888)
771  *image = image->convertToFormat(QImage::Format_RGBA8888);
772 
773  // Update height and width
774  width = image->width();
775  height = image->height();
776  has_image_data = true;
777  }
778 }
779 
780 // Add (or replace) pixel data to the frame
781 void Frame::AddImage(std::shared_ptr<QImage> new_image)
782 {
783  // Ignore blank images
784  if (!new_image)
785  return;
786 
787  // assign image data
788  const GenericScopedLock<juce::CriticalSection> lock(addingImageSection);
789  #pragma omp critical (AddImage)
790  {
791  image = new_image;
792 
793  // Always convert to RGBA8888 (if different)
794  if (image->format() != QImage::Format_RGBA8888)
795  *image = image->convertToFormat(QImage::Format_RGBA8888);
796 
797  // Update height and width
798  width = image->width();
799  height = image->height();
800  has_image_data = true;
801  }
802 }
803 
804 // Add (or replace) pixel data to the frame (for only the odd or even lines)
805 void Frame::AddImage(std::shared_ptr<QImage> new_image, bool only_odd_lines)
806 {
807  // Ignore blank new_image
808  if (!new_image)
809  return;
810 
811  // Check for blank source image
812  if (!image) {
813  // Replace the blank source image
814  AddImage(new_image);
815 
816  } else {
817 
818  // Ignore image of different sizes or formats
819  bool ret=false;
820  #pragma omp critical (AddImage)
821  {
822  if (image == new_image || image->size() != new_image->size()) {
823  ret = true;
824  }
825  else if (new_image->format() != image->format()) {
826  new_image = std::shared_ptr<QImage>(new QImage(new_image->convertToFormat(image->format())));
827  }
828  }
829  if (ret) {
830  return;
831  }
832 
833  // Get the frame's image
834  const GenericScopedLock<juce::CriticalSection> lock(addingImageSection);
835  #pragma omp critical (AddImage)
836  {
837  unsigned char *pixels = image->bits();
838  const unsigned char *new_pixels = new_image->constBits();
839 
840  // Loop through the scanlines of the image (even or odd)
841  int start = 0;
842  if (only_odd_lines)
843  start = 1;
844 
845  for (int row = start; row < image->height(); row += 2) {
846  int offset = row * image->bytesPerLine();
847  memcpy(pixels + offset, new_pixels + offset, image->bytesPerLine());
848  }
849 
850  // Update height and width
851  height = image->height();
852  width = image->width();
853  has_image_data = true;
854  }
855  }
856 }
857 
858 
859 // Resize audio container to hold more (or less) samples and channels
860 void Frame::ResizeAudio(int channels, int length, int rate, ChannelLayout layout)
861 {
862  const GenericScopedLock<juce::CriticalSection> lock(addingAudioSection);
863 
864  // Resize JUCE audio buffer
865  audio->setSize(channels, length, true, true, false);
866  channel_layout = layout;
867  sample_rate = rate;
868 
869  // Calculate max audio sample added
870  max_audio_sample = length;
871 }
872 
873 // Add audio samples to a specific channel
874 void Frame::AddAudio(bool replaceSamples, int destChannel, int destStartSample, const float* source, int numSamples, float gainToApplyToSource = 1.0f) {
875  const GenericScopedLock<juce::CriticalSection> lock(addingAudioSection);
876  #pragma omp critical (adding_audio)
877  {
878  // Clamp starting sample to 0
879  int destStartSampleAdjusted = max(destStartSample, 0);
880 
881  // Extend audio container to hold more (or less) samples and channels.. if needed
882  int new_length = destStartSampleAdjusted + numSamples;
883  int new_channel_length = audio->getNumChannels();
884  if (destChannel >= new_channel_length)
885  new_channel_length = destChannel + 1;
886  if (new_length > audio->getNumSamples() || new_channel_length > audio->getNumChannels())
887  audio->setSize(new_channel_length, new_length, true, true, false);
888 
889  // Clear the range of samples first (if needed)
890  if (replaceSamples)
891  audio->clear(destChannel, destStartSampleAdjusted, numSamples);
892 
893  // Add samples to frame's audio buffer
894  audio->addFrom(destChannel, destStartSampleAdjusted, source, numSamples, gainToApplyToSource);
895  has_audio_data = true;
896 
897  // Calculate max audio sample added
898  if (new_length > max_audio_sample)
899  max_audio_sample = new_length;
900  }
901 }
902 
903 // Apply gain ramp (i.e. fading volume)
904 void Frame::ApplyGainRamp(int destChannel, int destStartSample, int numSamples, float initial_gain = 0.0f, float final_gain = 1.0f)
905 {
906  const GenericScopedLock<juce::CriticalSection> lock(addingAudioSection);
907 
908  // Apply gain ramp
909  audio->applyGainRamp(destChannel, destStartSample, numSamples, initial_gain, final_gain);
910 }
911 
912 // Get pointer to Magick++ image object
913 std::shared_ptr<QImage> Frame::GetImage()
914 {
915  // Check for blank image
916  if (!image)
917  // Fill with black
918  AddColor(width, height, color);
919 
920  return image;
921 }
922 
923 #ifdef USE_IMAGEMAGICK
924 // Get pointer to ImageMagick image object
925 std::shared_ptr<Magick::Image> Frame::GetMagickImage()
926 {
927  // Check for blank image
928  if (!image)
929  // Fill with black
930  AddColor(width, height, "#000000");
931 
932  // Get the pixels from the frame image
933  const QRgb *tmpBits = (const QRgb*)image->constBits();
934 
935  // Create new image object, and fill with pixel data
936  std::shared_ptr<Magick::Image> magick_image = std::shared_ptr<Magick::Image>(new Magick::Image(image->width(), image->height(),"RGBA", Magick::CharPixel, tmpBits));
937 
938  // Give image a transparent background color
939  magick_image->backgroundColor(Magick::Color("none"));
940  magick_image->virtualPixelMethod(Magick::TransparentVirtualPixelMethod);
941  MAGICK_IMAGE_ALPHA(magick_image, true);
942 
943  return magick_image;
944 }
945 #endif
946 
947 #ifdef USE_IMAGEMAGICK
948 // Get pointer to QImage of frame
949 void Frame::AddMagickImage(std::shared_ptr<Magick::Image> new_image)
950 {
951  const int BPP = 4;
952  const std::size_t bufferSize = new_image->columns() * new_image->rows() * BPP;
953 
954  /// Use realloc for fast memory allocation.
955  /// TODO: consider locking the buffer for mt safety
956  //qbuffer = reinterpret_cast<unsigned char*>(realloc(qbuffer, bufferSize));
957  qbuffer = new unsigned char[bufferSize]();
958  unsigned char *buffer = (unsigned char*)qbuffer;
959 
960  MagickCore::ExceptionInfo exception;
961  // TODO: Actually do something, if we get an exception here
962  MagickCore::ExportImagePixels(new_image->constImage(), 0, 0, new_image->columns(), new_image->rows(), "RGBA", Magick::CharPixel, buffer, &exception);
963 
964  // Create QImage of frame data
965  image = std::shared_ptr<QImage>(new QImage(qbuffer, width, height, width * BPP, QImage::Format_RGBA8888, (QImageCleanupFunction) &cleanUpBuffer, (void*) qbuffer));
966 
967  // Update height and width
968  width = image->width();
969  height = image->height();
970  has_image_data = true;
971 }
972 #endif
973 
974 // Play audio samples for this frame
976 {
977  // Check if samples are present
978  if (!GetAudioSamplesCount())
979  return;
980 
981  juce::AudioDeviceManager deviceManager;
982  String error = deviceManager.initialise (0, /* number of input channels */
983  2, /* number of output channels */
984  0, /* no XML settings.. */
985  true /* select default device on failure */);
986 
987  // Output error (if any)
988  if (error.isNotEmpty()) {
989  cout << "Error on initialise(): " << error.toStdString() << endl;
990  }
991 
992  juce::AudioSourcePlayer audioSourcePlayer;
993  deviceManager.addAudioCallback (&audioSourcePlayer);
994 
995  ScopedPointer<AudioBufferSource> my_source;
996  my_source = new AudioBufferSource(audio.get());
997 
998  // Create TimeSliceThread for audio buffering
999  juce::TimeSliceThread my_thread("Audio buffer thread");
1000 
1001  // Start thread
1002  my_thread.startThread();
1003 
1004  AudioTransportSource transport1;
1005  transport1.setSource (my_source,
1006  5000, // tells it to buffer this many samples ahead
1007  &my_thread,
1008  (double) sample_rate,
1009  audio->getNumChannels()); // sample rate of source
1010  transport1.setPosition (0);
1011  transport1.setGain(1.0);
1012 
1013 
1014  // Create MIXER
1015  juce::MixerAudioSource mixer;
1016  mixer.addInputSource(&transport1, false);
1017  audioSourcePlayer.setSource (&mixer);
1018 
1019  // Start transports
1020  transport1.start();
1021 
1022  while (transport1.isPlaying())
1023  {
1024  cout << "playing" << endl;
1025  usleep(1000000);
1026  }
1027 
1028  cout << "DONE!!!" << endl;
1029 
1030  transport1.stop();
1031  transport1.setSource (0);
1032  audioSourcePlayer.setSource (0);
1033  my_thread.stopThread(500);
1034  deviceManager.removeAudioCallback (&audioSourcePlayer);
1035  deviceManager.closeAudioDevice();
1036  deviceManager.removeAllChangeListeners();
1037  deviceManager.dispatchPendingMessages();
1038 
1039  cout << "End of Play()" << endl;
1040 
1041 
1042 }
1043 
1044 // Clean up buffer after QImage is deleted
1045 void Frame::cleanUpBuffer(void *info)
1046 {
1047  if (info)
1048  {
1049  // Remove buffer since QImage tells us to
1050  unsigned char* ptr_to_qbuffer = (unsigned char*) info;
1051  delete[] ptr_to_qbuffer;
1052  }
1053 }
1054 
1055 // Add audio silence
1056 void Frame::AddAudioSilence(int numSamples)
1057 {
1058  const GenericScopedLock<juce::CriticalSection> lock(addingAudioSection);
1059 
1060  // Resize audio container
1061  audio->setSize(channels, numSamples, false, true, false);
1062  audio->clear();
1063  has_audio_data = true;
1064 
1065  // Calculate max audio sample added
1066  if (numSamples > max_audio_sample)
1067  max_audio_sample = numSamples;
1068 }
#define MAGICK_IMAGE_ALPHA(im, a)
This class is used to expose an AudioSampleBuffer as an AudioSource in JUCE.
This class is used to resample audio data for many sequential frames.
juce::AudioSampleBuffer * GetResampledBuffer()
Get the resampled audio buffer.
void SetBuffer(juce::AudioSampleBuffer *new_buffer, double sample_rate, double new_sample_rate)
Sets the audio buffer and key settings.
This class represents a fraction.
Definition: Fraction.h:45
int num
Numerator for the fraction.
Definition: Fraction.h:47
Fraction Reciprocal()
Return the reciprocal as a Fraction.
Definition: Fraction.cpp:84
double ToDouble()
Return this fraction as a double (i.e. 1/2 = 0.5)
Definition: Fraction.cpp:49
int den
Denominator for the fraction.
Definition: Fraction.h:48
This class represents a single frame of video (i.e. image & audio data)
Definition: Frame.h:107
Frame & operator=(const Frame &other)
Assignment operator.
Definition: Frame.cpp:96
static void cleanUpBuffer(void *info)
Clean up buffer after QImage is deleted.
Definition: Frame.cpp:1045
void AddColor(int new_width, int new_height, std::string new_color)
Add (or replace) pixel data to the frame (based on a solid color)
Definition: Frame.cpp:733
float * GetInterleavedAudioSamples(int new_sample_rate, openshot::AudioResampler *resampler, int *sample_count)
Get an array of sample data (all channels interleaved together), using any sample rate.
Definition: Frame.cpp:387
const unsigned char * GetPixels()
Get pixel data (as packets)
Definition: Frame.cpp:469
std::shared_ptr< QImage > GetWaveform(int width, int height, int Red, int Green, int Blue, int Alpha)
Get an audio waveform image.
Definition: Frame.cpp:176
std::shared_ptr< Magick::Image > GetMagickImage()
Get pointer to ImageMagick image object.
Definition: Frame.cpp:925
void Save(std::string path, float scale, std::string format="PNG", int quality=100)
Save the frame image to the specified path. The image format can be BMP, JPG, JPEG,...
Definition: Frame.cpp:578
void Display()
Display the frame image to the screen (primarily used for debugging reasons)
Definition: Frame.cpp:135
int GetAudioChannelsCount()
Get number of audio channels.
Definition: Frame.cpp:432
bool has_image_data
This frame has been loaded with pixel data.
Definition: Frame.h:131
int GetWidth()
Get height of image.
Definition: Frame.cpp:559
int SampleRate()
Get the original sample rate of this frame's audio data.
Definition: Frame.cpp:565
void ResizeAudio(int channels, int length, int sample_rate, openshot::ChannelLayout channel_layout)
Resize audio container to hold more (or less) samples and channels.
Definition: Frame.cpp:860
void DeepCopy(const Frame &other)
Copy data and pointers from another Frame instance.
Definition: Frame.cpp:105
float * GetPlanarAudioSamples(int new_sample_rate, openshot::AudioResampler *resampler, int *sample_count)
Definition: Frame.cpp:341
void AddMagickImage(std::shared_ptr< Magick::Image > new_image)
Add (or replace) pixel data to the frame from an ImageMagick Image.
Definition: Frame.cpp:949
void ClearWaveform()
Clear the waveform image (and deallocate its memory)
Definition: Frame.cpp:271
void Play()
Play audio samples for this frame.
Definition: Frame.cpp:975
bool has_audio_data
This frame has been loaded with audio data.
Definition: Frame.h:130
int64_t GetBytes()
Get the size in bytes of this frame (rough estimate)
Definition: Frame.cpp:454
openshot::ChannelLayout ChannelsLayout()
Definition: Frame.cpp:571
void AddAudioSilence(int numSamples)
Add audio silence.
Definition: Frame.cpp:1056
void Thumbnail(std::string path, int new_width, int new_height, std::string mask_path, std::string overlay_path, std::string background_color, bool ignore_aspect, std::string format="png", int quality=100, float rotate=0.0)
Definition: Frame.cpp:609
void DisplayWaveform()
Display the wave form.
Definition: Frame.cpp:288
juce::AudioSampleBuffer * GetAudioSampleBuffer()
Definition: Frame.cpp:448
bool CheckPixel(int row, int col, int red, int green, int blue, int alpha, int threshold)
Check a specific pixel color value (returns True/False)
Definition: Frame.cpp:488
float * GetAudioSamples(int channel)
Get an array of sample data.
Definition: Frame.cpp:334
float GetAudioSample(int channel, int sample, int magnitude_range)
Get magnitude of range of samples (if channel is -1, return average of all channels for that sample)
Definition: Frame.cpp:321
virtual ~Frame()
Destructor.
Definition: Frame.cpp:128
int GetSamplesPerFrame(openshot::Fraction fps, int sample_rate, int channels)
Calculate the # of samples per video frame (for the current frame number)
Definition: Frame.cpp:547
std::shared_ptr< QImage > GetImage()
Get pointer to Qt QImage image object.
Definition: Frame.cpp:913
Frame()
Constructor - blank frame (300x200 blank image, 48kHz audio silence)
Definition: Frame.cpp:37
void AddImage(int new_width, int new_height, int bytes_per_pixel, QImage::Format type, const unsigned char *pixels_)
Add (or replace) pixel data to the frame.
Definition: Frame.cpp:754
void ApplyGainRamp(int destChannel, int destStartSample, int numSamples, float initial_gain, float final_gain)
Apply gain ramp (i.e. fading volume)
Definition: Frame.cpp:904
int GetAudioSamplesCount()
Get number of audio samples.
Definition: Frame.cpp:442
void AddAudio(bool replaceSamples, int destChannel, int destStartSample, const float *source, int numSamples, float gainToApplyToSource)
Add audio samples to a specific channel.
Definition: Frame.cpp:874
const unsigned char * GetWaveformPixels(int width, int height, int Red, int Green, int Blue, int Alpha)
Get an audio waveform image pixels.
Definition: Frame.cpp:278
int GetHeight()
Get height of image.
Definition: Frame.cpp:553
void SetPixelRatio(int num, int den)
Set Pixel Aspect Ratio.
Definition: Frame.cpp:510
void SetFrameNumber(int64_t number)
Set frame number.
Definition: Frame.cpp:517
int64_t number
This is the frame number (starting at 1)
Definition: Frame.h:129
This namespace is the default namespace for all code in the openshot library.
ChannelLayout
This enumeration determines the audio channel layout (such as stereo, mono, 5 point surround,...