Music Hub  ..
A session-wide music playback service
playbin.h
Go to the documentation of this file.
1 /*
2  * Copyright © 2013 Canonical Ltd.
3  *
4  * This program is free software: you can redistribute it and/or modify it
5  * under the terms of the GNU Lesser General Public License version 3,
6  * as published by the Free Software Foundation.
7  *
8  * This program is distributed in the hope that it will be useful,
9  * but WITHOUT ANY WARRANTY; without even the implied warranty of
10  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11  * GNU Lesser General Public License for more details.
12  *
13  * You should have received a copy of the GNU Lesser General Public License
14  * along with this program. If not, see <http://www.gnu.org/licenses/>.
15  *
16  * Authored by: Thomas Voß <thomas.voss@canonical.com>
17  */
18 
19 #ifndef GSTREAMER_PLAYBIN_H_
20 #define GSTREAMER_PLAYBIN_H_
21 
22 #include "bus.h"
23 #include "../mpris/player.h"
24 
25 #include <hybris/media/surface_texture_client_hybris.h>
26 #include <hybris/media/media_codec_layer.h>
27 
28 #include <gio/gio.h>
29 #include <gst/gst.h>
30 
31 #include <chrono>
32 #include <string>
33 
34 // Uncomment to generate a dot file at the time that the pipeline
35 // goes to the PLAYING state. Make sure to export GST_DEBUG_DUMP_DOT_DIR
36 // before starting media-hub-server. To convert the dot file to something
37 // other image format, use: dot pipeline.dot -Tpng -o pipeline.png
38 //#define DEBUG_GST_PIPELINE
39 
40 namespace media = core::ubuntu::media;
41 
42 namespace gstreamer
43 {
44 struct Playbin
45 {
46  enum PlayFlags
47  {
48  GST_PLAY_FLAG_VIDEO = (1 << 0),
49  GST_PLAY_FLAG_AUDIO = (1 << 1),
50  GST_PLAY_FLAG_TEXT = (1 << 2)
51  };
52 
54  {
58  };
59 
60  static const std::string& pipeline_name()
61  {
62  static const std::string s{"playbin"};
63  return s;
64  }
65 
66  static void about_to_finish(GstElement*,
67  gpointer user_data)
68  {
69  auto thiz = static_cast<Playbin*>(user_data);
70  thiz->signals.about_to_finish();
71  }
72 
73  static void source_setup(GstElement*,
74  GstElement *source,
75  gpointer user_data)
76  {
77  if (user_data == nullptr)
78  return;
79 
80  static_cast<Playbin*>(user_data)->setup_source(source);
81  }
82 
84  : pipeline(gst_element_factory_make("playbin", pipeline_name().c_str())),
85  bus{gst_element_get_bus(pipeline)},
87  video_sink(nullptr),
88  video_height(0),
89  video_width(0),
91  bus.on_new_message.connect(
92  std::bind(
94  this,
95  std::placeholders::_1))),
96  is_seeking(false),
97  player_lifetime(media::Player::Lifetime::normal)
98  {
99  if (!pipeline)
100  throw std::runtime_error("Could not create pipeline for playbin.");
101 
102  // Add audio and/or video sink elements depending on environment variables
103  // being set or not set
105 
106  g_signal_connect(
107  pipeline,
108  "about-to-finish",
109  G_CALLBACK(about_to_finish),
110  this
111  );
112 
113  g_signal_connect(
114  pipeline,
115  "source-setup",
116  G_CALLBACK(source_setup),
117  this
118  );
119 
120  }
121 
123  {
124  if (pipeline)
125  gst_object_unref(pipeline);
126  }
127 
128  void reset()
129  {
130  std::cout << "Client died, resetting pipeline" << std::endl;
131  // When the client dies, tear down the current pipeline and get it
132  // in a state that is ready for the next client that connects to the
133  // service
134 
135  // Don't reset the pipeline if we want to resume
136  if (player_lifetime != media::Player::Lifetime::resumable) {
137  reset_pipeline();
138  }
139 
140  // Signal to the Player class that the client side has disconnected
141  signals.client_disconnected();
142  }
143 
145  {
146  std::cout << __PRETTY_FUNCTION__ << std::endl;
147  auto ret = gst_element_set_state(pipeline, GST_STATE_NULL);
148  switch(ret)
149  {
150  case GST_STATE_CHANGE_FAILURE:
151  std::cout << "Failed to reset the pipeline state. Client reconnect may not function properly." << std::endl;
152  break;
153  case GST_STATE_CHANGE_NO_PREROLL:
154  case GST_STATE_CHANGE_SUCCESS:
155  case GST_STATE_CHANGE_ASYNC:
156  break;
157  default:
158  std::cout << "Failed to reset the pipeline state. Client reconnect may not function properly." << std::endl;
159  }
161  }
162 
163  void on_new_message(const Bus::Message& message)
164  {
165  switch(message.type)
166  {
167  case GST_MESSAGE_ERROR:
168  signals.on_error(message.detail.error_warning_info);
169  break;
170  case GST_MESSAGE_WARNING:
171  signals.on_warning(message.detail.error_warning_info);
172  break;
173  case GST_MESSAGE_INFO:
174  signals.on_info(message.detail.error_warning_info);
175  break;
176  case GST_MESSAGE_TAG:
177  {
178  gchar *orientation;
179  if (gst_tag_list_get_string(message.detail.tag.tag_list, "image-orientation", &orientation))
180  {
181  // If the image-orientation tag is in the GstTagList, signal the Engine
182  signals.on_orientation_changed(orientation_lut(orientation));
183  g_free (orientation);
184  }
185 
186  signals.on_tag_available(message.detail.tag);
187  }
188  break;
189  case GST_MESSAGE_STATE_CHANGED:
190  signals.on_state_changed(message.detail.state_changed);
191  break;
192  case GST_MESSAGE_ASYNC_DONE:
193  if (is_seeking)
194  {
195  // FIXME: Pass the actual playback time position to the signal call
196  signals.on_seeked_to(0);
197  is_seeking = false;
198  }
199  break;
200  case GST_MESSAGE_EOS:
201  signals.on_end_of_stream();
202  default:
203  break;
204  }
205  }
206 
208  {
209  return bus;
210  }
211 
213  {
214  gint flags;
215  g_object_get (pipeline, "flags", &flags, nullptr);
216  flags |= GST_PLAY_FLAG_AUDIO;
217  flags |= GST_PLAY_FLAG_VIDEO;
218  flags &= ~GST_PLAY_FLAG_TEXT;
219  g_object_set (pipeline, "flags", flags, nullptr);
220 
221  if (::getenv("CORE_UBUNTU_MEDIA_SERVICE_AUDIO_SINK_NAME") != nullptr)
222  {
223  auto audio_sink = gst_element_factory_make (
224  ::getenv("CORE_UBUNTU_MEDIA_SERVICE_AUDIO_SINK_NAME"),
225  "audio-sink");
226 
227  std::cout << "audio_sink: " << ::getenv("CORE_UBUNTU_MEDIA_SERVICE_AUDIO_SINK_NAME") << std::endl;
228 
229  g_object_set (
230  pipeline,
231  "audio-sink",
232  audio_sink,
233  NULL);
234  }
235 
236  if (::getenv("CORE_UBUNTU_MEDIA_SERVICE_VIDEO_SINK_NAME") != nullptr)
237  {
238  video_sink = gst_element_factory_make (
239  ::getenv("CORE_UBUNTU_MEDIA_SERVICE_VIDEO_SINK_NAME"),
240  "video-sink");
241 
242  std::cout << "video_sink: " << ::getenv("CORE_UBUNTU_MEDIA_SERVICE_VIDEO_SINK_NAME") << std::endl;
243 
244  g_object_set (
245  pipeline,
246  "video-sink",
247  video_sink,
248  NULL);
249  }
250  }
251 
252  void create_video_sink(uint32_t texture_id)
253  {
254  std::cout << "Creating video sink for texture_id: " << texture_id << std::endl;
255 
256  if (::getenv("CORE_UBUNTU_MEDIA_SERVICE_VIDEO_SINK_NAME") != nullptr)
257  {
258  g_object_get (pipeline, "video_sink", &video_sink, NULL);
259 
260  // Get the service-side BufferQueue (IGraphicBufferProducer) and associate it with
261  // the SurfaceTextureClientHybris instance
262  IGBPWrapperHybris igbp = decoding_service_get_igraphicbufferproducer();
263  SurfaceTextureClientHybris stc = surface_texture_client_create_by_igbp(igbp);
264  // Because mirsink is being loaded, we are definitely doing * hardware rendering.
265  surface_texture_client_set_hardware_rendering (stc, TRUE);
266  g_object_set (G_OBJECT (video_sink), "surface", static_cast<gpointer>(stc), static_cast<char*>(NULL));
267  }
268  }
269 
270  void set_volume(double new_volume)
271  {
272  g_object_set (pipeline, "volume", new_volume, NULL);
273  }
274 
276  static std::string get_audio_role_str(media::Player::AudioStreamRole audio_role)
277  {
278  switch (audio_role)
279  {
280  case media::Player::AudioStreamRole::alarm:
281  return "alarm";
282  break;
283  case media::Player::AudioStreamRole::alert:
284  return "alert";
285  break;
286  case media::Player::AudioStreamRole::multimedia:
287  return "multimedia";
288  break;
289  case media::Player::AudioStreamRole::phone:
290  return "phone";
291  break;
292  default:
293  return "multimedia";
294  break;
295  }
296  }
297 
298  media::Player::Orientation orientation_lut(const gchar *orientation)
299  {
300  if (g_strcmp0(orientation, "rotate-0") == 0)
301  return media::Player::Orientation::rotate0;
302  else if (g_strcmp0(orientation, "rotate-90") == 0)
303  return media::Player::Orientation::rotate90;
304  else if (g_strcmp0(orientation, "rotate-180") == 0)
305  return media::Player::Orientation::rotate180;
306  else if (g_strcmp0(orientation, "rotate-270") == 0)
307  return media::Player::Orientation::rotate270;
308  else
309  return media::Player::Orientation::rotate0;
310  }
311 
314  {
315  GstElement *audio_sink = NULL;
316  g_object_get (pipeline, "audio-sink", &audio_sink, NULL);
317 
318  std::string role_str("props,media.role=" + get_audio_role_str(new_audio_role));
319  std::cout << "Audio stream role: " << role_str << std::endl;
320 
321  GstStructure *props = gst_structure_from_string (role_str.c_str(), NULL);
322  if (audio_sink != nullptr && props != nullptr)
323  g_object_set (audio_sink, "stream-properties", props, NULL);
324  else
325  {
326  std::cerr <<
327  "Warning: couldn't set audio stream role - couldn't get audio_sink from pipeline" <<
328  std::endl;
329  }
330 
331  gst_structure_free (props);
332  }
333 
335  {
336  player_lifetime = lifetime;
337  }
338 
339  uint64_t position() const
340  {
341  int64_t pos = 0;
342  gst_element_query_position (pipeline, GST_FORMAT_TIME, &pos);
343 
344  // FIXME: this should be int64_t, but dbus-cpp doesn't seem to handle it correctly
345  return static_cast<uint64_t>(pos);
346  }
347 
348  uint64_t duration() const
349  {
350  int64_t dur = 0;
351  gst_element_query_duration (pipeline, GST_FORMAT_TIME, &dur);
352 
353  // FIXME: this should be int64_t, but dbus-cpp doesn't seem to handle it correctly
354  return static_cast<uint64_t>(dur);
355  }
356 
357  void set_uri(const std::string& uri,
359  {
360  reset_pipeline();
361 
362  g_object_set(pipeline, "uri", uri.c_str(), NULL);
363  if (is_video_file(uri))
365  else if (is_audio_file(uri))
367 
368  request_headers = headers;
369  }
370 
371  void setup_source(GstElement *source)
372  {
373  if (source == NULL || request_headers.empty())
374  return;
375 
376  if (request_headers.find("Cookie") != request_headers.end()) {
377  if (g_object_class_find_property(G_OBJECT_GET_CLASS(source),
378  "cookies") != NULL) {
379  gchar ** cookies = g_strsplit(request_headers["Cookie"].c_str(), ";", 0);
380  g_object_set(source, "cookies", cookies, NULL);
381  g_strfreev(cookies);
382  }
383  }
384 
385  if (request_headers.find("User-Agent") != request_headers.end()) {
386  if (g_object_class_find_property(G_OBJECT_GET_CLASS(source),
387  "user-agent") != NULL) {
388  g_object_set(source, "user-agent", request_headers["User-Agent"].c_str(), NULL);
389  }
390  }
391  }
392 
393  std::string uri() const
394  {
395  gchar* data = nullptr;
396  g_object_get(pipeline, "current-uri", &data, nullptr);
397 
398  std::string result((data == nullptr ? "" : data));
399  g_free(data);
400 
401  return result;
402  }
403 
404  bool set_state_and_wait(GstState new_state)
405  {
406  static const std::chrono::nanoseconds state_change_timeout
407  {
408  // We choose a quite high value here as tests are run under valgrind
409  // and gstreamer pipeline setup/state changes take longer in that scenario.
410  // The value does not negatively impact runtime performance.
411  std::chrono::milliseconds{5000}
412  };
413 
414  auto ret = gst_element_set_state(pipeline, new_state);
415  bool result = false; GstState current, pending;
416  switch(ret)
417  {
418  case GST_STATE_CHANGE_FAILURE:
419  result = false; break;
420  case GST_STATE_CHANGE_NO_PREROLL:
421  case GST_STATE_CHANGE_SUCCESS:
422  result = true; break;
423  case GST_STATE_CHANGE_ASYNC:
424  result = GST_STATE_CHANGE_SUCCESS == gst_element_get_state(
425  pipeline,
426  &current,
427  &pending,
428  state_change_timeout.count());
429 
430  if (new_state == GST_STATE_PLAYING)
431  {
432  // Get the video height/width from the video sink
434 #ifdef DEBUG_GST_PIPELINE
435  std::cout << "Dumping pipeline dot file" << std::endl;
436  GST_DEBUG_BIN_TO_DOT_FILE((GstBin*)pipeline, GST_DEBUG_GRAPH_SHOW_ALL, "pipeline");
437 #endif
438  }
439  break;
440  }
441 
442  return result;
443  }
444 
445  bool seek(const std::chrono::microseconds& ms)
446  {
447  is_seeking = true;
448  return gst_element_seek_simple(
449  pipeline,
450  GST_FORMAT_TIME,
451  (GstSeekFlags)(GST_SEEK_FLAG_FLUSH | GST_SEEK_FLAG_KEY_UNIT),
452  ms.count() * 1000);
453  }
454 
456  {
457  if (video_sink != nullptr && g_strcmp0(::getenv("CORE_UBUNTU_MEDIA_SERVICE_VIDEO_SINK_NAME"), "mirsink") == 0)
458  {
459  g_object_get (video_sink, "height", &video_height, nullptr);
460  g_object_get (video_sink, "width", &video_width, nullptr);
461  std::cout << "video_height: " << video_height << ", video_width: " << video_width << std::endl;
462  signals.on_add_frame_dimension(video_height, video_width);
463  }
464  else
465  std::cerr << "Could not get the height/width of each video frame" << std::endl;
466  }
467 
468  int get_video_height() const
469  {
470  return video_height;
471  }
472 
473  int get_video_width() const
474  {
475  return video_width;
476  }
477 
478  std::string get_file_content_type(const std::string& uri) const
479  {
480  if (uri.empty())
481  return std::string();
482 
483  std::string filename(uri);
484  std::cout << "filename: " << filename << std::endl;
485  size_t pos = uri.find("file://");
486  if (pos != std::string::npos)
487  filename = uri.substr(pos + 7, std::string::npos);
488  else
489  // Anything other than a file, for now claim that the type
490  // is both audio and video.
491  // FIXME: implement true net stream sampling and get the type from GstCaps
492  return std::string("audio/video/");
493 
494 
495  GError *error = nullptr;
496  std::unique_ptr<GFile, void(*)(void *)> file(
497  g_file_new_for_path(filename.c_str()), g_object_unref);
498  std::unique_ptr<GFileInfo, void(*)(void *)> info(
499  g_file_query_info(
500  file.get(), G_FILE_ATTRIBUTE_STANDARD_FAST_CONTENT_TYPE ","
501  G_FILE_ATTRIBUTE_ETAG_VALUE, G_FILE_QUERY_INFO_NONE,
502  /* cancellable */ NULL, &error),
503  g_object_unref);
504  if (!info)
505  {
506  std::string error_str(error->message);
507  g_error_free(error);
508 
509  std::cout << "Failed to query the URI for the presence of video content: "
510  << error_str << std::endl;
511  return std::string();
512  }
513 
514  std::string content_type(g_file_info_get_attribute_string(
515  info.get(), G_FILE_ATTRIBUTE_STANDARD_FAST_CONTENT_TYPE));
516 
517  return content_type;
518  }
519 
520  bool is_audio_file(const std::string& uri) const
521  {
522  if (uri.empty())
523  return false;
524 
525  if (get_file_content_type(uri).find("audio/") == 0)
526  {
527  std::cout << "Found audio content" << std::endl;
528  return true;
529  }
530 
531  return false;
532  }
533 
534  bool is_video_file(const std::string& uri) const
535  {
536  if (uri.empty())
537  return false;
538 
539  if (get_file_content_type(uri).find("video/") == 0)
540  {
541  std::cout << "Found video content" << std::endl;
542  return true;
543  }
544 
545  return false;
546  }
547 
549  {
550  return file_type;
551  }
552 
553  GstElement* pipeline;
556  SurfaceTextureClientHybris stc_hybris;
557  GstElement* video_sink;
558  uint32_t video_height;
559  uint32_t video_width;
560  core::Connection on_new_message_connection;
564  struct
565  {
566  core::Signal<void> about_to_finish;
567  core::Signal<Bus::Message::Detail::ErrorWarningInfo> on_error;
568  core::Signal<Bus::Message::Detail::ErrorWarningInfo> on_warning;
569  core::Signal<Bus::Message::Detail::ErrorWarningInfo> on_info;
570  core::Signal<Bus::Message::Detail::Tag> on_tag_available;
571  core::Signal<Bus::Message::Detail::StateChanged> on_state_changed;
572  core::Signal<uint64_t> on_seeked_to;
573  core::Signal<void> on_end_of_stream;
574  core::Signal<media::Player::PlaybackStatus> on_playback_status_changed;
575  core::Signal<media::Player::Orientation> on_orientation_changed;
576  core::Signal<uint32_t, uint32_t> on_add_frame_dimension;
577  core::Signal<void> client_disconnected;
578  } signals;
579 };
580 }
581 
582 #endif // GSTREAMER_PLAYBIN_H_
core::Signal< Bus::Message::Detail::StateChanged > on_state_changed
Definition: playbin.h:571
core::Signal< Message > on_new_message
Definition: bus.h:303
core::Signal< Bus::Message::Detail::Tag > on_tag_available
Definition: playbin.h:570
bool set_state_and_wait(GstState new_state)
Definition: playbin.h:404
void setup_source(GstElement *source)
Definition: playbin.h:371
void get_video_dimensions()
Definition: playbin.h:455
void set_audio_stream_role(media::Player::AudioStreamRole new_audio_role)
Definition: playbin.h:313
GstMessageType type
Definition: bus.h:181
Definition: bus.h:33
bool seek(const std::chrono::microseconds &ms)
Definition: playbin.h:445
static void about_to_finish(GstElement *, gpointer user_data)
Definition: playbin.h:66
uint64_t duration() const
Definition: playbin.h:348
core::Signal< void > about_to_finish
Definition: playbin.h:566
void reset_pipeline()
Definition: playbin.h:144
int get_video_width() const
Definition: playbin.h:473
core::Signal< Bus::Message::Detail::ErrorWarningInfo > on_warning
Definition: playbin.h:568
GstElement * video_sink
Definition: playbin.h:557
std::map< std::string, std::string > HeadersType
Definition: player.h:45
core::Connection on_new_message_connection
Definition: playbin.h:560
void set_uri(const std::string &uri, const core::ubuntu::media::Player::HeadersType &headers=core::ubuntu::media::Player::HeadersType())
Definition: playbin.h:357
int get_video_height() const
Definition: playbin.h:468
core::Signal< uint64_t > on_seeked_to
Definition: playbin.h:572
void on_new_message(const Bus::Message &message)
Definition: playbin.h:163
void set_lifetime(media::Player::Lifetime lifetime)
Definition: playbin.h:334
core::Signal< media::Player::Orientation > on_orientation_changed
Definition: playbin.h:575
media::Player::Orientation orientation_lut(const gchar *orientation)
Definition: playbin.h:298
SurfaceTextureClientHybris stc_hybris
Definition: playbin.h:556
MediaFileType media_file_type() const
Definition: playbin.h:548
std::string uri() const
Definition: playbin.h:393
void setup_pipeline_for_audio_video()
Definition: playbin.h:212
media::Player::Lifetime player_lifetime
Definition: playbin.h:563
union gstreamer::Bus::Message::Detail detail
uint32_t video_height
Definition: playbin.h:558
GstElement * pipeline
Definition: playbin.h:553
static std::string get_audio_role_str(media::Player::AudioStreamRole audio_role)
Definition: playbin.h:276
core::Signal< Bus::Message::Detail::ErrorWarningInfo > on_info
Definition: playbin.h:569
core::Signal< media::Player::PlaybackStatus > on_playback_status_changed
Definition: playbin.h:574
core::Signal< void > on_end_of_stream
Definition: playbin.h:573
std::string get_file_content_type(const std::string &uri) const
Definition: playbin.h:478
struct gstreamer::Playbin::@11 signals
struct gstreamer::Bus::Message::Detail::Tag tag
bool is_audio_file(const std::string &uri) const
Definition: playbin.h:520
bool is_video_file(const std::string &uri) const
Definition: playbin.h:534
struct gstreamer::Bus::Message::Detail::ErrorWarningInfo error_warning_info
core::Signal< uint32_t, uint32_t > on_add_frame_dimension
Definition: playbin.h:576
void set_volume(double new_volume)
Definition: playbin.h:270
core::Signal< Bus::Message::Detail::ErrorWarningInfo > on_error
Definition: playbin.h:567
void create_video_sink(uint32_t texture_id)
Definition: playbin.h:252
gstreamer::Bus bus
Definition: playbin.h:554
MediaFileType file_type
Definition: playbin.h:555
core::Signal< void > client_disconnected
Definition: playbin.h:577
core::ubuntu::media::Player::HeadersType request_headers
Definition: playbin.h:562
uint32_t video_width
Definition: playbin.h:559
static void source_setup(GstElement *, GstElement *source, gpointer user_data)
Definition: playbin.h:73
uint64_t position() const
Definition: playbin.h:339
static const std::string & pipeline_name()
Definition: playbin.h:60
gstreamer::Bus & message_bus()
Definition: playbin.h:207
struct gstreamer::Bus::Message::Detail::StateChanged state_changed