Music Hub  ..
A session-wide music playback service
playbin.cpp
Go to the documentation of this file.
1 /*
2  * Copyright © 2013 Canonical Ltd.
3  *
4  * This program is free software: you can redistribute it and/or modify it
5  * under the terms of the GNU Lesser General Public License version 3,
6  * as published by the Free Software Foundation.
7  *
8  * This program is distributed in the hope that it will be useful,
9  * but WITHOUT ANY WARRANTY; without even the implied warranty of
10  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11  * GNU Lesser General Public License for more details.
12  *
13  * You should have received a copy of the GNU Lesser General Public License
14  * along with this program. If not, see <http://www.gnu.org/licenses/>.
15  *
16  * Authored by: Thomas Voß <thomas.voss@canonical.com>
17  */
18 
20 
22 
23 #if defined(MEDIA_HUB_HAVE_HYBRIS_MEDIA_COMPAT_LAYER)
24 #include <hybris/media/surface_texture_client_hybris.h>
25 #include <hybris/media/media_codec_layer.h>
26 
27 #include <utility>
28 
29 namespace
30 {
31 void setup_video_sink_for_buffer_streaming(GstElement* video_sink)
32 {
33  // Get the service-side BufferQueue (IGraphicBufferProducer) and associate it with
34  // the SurfaceTextureClientHybris instance
35  IGBPWrapperHybris igbp = decoding_service_get_igraphicbufferproducer();
36  SurfaceTextureClientHybris stc = surface_texture_client_create_by_igbp(igbp);
37  // Because mirsink is being loaded, we are definitely doing * hardware rendering.
38  surface_texture_client_set_hardware_rendering (stc, TRUE);
39  g_object_set (G_OBJECT (video_sink), "surface", static_cast<gpointer>(stc), static_cast<char*>(NULL));
40 }
41 }
42 #else // MEDIA_HUB_HAVE_HYBRIS_MEDIA_COMPAT_LAYER
43 namespace
44 {
45 void setup_video_sink_for_buffer_streaming(GstElement*)
46 {
47  throw core::ubuntu::media::Player::Error::OutOfProcessBufferStreamingNotSupported{};
48 }
49 }
50 #endif // MEDIA_HUB_HAVE_HYBRIS_MEDIA_COMPAT_LAYER
51 
52 namespace
53 {
54 bool is_mir_video_sink()
55 {
56  return g_strcmp0(::getenv("CORE_UBUNTU_MEDIA_SERVICE_VIDEO_SINK_NAME"), "mirsink") == 0;
57 }
58 }
59 // Uncomment to generate a dot file at the time that the pipeline
60 // goes to the PLAYING state. Make sure to export GST_DEBUG_DUMP_DOT_DIR
61 // before starting media-hub-server. To convert the dot file to something
62 // other image format, use: dot pipeline.dot -Tpng -o pipeline.png
63 //#define DEBUG_GST_PIPELINE
64 
65 namespace media = core::ubuntu::media;
67 
69 {
70  static const std::string s{"playbin"};
71  return s;
72 }
73 
74 void gstreamer::Playbin::about_to_finish(GstElement*, gpointer user_data)
75 {
76  auto thiz = static_cast<Playbin*>(user_data);
77  thiz->signals.about_to_finish();
78 }
79 
81  GstElement *source,
82  gpointer user_data)
83 {
84  if (user_data == nullptr)
85  return;
86 
87  static_cast<Playbin*>(user_data)->setup_source(source);
88 }
89 
91  : pipeline(gst_element_factory_make("playbin", pipeline_name().c_str())),
92  bus{gst_element_get_bus(pipeline)},
93  file_type(MEDIA_FILE_TYPE_NONE),
94  video_sink(nullptr),
95  on_new_message_connection_async(
96  bus.on_new_message_async.connect(
97  std::bind(
99  this,
100  std::placeholders::_1))),
101  is_seeking(false),
102  previous_position(0),
103  player_lifetime(media::Player::Lifetime::normal),
104  is_eos(false),
105  about_to_finish_handler_id(0),
106  source_setup_handler_id(0)
107 {
108  if (!pipeline)
109  throw std::runtime_error("Could not create pipeline for playbin.");
110 
111  // Add audio and/or video sink elements depending on environment variables
112  // being set or not set
113  setup_pipeline_for_audio_video();
114 
115  about_to_finish_handler_id = g_signal_connect(
116  pipeline,
117  "about-to-finish",
118  G_CALLBACK(about_to_finish),
119  this
120  );
121 
122  source_setup_handler_id = g_signal_connect(
123  pipeline,
124  "source-setup",
125  G_CALLBACK(source_setup),
126  this
127  );
128 }
129 
131 {
132  g_signal_handler_disconnect(pipeline, about_to_finish_handler_id);
133  g_signal_handler_disconnect(pipeline, source_setup_handler_id);
134 
135  if (pipeline)
136  gst_object_unref(pipeline);
137 }
138 
140 {
141  std::cout << "Client died, resetting pipeline" << std::endl;
142  // When the client dies, tear down the current pipeline and get it
143  // in a state that is ready for the next client that connects to the
144  // service
145 
146  // Don't reset the pipeline if we want to resume
147  if (player_lifetime != media::Player::Lifetime::resumable) {
148  reset_pipeline();
149  }
150  // Signal to the Player class that the client side has disconnected
151  signals.client_disconnected();
152 }
153 
155 {
156  std::cout << __PRETTY_FUNCTION__ << std::endl;
157  auto ret = gst_element_set_state(pipeline, GST_STATE_NULL);
158  switch(ret)
159  {
160  case GST_STATE_CHANGE_FAILURE:
161  std::cout << "Failed to reset the pipeline state. Client reconnect may not function properly." << std::endl;
162  break;
163  case GST_STATE_CHANGE_NO_PREROLL:
164  case GST_STATE_CHANGE_SUCCESS:
165  case GST_STATE_CHANGE_ASYNC:
166  break;
167  default:
168  std::cout << "Failed to reset the pipeline state. Client reconnect may not function properly." << std::endl;
169  }
170  file_type = MEDIA_FILE_TYPE_NONE;
171 }
172 
174 {
175  switch(message.type)
176  {
177  case GST_MESSAGE_ERROR:
178  signals.on_error(message.detail.error_warning_info);
179  break;
180  case GST_MESSAGE_WARNING:
181  signals.on_warning(message.detail.error_warning_info);
182  break;
183  case GST_MESSAGE_INFO:
184  signals.on_info(message.detail.error_warning_info);
185  break;
186  case GST_MESSAGE_STATE_CHANGED:
187  signals.on_state_changed(std::make_pair(message.detail.state_changed, message.source));
188  break;
189  case GST_MESSAGE_TAG:
190  {
191  gchar *orientation;
192  if (gst_tag_list_get_string(message.detail.tag.tag_list, "image-orientation", &orientation))
193  {
194  // If the image-orientation tag is in the GstTagList, signal the Engine
195  signals.on_orientation_changed(orientation_lut(orientation));
196  g_free (orientation);
197  }
198 
199  signals.on_tag_available(message.detail.tag);
200  }
201  break;
202  case GST_MESSAGE_ASYNC_DONE:
203  if (is_seeking)
204  {
205  // FIXME: Pass the actual playback time position to the signal call
206  signals.on_seeked_to(0);
207  is_seeking = false;
208  }
209  break;
210  case GST_MESSAGE_EOS:
211  is_eos = true;
212  signals.on_end_of_stream();
213  default:
214  break;
215  }
216 }
217 
219 {
220  return bus;
221 }
222 
224 {
225  gint flags;
226  g_object_get (pipeline, "flags", &flags, nullptr);
227  flags |= GST_PLAY_FLAG_AUDIO;
228  flags |= GST_PLAY_FLAG_VIDEO;
229  flags &= ~GST_PLAY_FLAG_TEXT;
230  g_object_set (pipeline, "flags", flags, nullptr);
231 
232  if (::getenv("CORE_UBUNTU_MEDIA_SERVICE_AUDIO_SINK_NAME") != nullptr)
233  {
234  auto audio_sink = gst_element_factory_make (
235  ::getenv("CORE_UBUNTU_MEDIA_SERVICE_AUDIO_SINK_NAME"),
236  "audio-sink");
237 
238  std::cout << "audio_sink: " << ::getenv("CORE_UBUNTU_MEDIA_SERVICE_AUDIO_SINK_NAME") << std::endl;
239 
240  g_object_set (
241  pipeline,
242  "audio-sink",
243  audio_sink,
244  NULL);
245  }
246 
247  if (::getenv("CORE_UBUNTU_MEDIA_SERVICE_VIDEO_SINK_NAME") != nullptr)
248  {
249  video_sink = gst_element_factory_make (
250  ::getenv("CORE_UBUNTU_MEDIA_SERVICE_VIDEO_SINK_NAME"),
251  "video-sink");
252 
253  std::cout << "video_sink: " << ::getenv("CORE_UBUNTU_MEDIA_SERVICE_VIDEO_SINK_NAME") << std::endl;
254 
255  g_object_set (
256  pipeline,
257  "video-sink",
258  video_sink,
259  NULL);
260  }
261 }
262 
264 {
265  if (not video_sink) throw std::logic_error
266  {
267  "No video sink configured for the current pipeline"
268  };
269 
270  setup_video_sink_for_buffer_streaming(video_sink);
271 }
272 
273 void gstreamer::Playbin::set_volume(double new_volume)
274 {
275  g_object_set (pipeline, "volume", new_volume, NULL);
276 }
277 
280 {
281  switch (audio_role)
282  {
283  case media::Player::AudioStreamRole::alarm:
284  return "alarm";
285  break;
286  case media::Player::AudioStreamRole::alert:
287  return "alert";
288  break;
289  case media::Player::AudioStreamRole::multimedia:
290  return "multimedia";
291  break;
292  case media::Player::AudioStreamRole::phone:
293  return "phone";
294  break;
295  default:
296  return "multimedia";
297  break;
298  }
299 }
300 
302 {
303  if (g_strcmp0(orientation, "rotate-0") == 0)
304  return media::Player::Orientation::rotate0;
305  else if (g_strcmp0(orientation, "rotate-90") == 0)
306  return media::Player::Orientation::rotate90;
307  else if (g_strcmp0(orientation, "rotate-180") == 0)
308  return media::Player::Orientation::rotate180;
309  else if (g_strcmp0(orientation, "rotate-270") == 0)
310  return media::Player::Orientation::rotate270;
311  else
312  return media::Player::Orientation::rotate0;
313 }
314 
317 {
318  GstElement *audio_sink = NULL;
319  g_object_get (pipeline, "audio-sink", &audio_sink, NULL);
320 
321  std::string role_str("props,media.role=" + get_audio_role_str(new_audio_role));
322  std::cout << "Audio stream role: " << role_str << std::endl;
323 
324  GstStructure *props = gst_structure_from_string (role_str.c_str(), NULL);
325  if (audio_sink != nullptr && props != nullptr)
326  g_object_set (audio_sink, "stream-properties", props, NULL);
327  else
328  {
329  std::cerr <<
330  "Warning: couldn't set audio stream role - couldn't get audio_sink from pipeline" <<
331  std::endl;
332  }
333 
334  gst_structure_free (props);
335 }
336 
338 {
339  player_lifetime = lifetime;
340 }
341 
343 {
344  int64_t pos = 0;
345  gst_element_query_position (pipeline, GST_FORMAT_TIME, &pos);
346 
347  // This prevents a 0 position from being reported to the app which happens while seeking.
348  // This is covering over a GStreamer issue
349  if ((static_cast<uint64_t>(pos) < duration()) && is_seeking && pos == 0)
350  {
351  return previous_position;
352  }
353 
354  // Save the current position to use just in case it's needed the next time position is
355  // requested
356  previous_position = static_cast<uint64_t>(pos);
357 
358  // FIXME: this should be int64_t, but dbus-cpp doesn't seem to handle it correctly
359  return static_cast<uint64_t>(pos);
360 }
361 
363 {
364  int64_t dur = 0;
365  gst_element_query_duration (pipeline, GST_FORMAT_TIME, &dur);
366 
367  // FIXME: this should be int64_t, but dbus-cpp doesn't seem to handle it correctly
368  return static_cast<uint64_t>(dur);
369 }
370 
372  const std::string& uri,
374  bool do_pipeline_reset)
375 {
376  if (do_pipeline_reset)
377  reset_pipeline();
378 
379  g_object_set(pipeline, "uri", uri.c_str(), NULL);
380  if (is_video_file(uri))
381  file_type = MEDIA_FILE_TYPE_VIDEO;
382  else if (is_audio_file(uri))
383  file_type = MEDIA_FILE_TYPE_AUDIO;
384 
385  request_headers = headers;
386 }
387 
388 void gstreamer::Playbin::setup_source(GstElement *source)
389 {
390  if (source == NULL || request_headers.empty())
391  return;
392 
393  if (request_headers.find("Cookie") != request_headers.end()) {
394  if (g_object_class_find_property(G_OBJECT_GET_CLASS(source),
395  "cookies") != NULL) {
396  gchar ** cookies = g_strsplit(request_headers["Cookie"].c_str(), ";", 0);
397  g_object_set(source, "cookies", cookies, NULL);
398  g_strfreev(cookies);
399  }
400  }
401 
402  if (request_headers.find("User-Agent") != request_headers.end()) {
403  if (g_object_class_find_property(G_OBJECT_GET_CLASS(source),
404  "user-agent") != NULL) {
405  g_object_set(source, "user-agent", request_headers["User-Agent"].c_str(), NULL);
406  }
407  }
408 }
409 
410 std::string gstreamer::Playbin::uri() const
411 {
412  gchar* data = nullptr;
413  g_object_get(pipeline, "current-uri", &data, nullptr);
414 
415  std::string result((data == nullptr ? "" : data));
416  g_free(data);
417 
418  return result;
419 }
420 
422 {
423  static const std::chrono::nanoseconds state_change_timeout
424  {
425  // We choose a quite high value here as tests are run under valgrind
426  // and gstreamer pipeline setup/state changes take longer in that scenario.
427  // The value does not negatively impact runtime performance.
428  std::chrono::milliseconds{5000}
429  };
430 
431  auto ret = gst_element_set_state(pipeline, new_state);
432 
433  std::cout << __PRETTY_FUNCTION__ << ": requested state change." << std::endl;
434 
435  bool result = false; GstState current, pending;
436  switch(ret)
437  {
438  case GST_STATE_CHANGE_FAILURE:
439  result = false; break;
440  case GST_STATE_CHANGE_NO_PREROLL:
441  case GST_STATE_CHANGE_SUCCESS:
442  result = true; break;
443  case GST_STATE_CHANGE_ASYNC:
444  result = GST_STATE_CHANGE_SUCCESS == gst_element_get_state(
445  pipeline,
446  &current,
447  &pending,
448  state_change_timeout.count());
449  break;
450  }
451 
452  // We only should query the pipeline if we actually succeeded in
453  // setting the requested state. Also don't send on_video_dimensions_changed
454  // signal during EOS.
455  if (result && new_state == GST_STATE_PLAYING && !is_eos)
456  {
457  // Get the video height/width from the video sink
458  try
459  {
460  signals.on_video_dimensions_changed(get_video_dimensions());
461  }
462  catch (const std::exception& e)
463  {
464  std::cerr << "Problem querying video dimensions: " << e.what() << std::endl;
465  }
466  catch (...)
467  {
468  std::cerr << "Problem querying video dimensions." << std::endl;
469  }
470 
471 #ifdef DEBUG_GST_PIPELINE
472  std::cout << "Dumping pipeline dot file" << std::endl;
473  GST_DEBUG_BIN_TO_DOT_FILE((GstBin*)pipeline, GST_DEBUG_GRAPH_SHOW_ALL, "pipeline");
474 #endif
475  }
476 
477  return result;
478 }
479 
480 bool gstreamer::Playbin::seek(const std::chrono::microseconds& ms)
481 {
482  is_seeking = true;
483  return gst_element_seek_simple(
484  pipeline,
485  GST_FORMAT_TIME,
486  (GstSeekFlags)(GST_SEEK_FLAG_FLUSH | GST_SEEK_FLAG_KEY_UNIT),
487  ms.count() * 1000);
488 }
489 
491 {
492  if (not video_sink || not is_mir_video_sink())
493  throw std::runtime_error
494  {
495  "Missing video sink or video sink does not support query of width and height."
496  };
497 
498  // Initialize to default value prior to querying actual values from the sink.
499  uint32_t video_width = 0, video_height = 0;
500  g_object_get (video_sink, "height", &video_height, nullptr);
501  g_object_get (video_sink, "width", &video_width, nullptr);
502  // TODO(tvoss): We should probably check here if width and height are valid.
504  {
507  };
508 }
509 
510 std::string gstreamer::Playbin::get_file_content_type(const std::string& uri) const
511 {
512  if (uri.empty())
513  return std::string();
514 
515  std::string filename(uri);
516  size_t pos = uri.find("file://");
517  if (pos != std::string::npos)
518  filename = uri.substr(pos + 7, std::string::npos);
519  else
520  // Anything other than a file, for now claim that the type
521  // is both audio and video.
522  // FIXME: implement true net stream sampling and get the type from GstCaps
523  return std::string("audio/video/");
524 
525 
526  GError *error = nullptr;
527  std::unique_ptr<GFile, void(*)(void *)> file(
528  g_file_new_for_path(filename.c_str()), g_object_unref);
529  std::unique_ptr<GFileInfo, void(*)(void *)> info(
530  g_file_query_info(
531  file.get(), G_FILE_ATTRIBUTE_STANDARD_FAST_CONTENT_TYPE ","
532  G_FILE_ATTRIBUTE_ETAG_VALUE, G_FILE_QUERY_INFO_NONE,
533  /* cancellable */ NULL, &error),
534  g_object_unref);
535  if (!info)
536  {
537  std::string error_str(error->message);
538  g_error_free(error);
539 
540  std::cout << "Failed to query the URI for the presence of video content: "
541  << error_str << std::endl;
542  return std::string();
543  }
544 
545  std::string content_type(g_file_info_get_attribute_string(
546  info.get(), G_FILE_ATTRIBUTE_STANDARD_FAST_CONTENT_TYPE));
547 
548  return content_type;
549 }
550 
551 bool gstreamer::Playbin::is_audio_file(const std::string& uri) const
552 {
553  if (uri.empty())
554  return false;
555 
556  if (get_file_content_type(uri).find("audio/") == 0)
557  {
558  std::cout << "Found audio content" << std::endl;
559  return true;
560  }
561 
562  return false;
563 }
564 
565 bool gstreamer::Playbin::is_video_file(const std::string& uri) const
566 {
567  if (uri.empty())
568  return false;
569 
570  if (get_file_content_type(uri).find("video/") == 0)
571  {
572  std::cout << "Found video content" << std::endl;
573  return true;
574  }
575 
576  return false;
577 }
578 
580 {
581  return file_type;
582 }
static std::string get_audio_role_str(core::ubuntu::media::Player::AudioStreamRole audio_role)
Definition: playbin.cpp:279
bool set_state_and_wait(GstState new_state)
Definition: playbin.cpp:421
static void source_setup(GstElement *, GstElement *source, gpointer user_data)
Definition: playbin.cpp:80
void setup_source(GstElement *source)
Definition: playbin.cpp:388
std::tuple< Height, Width > Dimensions
Height and Width of a video.
Definition: dimensions.h:139
void set_uri(const std::string &uri, const core::ubuntu::media::Player::HeadersType &headers, bool do_pipeline_reset=true)
Definition: playbin.cpp:371
core::ubuntu::media::video::Dimensions get_video_dimensions() const
Definition: playbin.cpp:490
GstMessageType type
Definition: bus.h:181
bool seek(const std::chrono::microseconds &ms)
Definition: playbin.cpp:480
uint64_t duration() const
Definition: playbin.cpp:362
void reset_pipeline()
Definition: playbin.cpp:154
static const std::string & pipeline_name()
Definition: playbin.cpp:68
void set_lifetime(core::ubuntu::media::Player::Lifetime)
Definition: playbin.cpp:337
struct gstreamer::Playbin::@12 signals
void set_audio_stream_role(core::ubuntu::media::Player::AudioStreamRole new_audio_role)
Definition: playbin.cpp:316
std::map< std::string, std::string > HeadersType
Definition: player.h:49
core::ubuntu::media::Player::Orientation orientation_lut(const gchar *orientation)
Definition: playbin.cpp:301
gstreamer::Bus & message_bus()
Definition: playbin.cpp:218
core::Signal< void > about_to_finish
Definition: playbin.h:126
std::string uri() const
Definition: playbin.cpp:410
void setup_pipeline_for_audio_video()
Definition: playbin.cpp:223
union gstreamer::Bus::Message::Detail detail
GstElement * pipeline
Definition: playbin.h:112
std::string get_file_content_type(const std::string &uri) const
Definition: playbin.cpp:510
struct gstreamer::Bus::Message::Detail::Tag tag
boost::flyweight< std::string > source
Definition: bus.h:182
bool is_audio_file(const std::string &uri) const
Definition: playbin.cpp:551
bool is_video_file(const std::string &uri) const
Definition: playbin.cpp:565
struct gstreamer::Bus::Message::Detail::ErrorWarningInfo error_warning_info
void set_volume(double new_volume)
Definition: playbin.cpp:273
void on_new_message_async(const Bus::Message &message)
Definition: playbin.cpp:173
void create_video_sink(uint32_t texture_id)
Definition: playbin.cpp:263
uint64_t position() const
Definition: playbin.cpp:342
IntWrapper is a type-safe integer that allows for encoding/enforcing semantics by means of tags...
Definition: dimensions.h:68
MediaFileType media_file_type() const
Definition: playbin.cpp:579
struct gstreamer::Bus::Message::Detail::StateChanged state_changed