Music Hub  ..
A session-wide music playback service
playbin.cpp
Go to the documentation of this file.
1 /*
2  * Copyright © 2013 Canonical Ltd.
3  *
4  * This program is free software: you can redistribute it and/or modify it
5  * under the terms of the GNU Lesser General Public License version 3,
6  * as published by the Free Software Foundation.
7  *
8  * This program is distributed in the hope that it will be useful,
9  * but WITHOUT ANY WARRANTY; without even the implied warranty of
10  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11  * GNU Lesser General Public License for more details.
12  *
13  * You should have received a copy of the GNU Lesser General Public License
14  * along with this program. If not, see <http://www.gnu.org/licenses/>.
15  *
16  * Authored by: Thomas Voß <thomas.voss@canonical.com>
17  */
18 
20 
22 
23 #if defined(MEDIA_HUB_HAVE_HYBRIS_MEDIA_COMPAT_LAYER)
24 #include <hybris/media/surface_texture_client_hybris.h>
25 #include <hybris/media/media_codec_layer.h>
26 
27 #include <utility>
28 
29 namespace
30 {
31 void setup_video_sink_for_buffer_streaming(GstElement* video_sink)
32 {
33  // Get the service-side BufferQueue (IGraphicBufferProducer) and associate it with
34  // the SurfaceTextureClientHybris instance
35  IGBPWrapperHybris igbp = decoding_service_get_igraphicbufferproducer();
36  SurfaceTextureClientHybris stc = surface_texture_client_create_by_igbp(igbp);
37  // Because mirsink is being loaded, we are definitely doing * hardware rendering.
38  surface_texture_client_set_hardware_rendering (stc, TRUE);
39  g_object_set (G_OBJECT (video_sink), "surface", static_cast<gpointer>(stc), static_cast<char*>(NULL));
40 }
41 }
42 #else // MEDIA_HUB_HAVE_HYBRIS_MEDIA_COMPAT_LAYER
43 namespace
44 {
45 void setup_video_sink_for_buffer_streaming(GstElement*)
46 {
47  throw core::ubuntu::media::Player::Error::OutOfProcessBufferStreamingNotSupported{};
48 }
49 }
50 #endif // MEDIA_HUB_HAVE_HYBRIS_MEDIA_COMPAT_LAYER
51 
52 namespace
53 {
54 bool is_mir_video_sink()
55 {
56  return g_strcmp0(::getenv("CORE_UBUNTU_MEDIA_SERVICE_VIDEO_SINK_NAME"), "mirsink") == 0;
57 }
58 }
59 // Uncomment to generate a dot file at the time that the pipeline
60 // goes to the PLAYING state. Make sure to export GST_DEBUG_DUMP_DOT_DIR
61 // before starting media-hub-server. To convert the dot file to something
62 // other image format, use: dot pipeline.dot -Tpng -o pipeline.png
63 //#define DEBUG_GST_PIPELINE
64 
65 namespace media = core::ubuntu::media;
67 
69 {
70  static const std::string s{"playbin"};
71  return s;
72 }
73 
74 void gstreamer::Playbin::about_to_finish(GstElement*, gpointer user_data)
75 {
76  auto thiz = static_cast<Playbin*>(user_data);
77  thiz->signals.about_to_finish();
78 }
79 
81  GstElement *source,
82  gpointer user_data)
83 {
84  if (user_data == nullptr)
85  return;
86 
87  static_cast<Playbin*>(user_data)->setup_source(source);
88 }
89 
91  : pipeline(gst_element_factory_make("playbin", pipeline_name().c_str())),
92  bus{gst_element_get_bus(pipeline)},
93  file_type(MEDIA_FILE_TYPE_NONE),
94  video_sink(nullptr),
95  on_new_message_connection_async(
96  bus.on_new_message_async.connect(
97  std::bind(
99  this,
100  std::placeholders::_1))),
101  is_seeking(false),
102  previous_position(0),
103  cached_video_dimensions{
106  player_lifetime(media::Player::Lifetime::normal),
107  about_to_finish_handler_id(0),
108  source_setup_handler_id(0)
109 {
110  if (!pipeline)
111  throw std::runtime_error("Could not create pipeline for playbin.");
112 
113  // Add audio and/or video sink elements depending on environment variables
114  // being set or not set
115  setup_pipeline_for_audio_video();
116 
117  about_to_finish_handler_id = g_signal_connect(
118  pipeline,
119  "about-to-finish",
120  G_CALLBACK(about_to_finish),
121  this
122  );
123 
124  source_setup_handler_id = g_signal_connect(
125  pipeline,
126  "source-setup",
127  G_CALLBACK(source_setup),
128  this
129  );
130 }
131 
133 {
134  g_signal_handler_disconnect(pipeline, about_to_finish_handler_id);
135  g_signal_handler_disconnect(pipeline, source_setup_handler_id);
136 
137  if (pipeline)
138  gst_object_unref(pipeline);
139 }
140 
142 {
143  std::cout << "Client died, resetting pipeline" << std::endl;
144  // When the client dies, tear down the current pipeline and get it
145  // in a state that is ready for the next client that connects to the
146  // service
147 
148  // Don't reset the pipeline if we want to resume
149  if (player_lifetime != media::Player::Lifetime::resumable) {
150  reset_pipeline();
151  }
152  // Signal to the Player class that the client side has disconnected
153  signals.client_disconnected();
154 }
155 
157 {
158  std::cout << __PRETTY_FUNCTION__ << std::endl;
159  auto ret = gst_element_set_state(pipeline, GST_STATE_NULL);
160  switch(ret)
161  {
162  case GST_STATE_CHANGE_FAILURE:
163  std::cout << "Failed to reset the pipeline state. Client reconnect may not function properly." << std::endl;
164  break;
165  case GST_STATE_CHANGE_NO_PREROLL:
166  case GST_STATE_CHANGE_SUCCESS:
167  case GST_STATE_CHANGE_ASYNC:
168  break;
169  default:
170  std::cout << "Failed to reset the pipeline state. Client reconnect may not function properly." << std::endl;
171  }
172  file_type = MEDIA_FILE_TYPE_NONE;
173 }
174 
176 {
177  switch(message.type)
178  {
179  case GST_MESSAGE_ERROR:
180  signals.on_error(message.detail.error_warning_info);
181  break;
182  case GST_MESSAGE_WARNING:
183  signals.on_warning(message.detail.error_warning_info);
184  break;
185  case GST_MESSAGE_INFO:
186  signals.on_info(message.detail.error_warning_info);
187  break;
188  case GST_MESSAGE_STATE_CHANGED:
189  signals.on_state_changed(std::make_pair(message.detail.state_changed, message.source));
190  break;
191  case GST_MESSAGE_TAG:
192  {
193  gchar *orientation;
194  if (gst_tag_list_get_string(message.detail.tag.tag_list, "image-orientation", &orientation))
195  {
196  // If the image-orientation tag is in the GstTagList, signal the Engine
197  signals.on_orientation_changed(orientation_lut(orientation));
198  g_free (orientation);
199  }
200 
201  signals.on_tag_available(message.detail.tag);
202  }
203  break;
204  case GST_MESSAGE_ASYNC_DONE:
205  if (is_seeking)
206  {
207  // FIXME: Pass the actual playback time position to the signal call
208  signals.on_seeked_to(0);
209  is_seeking = false;
210  }
211  break;
212  case GST_MESSAGE_EOS:
213  signals.on_end_of_stream();
214  default:
215  break;
216  }
217 }
218 
220 {
221  return bus;
222 }
223 
225 {
226  gint flags;
227  g_object_get (pipeline, "flags", &flags, nullptr);
228  flags |= GST_PLAY_FLAG_AUDIO;
229  flags |= GST_PLAY_FLAG_VIDEO;
230  flags &= ~GST_PLAY_FLAG_TEXT;
231  g_object_set (pipeline, "flags", flags, nullptr);
232 
233  if (::getenv("CORE_UBUNTU_MEDIA_SERVICE_AUDIO_SINK_NAME") != nullptr)
234  {
235  auto audio_sink = gst_element_factory_make (
236  ::getenv("CORE_UBUNTU_MEDIA_SERVICE_AUDIO_SINK_NAME"),
237  "audio-sink");
238 
239  std::cout << "audio_sink: " << ::getenv("CORE_UBUNTU_MEDIA_SERVICE_AUDIO_SINK_NAME") << std::endl;
240 
241  g_object_set (
242  pipeline,
243  "audio-sink",
244  audio_sink,
245  NULL);
246  }
247 
248  if (::getenv("CORE_UBUNTU_MEDIA_SERVICE_VIDEO_SINK_NAME") != nullptr)
249  {
250  video_sink = gst_element_factory_make (
251  ::getenv("CORE_UBUNTU_MEDIA_SERVICE_VIDEO_SINK_NAME"),
252  "video-sink");
253 
254  std::cout << "video_sink: " << ::getenv("CORE_UBUNTU_MEDIA_SERVICE_VIDEO_SINK_NAME") << std::endl;
255 
256  g_object_set (
257  pipeline,
258  "video-sink",
259  video_sink,
260  NULL);
261  }
262 }
263 
265 {
266  if (not video_sink) throw std::logic_error
267  {
268  "No video sink configured for the current pipeline"
269  };
270 
271  setup_video_sink_for_buffer_streaming(video_sink);
272 }
273 
274 void gstreamer::Playbin::set_volume(double new_volume)
275 {
276  g_object_set (pipeline, "volume", new_volume, NULL);
277 }
278 
281 {
282  switch (audio_role)
283  {
284  case media::Player::AudioStreamRole::alarm:
285  return "alarm";
286  break;
287  case media::Player::AudioStreamRole::alert:
288  return "alert";
289  break;
290  case media::Player::AudioStreamRole::multimedia:
291  return "multimedia";
292  break;
293  case media::Player::AudioStreamRole::phone:
294  return "phone";
295  break;
296  default:
297  return "multimedia";
298  break;
299  }
300 }
301 
303 {
304  if (g_strcmp0(orientation, "rotate-0") == 0)
305  return media::Player::Orientation::rotate0;
306  else if (g_strcmp0(orientation, "rotate-90") == 0)
307  return media::Player::Orientation::rotate90;
308  else if (g_strcmp0(orientation, "rotate-180") == 0)
309  return media::Player::Orientation::rotate180;
310  else if (g_strcmp0(orientation, "rotate-270") == 0)
311  return media::Player::Orientation::rotate270;
312  else
313  return media::Player::Orientation::rotate0;
314 }
315 
318 {
319  GstElement *audio_sink = NULL;
320  g_object_get (pipeline, "audio-sink", &audio_sink, NULL);
321 
322  std::string role_str("props,media.role=" + get_audio_role_str(new_audio_role));
323  std::cout << "Audio stream role: " << role_str << std::endl;
324 
325  GstStructure *props = gst_structure_from_string (role_str.c_str(), NULL);
326  if (audio_sink != nullptr && props != nullptr)
327  g_object_set (audio_sink, "stream-properties", props, NULL);
328  else
329  {
330  std::cerr <<
331  "Warning: couldn't set audio stream role - couldn't get audio_sink from pipeline" <<
332  std::endl;
333  }
334 
335  gst_structure_free (props);
336 }
337 
339 {
340  player_lifetime = lifetime;
341 }
342 
344 {
345  int64_t pos = 0;
346  gst_element_query_position (pipeline, GST_FORMAT_TIME, &pos);
347 
348  // This prevents a 0 position from being reported to the app which happens while seeking.
349  // This is covering over a GStreamer issue
350  if ((static_cast<uint64_t>(pos) < duration()) && is_seeking && pos == 0)
351  {
352  return previous_position;
353  }
354 
355  // Save the current position to use just in case it's needed the next time position is
356  // requested
357  previous_position = static_cast<uint64_t>(pos);
358 
359  // FIXME: this should be int64_t, but dbus-cpp doesn't seem to handle it correctly
360  return static_cast<uint64_t>(pos);
361 }
362 
364 {
365  int64_t dur = 0;
366  gst_element_query_duration (pipeline, GST_FORMAT_TIME, &dur);
367 
368  // FIXME: this should be int64_t, but dbus-cpp doesn't seem to handle it correctly
369  return static_cast<uint64_t>(dur);
370 }
371 
373  const std::string& uri,
375  bool do_pipeline_reset)
376 {
377  if (do_pipeline_reset)
378  reset_pipeline();
379 
380  g_object_set(pipeline, "uri", uri.c_str(), NULL);
381  if (is_video_file(uri))
382  file_type = MEDIA_FILE_TYPE_VIDEO;
383  else if (is_audio_file(uri))
384  file_type = MEDIA_FILE_TYPE_AUDIO;
385 
386  request_headers = headers;
387 }
388 
389 void gstreamer::Playbin::setup_source(GstElement *source)
390 {
391  if (source == NULL || request_headers.empty())
392  return;
393 
394  if (request_headers.find("Cookie") != request_headers.end()) {
395  if (g_object_class_find_property(G_OBJECT_GET_CLASS(source),
396  "cookies") != NULL) {
397  gchar ** cookies = g_strsplit(request_headers["Cookie"].c_str(), ";", 0);
398  g_object_set(source, "cookies", cookies, NULL);
399  g_strfreev(cookies);
400  }
401  }
402 
403  if (request_headers.find("User-Agent") != request_headers.end()) {
404  if (g_object_class_find_property(G_OBJECT_GET_CLASS(source),
405  "user-agent") != NULL) {
406  g_object_set(source, "user-agent", request_headers["User-Agent"].c_str(), NULL);
407  }
408  }
409 }
410 
411 std::string gstreamer::Playbin::uri() const
412 {
413  gchar* data = nullptr;
414  g_object_get(pipeline, "current-uri", &data, nullptr);
415 
416  std::string result((data == nullptr ? "" : data));
417  g_free(data);
418 
419  return result;
420 }
421 
423 {
424  static const std::chrono::nanoseconds state_change_timeout
425  {
426  // We choose a quite high value here as tests are run under valgrind
427  // and gstreamer pipeline setup/state changes take longer in that scenario.
428  // The value does not negatively impact runtime performance.
429  std::chrono::milliseconds{5000}
430  };
431 
432  auto ret = gst_element_set_state(pipeline, new_state);
433 
434  std::cout << __PRETTY_FUNCTION__ << ": requested state change." << std::endl;
435 
436  bool result = false; GstState current, pending;
437  switch(ret)
438  {
439  case GST_STATE_CHANGE_FAILURE:
440  result = false; break;
441  case GST_STATE_CHANGE_NO_PREROLL:
442  case GST_STATE_CHANGE_SUCCESS:
443  result = true; break;
444  case GST_STATE_CHANGE_ASYNC:
445  result = GST_STATE_CHANGE_SUCCESS == gst_element_get_state(
446  pipeline,
447  &current,
448  &pending,
449  state_change_timeout.count());
450  break;
451  }
452 
453  // We only should query the pipeline if we actually succeeded in
454  // setting the requested state.
455  if (result && new_state == GST_STATE_PLAYING)
456  {
457  // Get the video height/width from the video sink
458  try
459  {
460  const core::ubuntu::media::video::Dimensions new_dimensions = get_video_dimensions();
461  emit_video_dimensions_changed_if_changed(new_dimensions);
462  cached_video_dimensions = new_dimensions;
463  }
464  catch (const std::exception& e)
465  {
466  std::cerr << "Problem querying video dimensions: " << e.what() << std::endl;
467  }
468  catch (...)
469  {
470  std::cerr << "Problem querying video dimensions." << std::endl;
471  }
472 
473 #ifdef DEBUG_GST_PIPELINE
474  std::cout << "Dumping pipeline dot file" << std::endl;
475  GST_DEBUG_BIN_TO_DOT_FILE((GstBin*)pipeline, GST_DEBUG_GRAPH_SHOW_ALL, "pipeline");
476 #endif
477  }
478 
479  return result;
480 }
481 
482 bool gstreamer::Playbin::seek(const std::chrono::microseconds& ms)
483 {
484  is_seeking = true;
485  return gst_element_seek_simple(
486  pipeline,
487  GST_FORMAT_TIME,
488  (GstSeekFlags)(GST_SEEK_FLAG_FLUSH | GST_SEEK_FLAG_KEY_UNIT),
489  ms.count() * 1000);
490 }
491 
493 {
494  if (not video_sink || not is_mir_video_sink())
495  throw std::runtime_error
496  {
497  "Missing video sink or video sink does not support query of width and height."
498  };
499 
500  // Initialize to default value prior to querying actual values from the sink.
501  uint32_t video_width = 0, video_height = 0;
502  g_object_get (video_sink, "height", &video_height, nullptr);
503  g_object_get (video_sink, "width", &video_width, nullptr);
504 
505  // TODO(tvoss): We should probably check here if width and height are valid.
507  {
510  };
511 }
512 
514 {
515  // Only signal the application layer if the dimensions have in fact changed. This might happen
516  // if reusing the same media-hub session to play two different video sources.
517  if (new_dimensions != cached_video_dimensions)
518  signals.on_video_dimensions_changed(new_dimensions);
519 }
520 
521 std::string gstreamer::Playbin::get_file_content_type(const std::string& uri) const
522 {
523  if (uri.empty())
524  return std::string();
525 
526  std::string filename(uri);
527  size_t pos = uri.find("file://");
528  if (pos != std::string::npos)
529  filename = uri.substr(pos + 7, std::string::npos);
530  else
531  // Anything other than a file, for now claim that the type
532  // is both audio and video.
533  // FIXME: implement true net stream sampling and get the type from GstCaps
534  return std::string("audio/video/");
535 
536 
537  GError *error = nullptr;
538  std::unique_ptr<GFile, void(*)(void *)> file(
539  g_file_new_for_path(filename.c_str()), g_object_unref);
540  std::unique_ptr<GFileInfo, void(*)(void *)> info(
541  g_file_query_info(
542  file.get(), G_FILE_ATTRIBUTE_STANDARD_FAST_CONTENT_TYPE ","
543  G_FILE_ATTRIBUTE_ETAG_VALUE, G_FILE_QUERY_INFO_NONE,
544  /* cancellable */ NULL, &error),
545  g_object_unref);
546  if (!info)
547  {
548  std::string error_str(error->message);
549  g_error_free(error);
550 
551  std::cout << "Failed to query the URI for the presence of video content: "
552  << error_str << std::endl;
553  return std::string();
554  }
555 
556  std::string content_type(g_file_info_get_attribute_string(
557  info.get(), G_FILE_ATTRIBUTE_STANDARD_FAST_CONTENT_TYPE));
558 
559  return content_type;
560 }
561 
562 bool gstreamer::Playbin::is_audio_file(const std::string& uri) const
563 {
564  if (uri.empty())
565  return false;
566 
567  if (get_file_content_type(uri).find("audio/") == 0)
568  {
569  std::cout << "Found audio content" << std::endl;
570  return true;
571  }
572 
573  return false;
574 }
575 
576 bool gstreamer::Playbin::is_video_file(const std::string& uri) const
577 {
578  if (uri.empty())
579  return false;
580 
581  if (get_file_content_type(uri).find("video/") == 0)
582  {
583  std::cout << "Found video content" << std::endl;
584  return true;
585  }
586 
587  return false;
588 }
589 
591 {
592  return file_type;
593 }
static std::string get_audio_role_str(core::ubuntu::media::Player::AudioStreamRole audio_role)
Definition: playbin.cpp:280
bool set_state_and_wait(GstState new_state)
Definition: playbin.cpp:422
static void source_setup(GstElement *, GstElement *source, gpointer user_data)
Definition: playbin.cpp:80
void setup_source(GstElement *source)
Definition: playbin.cpp:389
std::tuple< Height, Width > Dimensions
Height and Width of a video.
Definition: dimensions.h:139
void set_uri(const std::string &uri, const core::ubuntu::media::Player::HeadersType &headers, bool do_pipeline_reset=true)
Definition: playbin.cpp:372
core::ubuntu::media::video::Dimensions get_video_dimensions() const
Definition: playbin.cpp:492
GstMessageType type
Definition: bus.h:181
bool seek(const std::chrono::microseconds &ms)
Definition: playbin.cpp:482
uint64_t duration() const
Definition: playbin.cpp:363
void reset_pipeline()
Definition: playbin.cpp:156
static const std::string & pipeline_name()
Definition: playbin.cpp:68
void emit_video_dimensions_changed_if_changed(const core::ubuntu::media::video::Dimensions &new_dimensions)
Definition: playbin.cpp:513
void set_lifetime(core::ubuntu::media::Player::Lifetime)
Definition: playbin.cpp:338
struct gstreamer::Playbin::@12 signals
void set_audio_stream_role(core::ubuntu::media::Player::AudioStreamRole new_audio_role)
Definition: playbin.cpp:317
std::map< std::string, std::string > HeadersType
Definition: player.h:49
core::ubuntu::media::Player::Orientation orientation_lut(const gchar *orientation)
Definition: playbin.cpp:302
gstreamer::Bus & message_bus()
Definition: playbin.cpp:219
core::Signal< void > about_to_finish
Definition: playbin.h:127
std::string uri() const
Definition: playbin.cpp:411
void setup_pipeline_for_audio_video()
Definition: playbin.cpp:224
union gstreamer::Bus::Message::Detail detail
GstElement * pipeline
Definition: playbin.h:113
std::string get_file_content_type(const std::string &uri) const
Definition: playbin.cpp:521
struct gstreamer::Bus::Message::Detail::Tag tag
boost::flyweight< std::string > source
Definition: bus.h:182
bool is_audio_file(const std::string &uri) const
Definition: playbin.cpp:562
bool is_video_file(const std::string &uri) const
Definition: playbin.cpp:576
struct gstreamer::Bus::Message::Detail::ErrorWarningInfo error_warning_info
void set_volume(double new_volume)
Definition: playbin.cpp:274
void on_new_message_async(const Bus::Message &message)
Definition: playbin.cpp:175
void create_video_sink(uint32_t texture_id)
Definition: playbin.cpp:264
uint64_t position() const
Definition: playbin.cpp:343
IntWrapper is a type-safe integer that allows for encoding/enforcing semantics by means of tags...
Definition: dimensions.h:68
MediaFileType media_file_type() const
Definition: playbin.cpp:590
struct gstreamer::Bus::Message::Detail::StateChanged state_changed