Music Hub  ..
A session-wide music playback service
playbin.cpp
Go to the documentation of this file.
1 /*
2  * Copyright © 2013 Canonical Ltd.
3  *
4  * This program is free software: you can redistribute it and/or modify it
5  * under the terms of the GNU Lesser General Public License version 3,
6  * as published by the Free Software Foundation.
7  *
8  * This program is distributed in the hope that it will be useful,
9  * but WITHOUT ANY WARRANTY; without even the implied warranty of
10  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11  * GNU Lesser General Public License for more details.
12  *
13  * You should have received a copy of the GNU Lesser General Public License
14  * along with this program. If not, see <http://www.gnu.org/licenses/>.
15  *
16  * Authored by: Thomas Voß <thomas.voss@canonical.com>
17  */
18 
20 
22 
23 #if defined(MEDIA_HUB_HAVE_HYBRIS_MEDIA_COMPAT_LAYER)
24 #include <hybris/media/surface_texture_client_hybris.h>
25 #include <hybris/media/media_codec_layer.h>
26 
27 #include <utility>
28 
29 namespace
30 {
31 void setup_video_sink_for_buffer_streaming(GstElement* pipeline)
32 {
33  // Get the service-side BufferQueue (IGraphicBufferProducer) and associate it with
34  // the SurfaceTextureClientHybris instance
35  IGBPWrapperHybris igbp = decoding_service_get_igraphicbufferproducer();
36  SurfaceTextureClientHybris stc = surface_texture_client_create_by_igbp(igbp);
37 
38  // Because mirsink is being loaded, we are definitely doing * hardware rendering.
39  surface_texture_client_set_hardware_rendering(stc, TRUE);
40 
41  GstContext *context = gst_context_new("gst.mir.MirContext", TRUE);
42  GstStructure *structure = gst_context_writable_structure(context);
43  gst_structure_set(structure, "gst_mir_context", G_TYPE_POINTER, stc, NULL);
44 
45  /* Propagate context in pipeline (needed by amchybris and mirsink) */
46  gst_element_set_context(pipeline, context);
47 }
48 }
49 #else // MEDIA_HUB_HAVE_HYBRIS_MEDIA_COMPAT_LAYER
50 namespace
51 {
52 void setup_video_sink_for_buffer_streaming(GstElement*)
53 {
55 }
56 }
57 #endif // MEDIA_HUB_HAVE_HYBRIS_MEDIA_COMPAT_LAYER
58 
59 namespace
60 {
61 bool is_mir_video_sink()
62 {
63  return g_strcmp0(::getenv("CORE_UBUNTU_MEDIA_SERVICE_VIDEO_SINK_NAME"), "mirsink") == 0;
64 }
65 }
66 // Uncomment to generate a dot file at the time that the pipeline
67 // goes to the PLAYING state. Make sure to export GST_DEBUG_DUMP_DOT_DIR
68 // before starting media-hub-server. To convert the dot file to something
69 // other image format, use: dot pipeline.dot -Tpng -o pipeline.png
70 //#define DEBUG_GST_PIPELINE
71 
72 namespace media = core::ubuntu::media;
74 
76 {
77  static const std::string s{"playbin"};
78  return s;
79 }
80 
81 void gstreamer::Playbin::about_to_finish(GstElement*, gpointer user_data)
82 {
83  auto thiz = static_cast<Playbin*>(user_data);
84  thiz->signals.about_to_finish();
85 }
86 
88  GstElement *source,
89  gpointer user_data)
90 {
91  if (user_data == nullptr)
92  return;
93 
94  static_cast<Playbin*>(user_data)->setup_source(source);
95 }
96 
98  : pipeline(gst_element_factory_make("playbin", pipeline_name().c_str())),
99  bus{gst_element_get_bus(pipeline)},
100  file_type(MEDIA_FILE_TYPE_NONE),
101  video_sink(nullptr),
102  on_new_message_connection_async(
103  bus.on_new_message_async.connect(
104  std::bind(
106  this,
107  std::placeholders::_1))),
108  is_seeking(false),
109  previous_position(0),
110  cached_video_dimensions{
113  player_lifetime(media::Player::Lifetime::normal),
114  about_to_finish_handler_id(0),
115  source_setup_handler_id(0)
116 {
117  if (!pipeline)
118  throw std::runtime_error("Could not create pipeline for playbin.");
119 
120  // Add audio and/or video sink elements depending on environment variables
121  // being set or not set
122  setup_pipeline_for_audio_video();
123 
124  about_to_finish_handler_id = g_signal_connect(
125  pipeline,
126  "about-to-finish",
127  G_CALLBACK(about_to_finish),
128  this
129  );
130 
131  source_setup_handler_id = g_signal_connect(
132  pipeline,
133  "source-setup",
134  G_CALLBACK(source_setup),
135  this
136  );
137 }
138 
140 {
141  g_signal_handler_disconnect(pipeline, about_to_finish_handler_id);
142  g_signal_handler_disconnect(pipeline, source_setup_handler_id);
143 
144  if (pipeline)
145  gst_object_unref(pipeline);
146 }
147 
149 {
150  std::cout << "Client died, resetting pipeline" << std::endl;
151  // When the client dies, tear down the current pipeline and get it
152  // in a state that is ready for the next client that connects to the
153  // service
154 
155  // Don't reset the pipeline if we want to resume
156  if (player_lifetime != media::Player::Lifetime::resumable) {
157  reset_pipeline();
158  }
159  // Signal to the Player class that the client side has disconnected
160  signals.client_disconnected();
161 }
162 
164 {
165  std::cout << __PRETTY_FUNCTION__ << std::endl;
166  auto ret = gst_element_set_state(pipeline, GST_STATE_NULL);
167  switch(ret)
168  {
169  case GST_STATE_CHANGE_FAILURE:
170  std::cout << "Failed to reset the pipeline state. Client reconnect may not function properly." << std::endl;
171  break;
172  case GST_STATE_CHANGE_NO_PREROLL:
173  case GST_STATE_CHANGE_SUCCESS:
174  case GST_STATE_CHANGE_ASYNC:
175  break;
176  default:
177  std::cout << "Failed to reset the pipeline state. Client reconnect may not function properly." << std::endl;
178  }
179  file_type = MEDIA_FILE_TYPE_NONE;
180 }
181 
183 {
184  switch(message.type)
185  {
186  case GST_MESSAGE_ERROR:
187  signals.on_error(message.detail.error_warning_info);
188  break;
189  case GST_MESSAGE_WARNING:
190  signals.on_warning(message.detail.error_warning_info);
191  break;
192  case GST_MESSAGE_INFO:
193  signals.on_info(message.detail.error_warning_info);
194  break;
195  case GST_MESSAGE_STATE_CHANGED:
196  signals.on_state_changed(std::make_pair(message.detail.state_changed, message.source));
197  break;
198  case GST_MESSAGE_TAG:
199  {
200  gchar *orientation;
201  if (gst_tag_list_get_string(message.detail.tag.tag_list, "image-orientation", &orientation))
202  {
203  // If the image-orientation tag is in the GstTagList, signal the Engine
204  signals.on_orientation_changed(orientation_lut(orientation));
205  g_free (orientation);
206  }
207 
208  signals.on_tag_available(message.detail.tag);
209  }
210  break;
211  case GST_MESSAGE_ASYNC_DONE:
212  if (is_seeking)
213  {
214  // FIXME: Pass the actual playback time position to the signal call
215  signals.on_seeked_to(0);
216  is_seeking = false;
217  }
218  break;
219  case GST_MESSAGE_EOS:
220  signals.on_end_of_stream();
221  default:
222  break;
223  }
224 }
225 
227 {
228  return bus;
229 }
230 
232 {
233  gint flags;
234  g_object_get (pipeline, "flags", &flags, nullptr);
235  flags |= GST_PLAY_FLAG_AUDIO;
236  flags |= GST_PLAY_FLAG_VIDEO;
237  flags &= ~GST_PLAY_FLAG_TEXT;
238  g_object_set (pipeline, "flags", flags, nullptr);
239 
240  if (::getenv("CORE_UBUNTU_MEDIA_SERVICE_AUDIO_SINK_NAME") != nullptr)
241  {
242  auto audio_sink = gst_element_factory_make (
243  ::getenv("CORE_UBUNTU_MEDIA_SERVICE_AUDIO_SINK_NAME"),
244  "audio-sink");
245 
246  std::cout << "audio_sink: " << ::getenv("CORE_UBUNTU_MEDIA_SERVICE_AUDIO_SINK_NAME") << std::endl;
247 
248  g_object_set (
249  pipeline,
250  "audio-sink",
251  audio_sink,
252  NULL);
253  }
254 
255  if (::getenv("CORE_UBUNTU_MEDIA_SERVICE_VIDEO_SINK_NAME") != nullptr)
256  {
257  video_sink = gst_element_factory_make (
258  ::getenv("CORE_UBUNTU_MEDIA_SERVICE_VIDEO_SINK_NAME"),
259  "video-sink");
260 
261  std::cout << "video_sink: " << ::getenv("CORE_UBUNTU_MEDIA_SERVICE_VIDEO_SINK_NAME") << std::endl;
262 
263  g_object_set (
264  pipeline,
265  "video-sink",
266  video_sink,
267  NULL);
268  }
269 }
270 
272 {
273  if (not video_sink) throw std::logic_error
274  {
275  "No video sink configured for the current pipeline"
276  };
277 
278  setup_video_sink_for_buffer_streaming(pipeline);
279 }
280 
281 void gstreamer::Playbin::set_volume(double new_volume)
282 {
283  g_object_set (pipeline, "volume", new_volume, NULL);
284 }
285 
288 {
289  switch (audio_role)
290  {
291  case media::Player::AudioStreamRole::alarm:
292  return "alarm";
293  break;
294  case media::Player::AudioStreamRole::alert:
295  return "alert";
296  break;
297  case media::Player::AudioStreamRole::multimedia:
298  return "multimedia";
299  break;
300  case media::Player::AudioStreamRole::phone:
301  return "phone";
302  break;
303  default:
304  return "multimedia";
305  break;
306  }
307 }
308 
310 {
311  if (g_strcmp0(orientation, "rotate-0") == 0)
312  return media::Player::Orientation::rotate0;
313  else if (g_strcmp0(orientation, "rotate-90") == 0)
314  return media::Player::Orientation::rotate90;
315  else if (g_strcmp0(orientation, "rotate-180") == 0)
316  return media::Player::Orientation::rotate180;
317  else if (g_strcmp0(orientation, "rotate-270") == 0)
318  return media::Player::Orientation::rotate270;
319  else
320  return media::Player::Orientation::rotate0;
321 }
322 
325 {
326  GstElement *audio_sink = NULL;
327  g_object_get (pipeline, "audio-sink", &audio_sink, NULL);
328 
329  std::string role_str("props,media.role=" + get_audio_role_str(new_audio_role));
330  std::cout << "Audio stream role: " << role_str << std::endl;
331 
332  GstStructure *props = gst_structure_from_string (role_str.c_str(), NULL);
333  if (audio_sink != nullptr && props != nullptr)
334  g_object_set (audio_sink, "stream-properties", props, NULL);
335  else
336  {
337  std::cerr <<
338  "Warning: couldn't set audio stream role - couldn't get audio_sink from pipeline" <<
339  std::endl;
340  }
341 
342  gst_structure_free (props);
343 }
344 
346 {
347  player_lifetime = lifetime;
348 }
349 
351 {
352  int64_t pos = 0;
353  gst_element_query_position (pipeline, GST_FORMAT_TIME, &pos);
354 
355  // This prevents a 0 position from being reported to the app which happens while seeking.
356  // This is covering over a GStreamer issue
357  if ((static_cast<uint64_t>(pos) < duration()) && is_seeking && pos == 0)
358  {
359  return previous_position;
360  }
361 
362  // Save the current position to use just in case it's needed the next time position is
363  // requested
364  previous_position = static_cast<uint64_t>(pos);
365 
366  // FIXME: this should be int64_t, but dbus-cpp doesn't seem to handle it correctly
367  return static_cast<uint64_t>(pos);
368 }
369 
371 {
372  int64_t dur = 0;
373  gst_element_query_duration (pipeline, GST_FORMAT_TIME, &dur);
374 
375  // FIXME: this should be int64_t, but dbus-cpp doesn't seem to handle it correctly
376  return static_cast<uint64_t>(dur);
377 }
378 
380  const std::string& uri,
382  bool do_pipeline_reset)
383 {
384  if (do_pipeline_reset)
385  reset_pipeline();
386 
387  g_object_set(pipeline, "uri", uri.c_str(), NULL);
388  if (is_video_file(uri))
389  file_type = MEDIA_FILE_TYPE_VIDEO;
390  else if (is_audio_file(uri))
391  file_type = MEDIA_FILE_TYPE_AUDIO;
392 
393  request_headers = headers;
394 }
395 
396 void gstreamer::Playbin::setup_source(GstElement *source)
397 {
398  if (source == NULL || request_headers.empty())
399  return;
400 
401  if (request_headers.find("Cookie") != request_headers.end()) {
402  if (g_object_class_find_property(G_OBJECT_GET_CLASS(source),
403  "cookies") != NULL) {
404  gchar ** cookies = g_strsplit(request_headers["Cookie"].c_str(), ";", 0);
405  g_object_set(source, "cookies", cookies, NULL);
406  g_strfreev(cookies);
407  }
408  }
409 
410  if (request_headers.find("User-Agent") != request_headers.end()) {
411  if (g_object_class_find_property(G_OBJECT_GET_CLASS(source),
412  "user-agent") != NULL) {
413  g_object_set(source, "user-agent", request_headers["User-Agent"].c_str(), NULL);
414  }
415  }
416 }
417 
418 std::string gstreamer::Playbin::uri() const
419 {
420  gchar* data = nullptr;
421  g_object_get(pipeline, "current-uri", &data, nullptr);
422 
423  std::string result((data == nullptr ? "" : data));
424  g_free(data);
425 
426  return result;
427 }
428 
430 {
431  static const std::chrono::nanoseconds state_change_timeout
432  {
433  // We choose a quite high value here as tests are run under valgrind
434  // and gstreamer pipeline setup/state changes take longer in that scenario.
435  // The value does not negatively impact runtime performance.
436  std::chrono::milliseconds{5000}
437  };
438 
439  auto ret = gst_element_set_state(pipeline, new_state);
440 
441  std::cout << __PRETTY_FUNCTION__ << ": requested state change." << std::endl;
442 
443  bool result = false; GstState current, pending;
444  switch(ret)
445  {
446  case GST_STATE_CHANGE_FAILURE:
447  result = false; break;
448  case GST_STATE_CHANGE_NO_PREROLL:
449  case GST_STATE_CHANGE_SUCCESS:
450  result = true; break;
451  case GST_STATE_CHANGE_ASYNC:
452  result = GST_STATE_CHANGE_SUCCESS == gst_element_get_state(
453  pipeline,
454  &current,
455  &pending,
456  state_change_timeout.count());
457  break;
458  }
459 
460  // We only should query the pipeline if we actually succeeded in
461  // setting the requested state.
462  if (result && new_state == GST_STATE_PLAYING)
463  {
464  // Get the video height/width from the video sink
465  try
466  {
467  const core::ubuntu::media::video::Dimensions new_dimensions = get_video_dimensions();
468  emit_video_dimensions_changed_if_changed(new_dimensions);
469  cached_video_dimensions = new_dimensions;
470  }
471  catch (const std::exception& e)
472  {
473  std::cerr << "Problem querying video dimensions: " << e.what() << std::endl;
474  }
475  catch (...)
476  {
477  std::cerr << "Problem querying video dimensions." << std::endl;
478  }
479 
480 #ifdef DEBUG_GST_PIPELINE
481  std::cout << "Dumping pipeline dot file" << std::endl;
482  GST_DEBUG_BIN_TO_DOT_FILE((GstBin*)pipeline, GST_DEBUG_GRAPH_SHOW_ALL, "pipeline");
483 #endif
484  }
485 
486  return result;
487 }
488 
489 bool gstreamer::Playbin::seek(const std::chrono::microseconds& ms)
490 {
491  is_seeking = true;
492  return gst_element_seek_simple(
493  pipeline,
494  GST_FORMAT_TIME,
495  (GstSeekFlags)(GST_SEEK_FLAG_FLUSH | GST_SEEK_FLAG_KEY_UNIT),
496  ms.count() * 1000);
497 }
498 
500 {
501  if (not video_sink || not is_mir_video_sink())
502  throw std::runtime_error
503  {
504  "Missing video sink or video sink does not support query of width and height."
505  };
506 
507  // Initialize to default value prior to querying actual values from the sink.
508  uint32_t video_width = 0, video_height = 0;
509  g_object_get (video_sink, "height", &video_height, nullptr);
510  g_object_get (video_sink, "width", &video_width, nullptr);
511 
512  // TODO(tvoss): We should probably check here if width and height are valid.
514  {
517  };
518 }
519 
521 {
522  // Only signal the application layer if the dimensions have in fact changed. This might happen
523  // if reusing the same media-hub session to play two different video sources.
524  if (new_dimensions != cached_video_dimensions)
525  signals.on_video_dimensions_changed(new_dimensions);
526 }
527 
528 std::string gstreamer::Playbin::get_file_content_type(const std::string& uri) const
529 {
530  if (uri.empty())
531  return std::string();
532 
533  std::string filename(uri);
534  size_t pos = uri.find("file://");
535  if (pos != std::string::npos)
536  filename = uri.substr(pos + 7, std::string::npos);
537  else
538  // Anything other than a file, for now claim that the type
539  // is both audio and video.
540  // FIXME: implement true net stream sampling and get the type from GstCaps
541  return std::string("audio/video/");
542 
543 
544  GError *error = nullptr;
545  std::unique_ptr<GFile, void(*)(void *)> file(
546  g_file_new_for_path(filename.c_str()), g_object_unref);
547  std::unique_ptr<GFileInfo, void(*)(void *)> info(
548  g_file_query_info(
549  file.get(), G_FILE_ATTRIBUTE_STANDARD_FAST_CONTENT_TYPE ","
550  G_FILE_ATTRIBUTE_ETAG_VALUE, G_FILE_QUERY_INFO_NONE,
551  /* cancellable */ NULL, &error),
552  g_object_unref);
553  if (!info)
554  {
555  std::string error_str(error->message);
556  g_error_free(error);
557 
558  std::cout << "Failed to query the URI for the presence of video content: "
559  << error_str << std::endl;
560  return std::string();
561  }
562 
563  std::string content_type(g_file_info_get_attribute_string(
564  info.get(), G_FILE_ATTRIBUTE_STANDARD_FAST_CONTENT_TYPE));
565 
566  return content_type;
567 }
568 
569 bool gstreamer::Playbin::is_audio_file(const std::string& uri) const
570 {
571  if (uri.empty())
572  return false;
573 
574  if (get_file_content_type(uri).find("audio/") == 0)
575  {
576  std::cout << "Found audio content" << std::endl;
577  return true;
578  }
579 
580  return false;
581 }
582 
583 bool gstreamer::Playbin::is_video_file(const std::string& uri) const
584 {
585  if (uri.empty())
586  return false;
587 
588  if (get_file_content_type(uri).find("video/") == 0)
589  {
590  std::cout << "Found video content" << std::endl;
591  return true;
592  }
593 
594  return false;
595 }
596 
598 {
599  return file_type;
600 }
static std::string get_audio_role_str(core::ubuntu::media::Player::AudioStreamRole audio_role)
Definition: playbin.cpp:287
bool set_state_and_wait(GstState new_state)
Definition: playbin.cpp:429
static void source_setup(GstElement *, GstElement *source, gpointer user_data)
Definition: playbin.cpp:87
void setup_source(GstElement *source)
Definition: playbin.cpp:396
std::tuple< Height, Width > Dimensions
Height and Width of a video.
Definition: dimensions.h:139
void set_uri(const std::string &uri, const core::ubuntu::media::Player::HeadersType &headers, bool do_pipeline_reset=true)
Definition: playbin.cpp:379
core::ubuntu::media::video::Dimensions get_video_dimensions() const
Definition: playbin.cpp:499
GstMessageType type
Definition: bus.h:181
bool seek(const std::chrono::microseconds &ms)
Definition: playbin.cpp:489
uint64_t duration() const
Definition: playbin.cpp:370
void reset_pipeline()
Definition: playbin.cpp:163
static const std::string & pipeline_name()
Definition: playbin.cpp:75
void emit_video_dimensions_changed_if_changed(const core::ubuntu::media::video::Dimensions &new_dimensions)
Definition: playbin.cpp:520
void set_lifetime(core::ubuntu::media::Player::Lifetime)
Definition: playbin.cpp:345
struct gstreamer::Playbin::@12 signals
void set_audio_stream_role(core::ubuntu::media::Player::AudioStreamRole new_audio_role)
Definition: playbin.cpp:324
std::map< std::string, std::string > HeadersType
Definition: player.h:49
core::ubuntu::media::Player::Orientation orientation_lut(const gchar *orientation)
Definition: playbin.cpp:309
gstreamer::Bus & message_bus()
Definition: playbin.cpp:226
core::Signal< void > about_to_finish
Definition: playbin.h:127
std::string uri() const
Definition: playbin.cpp:418
void setup_pipeline_for_audio_video()
Definition: playbin.cpp:231
union gstreamer::Bus::Message::Detail detail
GstElement * pipeline
Definition: playbin.h:113
std::string get_file_content_type(const std::string &uri) const
Definition: playbin.cpp:528
struct gstreamer::Bus::Message::Detail::Tag tag
boost::flyweight< std::string > source
Definition: bus.h:182
bool is_audio_file(const std::string &uri) const
Definition: playbin.cpp:569
bool is_video_file(const std::string &uri) const
Definition: playbin.cpp:583
struct gstreamer::Bus::Message::Detail::ErrorWarningInfo error_warning_info
void set_volume(double new_volume)
Definition: playbin.cpp:281
void on_new_message_async(const Bus::Message &message)
Definition: playbin.cpp:182
void create_video_sink(uint32_t texture_id)
Definition: playbin.cpp:271
uint64_t position() const
Definition: playbin.cpp:350
IntWrapper is a type-safe integer that allows for encoding/enforcing semantics by means of tags...
Definition: dimensions.h:68
MediaFileType media_file_type() const
Definition: playbin.cpp:597
struct gstreamer::Bus::Message::Detail::StateChanged state_changed