Music Hub  ..
A session-wide music playback service
playbin.cpp
Go to the documentation of this file.
1 /*
2  * Copyright © 2013 Canonical Ltd.
3  *
4  * This program is free software: you can redistribute it and/or modify it
5  * under the terms of the GNU Lesser General Public License version 3,
6  * as published by the Free Software Foundation.
7  *
8  * This program is distributed in the hope that it will be useful,
9  * but WITHOUT ANY WARRANTY; without even the implied warranty of
10  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11  * GNU Lesser General Public License for more details.
12  *
13  * You should have received a copy of the GNU Lesser General Public License
14  * along with this program. If not, see <http://www.gnu.org/licenses/>.
15  *
16  * Authored by: Thomas Voß <thomas.voss@canonical.com>
17  */
18 
20 
22 
23 #if defined(MEDIA_HUB_HAVE_HYBRIS_MEDIA_COMPAT_LAYER)
24 #include <hybris/media/surface_texture_client_hybris.h>
25 #include <hybris/media/media_codec_layer.h>
26 
27 namespace
28 {
29 void setup_video_sink_for_buffer_streaming(GstElement* video_sink)
30 {
31  // Get the service-side BufferQueue (IGraphicBufferProducer) and associate it with
32  // the SurfaceTextureClientHybris instance
33  IGBPWrapperHybris igbp = decoding_service_get_igraphicbufferproducer();
34  SurfaceTextureClientHybris stc = surface_texture_client_create_by_igbp(igbp);
35  // Because mirsink is being loaded, we are definitely doing * hardware rendering.
36  surface_texture_client_set_hardware_rendering (stc, TRUE);
37  g_object_set (G_OBJECT (video_sink), "surface", static_cast<gpointer>(stc), static_cast<char*>(NULL));
38 }
39 }
40 #else // MEDIA_HUB_HAVE_HYBRIS_MEDIA_COMPAT_LAYER
41 namespace
42 {
43 void setup_video_sink_for_buffer_streaming(GstElement*)
44 {
45  throw core::ubuntu::media::Player::Error::OutOfProcessBufferStreamingNotSupported{};
46 }
47 }
48 #endif // MEDIA_HUB_HAVE_HYBRIS_MEDIA_COMPAT_LAYER
49 
50 namespace
51 {
52 bool is_mir_video_sink()
53 {
54  return g_strcmp0(::getenv("CORE_UBUNTU_MEDIA_SERVICE_VIDEO_SINK_NAME"), "mirsink") == 0;
55 }
56 }
57 // Uncomment to generate a dot file at the time that the pipeline
58 // goes to the PLAYING state. Make sure to export GST_DEBUG_DUMP_DOT_DIR
59 // before starting media-hub-server. To convert the dot file to something
60 // other image format, use: dot pipeline.dot -Tpng -o pipeline.png
61 //#define DEBUG_GST_PIPELINE
62 
63 namespace media = core::ubuntu::media;
65 
67 {
68  static const std::string s{"playbin"};
69  return s;
70 }
71 
72 void gstreamer::Playbin::about_to_finish(GstElement*, gpointer user_data)
73 {
74  auto thiz = static_cast<Playbin*>(user_data);
75  thiz->signals.about_to_finish();
76 }
77 
79  GstElement *source,
80  gpointer user_data)
81 {
82  if (user_data == nullptr)
83  return;
84 
85  static_cast<Playbin*>(user_data)->setup_source(source);
86 }
87 
89  : pipeline(gst_element_factory_make("playbin", pipeline_name().c_str())),
90  bus{gst_element_get_bus(pipeline)},
91  file_type(MEDIA_FILE_TYPE_NONE),
92  video_sink(nullptr),
93  on_new_message_connection(
94  bus.on_new_message.connect(
95  std::bind(
97  this,
98  std::placeholders::_1))),
99  is_seeking(false),
100  previous_position(0),
101  player_lifetime(media::Player::Lifetime::normal),
102  is_eos(false)
103 {
104  if (!pipeline)
105  throw std::runtime_error("Could not create pipeline for playbin.");
106 
107  is_eos = false;
108 
109  // Add audio and/or video sink elements depending on environment variables
110  // being set or not set
111  setup_pipeline_for_audio_video();
112 
113  g_signal_connect(
114  pipeline,
115  "about-to-finish",
116  G_CALLBACK(about_to_finish),
117  this
118  );
119 
120  g_signal_connect(
121  pipeline,
122  "source-setup",
123  G_CALLBACK(source_setup),
124  this
125  );
126 }
127 
129 {
130  if (pipeline)
131  gst_object_unref(pipeline);
132 }
133 
135 {
136  std::cout << "Client died, resetting pipeline" << std::endl;
137  // When the client dies, tear down the current pipeline and get it
138  // in a state that is ready for the next client that connects to the
139  // service
140 
141  // Don't reset the pipeline if we want to resume
142  if (player_lifetime != media::Player::Lifetime::resumable) {
143  reset_pipeline();
144  }
145  // Signal to the Player class that the client side has disconnected
146  signals.client_disconnected();
147 }
148 
150 {
151  std::cout << __PRETTY_FUNCTION__ << std::endl;
152  auto ret = gst_element_set_state(pipeline, GST_STATE_NULL);
153  switch(ret)
154  {
155  case GST_STATE_CHANGE_FAILURE:
156  std::cout << "Failed to reset the pipeline state. Client reconnect may not function properly." << std::endl;
157  break;
158  case GST_STATE_CHANGE_NO_PREROLL:
159  case GST_STATE_CHANGE_SUCCESS:
160  case GST_STATE_CHANGE_ASYNC:
161  break;
162  default:
163  std::cout << "Failed to reset the pipeline state. Client reconnect may not function properly." << std::endl;
164  }
165  file_type = MEDIA_FILE_TYPE_NONE;
166 }
167 
169 {
170  switch(message.type)
171  {
172  case GST_MESSAGE_ERROR:
173  signals.on_error(message.detail.error_warning_info);
174  break;
175  case GST_MESSAGE_WARNING:
176  signals.on_warning(message.detail.error_warning_info);
177  break;
178  case GST_MESSAGE_INFO:
179  signals.on_info(message.detail.error_warning_info);
180  break;
181  case GST_MESSAGE_TAG:
182  {
183  gchar *orientation;
184  if (gst_tag_list_get_string(message.detail.tag.tag_list, "image-orientation", &orientation))
185  {
186  // If the image-orientation tag is in the GstTagList, signal the Engine
187  signals.on_orientation_changed(orientation_lut(orientation));
188  g_free (orientation);
189  }
190 
191  signals.on_tag_available(message.detail.tag);
192  }
193  break;
194  case GST_MESSAGE_STATE_CHANGED:
195  signals.on_state_changed(message.detail.state_changed);
196  break;
197  case GST_MESSAGE_ASYNC_DONE:
198  if (is_seeking)
199  {
200  // FIXME: Pass the actual playback time position to the signal call
201  signals.on_seeked_to(0);
202  is_seeking = false;
203  }
204  break;
205  case GST_MESSAGE_EOS:
206  is_eos = true;
207  signals.on_end_of_stream();
208  default:
209  break;
210  }
211 }
212 
214 {
215  return bus;
216 }
217 
219 {
220  gint flags;
221  g_object_get (pipeline, "flags", &flags, nullptr);
222  flags |= GST_PLAY_FLAG_AUDIO;
223  flags |= GST_PLAY_FLAG_VIDEO;
224  flags &= ~GST_PLAY_FLAG_TEXT;
225  g_object_set (pipeline, "flags", flags, nullptr);
226 
227  if (::getenv("CORE_UBUNTU_MEDIA_SERVICE_AUDIO_SINK_NAME") != nullptr)
228  {
229  auto audio_sink = gst_element_factory_make (
230  ::getenv("CORE_UBUNTU_MEDIA_SERVICE_AUDIO_SINK_NAME"),
231  "audio-sink");
232 
233  std::cout << "audio_sink: " << ::getenv("CORE_UBUNTU_MEDIA_SERVICE_AUDIO_SINK_NAME") << std::endl;
234 
235  g_object_set (
236  pipeline,
237  "audio-sink",
238  audio_sink,
239  NULL);
240  }
241 
242  if (::getenv("CORE_UBUNTU_MEDIA_SERVICE_VIDEO_SINK_NAME") != nullptr)
243  {
244  video_sink = gst_element_factory_make (
245  ::getenv("CORE_UBUNTU_MEDIA_SERVICE_VIDEO_SINK_NAME"),
246  "video-sink");
247 
248  std::cout << "video_sink: " << ::getenv("CORE_UBUNTU_MEDIA_SERVICE_VIDEO_SINK_NAME") << std::endl;
249 
250  g_object_set (
251  pipeline,
252  "video-sink",
253  video_sink,
254  NULL);
255  }
256 }
257 
259 {
260  if (not video_sink) throw std::logic_error
261  {
262  "No video sink configured for the current pipeline"
263  };
264 
265  setup_video_sink_for_buffer_streaming(video_sink);
266 }
267 
268 void gstreamer::Playbin::set_volume(double new_volume)
269 {
270  g_object_set (pipeline, "volume", new_volume, NULL);
271 }
272 
275 {
276  switch (audio_role)
277  {
278  case media::Player::AudioStreamRole::alarm:
279  return "alarm";
280  break;
281  case media::Player::AudioStreamRole::alert:
282  return "alert";
283  break;
284  case media::Player::AudioStreamRole::multimedia:
285  return "multimedia";
286  break;
287  case media::Player::AudioStreamRole::phone:
288  return "phone";
289  break;
290  default:
291  return "multimedia";
292  break;
293  }
294 }
295 
297 {
298  if (g_strcmp0(orientation, "rotate-0") == 0)
299  return media::Player::Orientation::rotate0;
300  else if (g_strcmp0(orientation, "rotate-90") == 0)
301  return media::Player::Orientation::rotate90;
302  else if (g_strcmp0(orientation, "rotate-180") == 0)
303  return media::Player::Orientation::rotate180;
304  else if (g_strcmp0(orientation, "rotate-270") == 0)
305  return media::Player::Orientation::rotate270;
306  else
307  return media::Player::Orientation::rotate0;
308 }
309 
312 {
313  GstElement *audio_sink = NULL;
314  g_object_get (pipeline, "audio-sink", &audio_sink, NULL);
315 
316  std::string role_str("props,media.role=" + get_audio_role_str(new_audio_role));
317  std::cout << "Audio stream role: " << role_str << std::endl;
318 
319  GstStructure *props = gst_structure_from_string (role_str.c_str(), NULL);
320  if (audio_sink != nullptr && props != nullptr)
321  g_object_set (audio_sink, "stream-properties", props, NULL);
322  else
323  {
324  std::cerr <<
325  "Warning: couldn't set audio stream role - couldn't get audio_sink from pipeline" <<
326  std::endl;
327  }
328 
329  gst_structure_free (props);
330 }
331 
333 {
334  player_lifetime = lifetime;
335 }
336 
338 {
339  int64_t pos = 0;
340  gst_element_query_position (pipeline, GST_FORMAT_TIME, &pos);
341 
342  // This prevents a 0 position from being reported to the app which happens while seeking.
343  // This is covering over a GStreamer issue
344  if ((static_cast<uint64_t>(pos) < duration()) && is_seeking && pos == 0)
345  {
346  return previous_position;
347  }
348 
349  // Save the current position to use just in case it's needed the next time position is
350  // requested
351  previous_position = static_cast<uint64_t>(pos);
352 
353  // FIXME: this should be int64_t, but dbus-cpp doesn't seem to handle it correctly
354  return static_cast<uint64_t>(pos);
355 }
356 
358 {
359  int64_t dur = 0;
360  gst_element_query_duration (pipeline, GST_FORMAT_TIME, &dur);
361 
362  // FIXME: this should be int64_t, but dbus-cpp doesn't seem to handle it correctly
363  return static_cast<uint64_t>(dur);
364 }
365 
367  const std::string& uri,
369 {
370  reset_pipeline();
371 
372  g_object_set(pipeline, "uri", uri.c_str(), NULL);
373  if (is_video_file(uri))
374  file_type = MEDIA_FILE_TYPE_VIDEO;
375  else if (is_audio_file(uri))
376  file_type = MEDIA_FILE_TYPE_AUDIO;
377 
378  request_headers = headers;
379 }
380 
381 void gstreamer::Playbin::setup_source(GstElement *source)
382 {
383  if (source == NULL || request_headers.empty())
384  return;
385 
386  if (request_headers.find("Cookie") != request_headers.end()) {
387  if (g_object_class_find_property(G_OBJECT_GET_CLASS(source),
388  "cookies") != NULL) {
389  gchar ** cookies = g_strsplit(request_headers["Cookie"].c_str(), ";", 0);
390  g_object_set(source, "cookies", cookies, NULL);
391  g_strfreev(cookies);
392  }
393  }
394 
395  if (request_headers.find("User-Agent") != request_headers.end()) {
396  if (g_object_class_find_property(G_OBJECT_GET_CLASS(source),
397  "user-agent") != NULL) {
398  g_object_set(source, "user-agent", request_headers["User-Agent"].c_str(), NULL);
399  }
400  }
401 }
402 
403 std::string gstreamer::Playbin::uri() const
404 {
405  gchar* data = nullptr;
406  g_object_get(pipeline, "current-uri", &data, nullptr);
407 
408  std::string result((data == nullptr ? "" : data));
409  g_free(data);
410 
411  return result;
412 }
413 
415 {
416  static const std::chrono::nanoseconds state_change_timeout
417  {
418  // We choose a quite high value here as tests are run under valgrind
419  // and gstreamer pipeline setup/state changes take longer in that scenario.
420  // The value does not negatively impact runtime performance.
421  std::chrono::milliseconds{5000}
422  };
423 
424  auto ret = gst_element_set_state(pipeline, new_state);
425 
426  std::cout << __PRETTY_FUNCTION__ << ": requested state change." << std::endl;
427 
428  bool result = false; GstState current, pending;
429  switch(ret)
430  {
431  case GST_STATE_CHANGE_FAILURE:
432  result = false; break;
433  case GST_STATE_CHANGE_NO_PREROLL:
434  case GST_STATE_CHANGE_SUCCESS:
435  result = true; break;
436  case GST_STATE_CHANGE_ASYNC:
437  result = GST_STATE_CHANGE_SUCCESS == gst_element_get_state(
438  pipeline,
439  &current,
440  &pending,
441  state_change_timeout.count());
442  break;
443  }
444 
445  // We only should query the pipeline if we actually succeeded in
446  // setting the requested state. Also don't send on_video_dimensions_changed
447  // signal during EOS.
448  if (result && new_state == GST_STATE_PLAYING && !is_eos)
449  {
450  // Get the video height/width from the video sink
451  try
452  {
453  signals.on_video_dimensions_changed(get_video_dimensions());
454  }
455  catch (const std::exception& e)
456  {
457  std::cerr << "Problem querying video dimensions: " << e.what() << std::endl;
458  }
459  catch (...)
460  {
461  std::cerr << "Problem querying video dimensions." << std::endl;
462  }
463 
464 #ifdef DEBUG_GST_PIPELINE
465  std::cout << "Dumping pipeline dot file" << std::endl;
466  GST_DEBUG_BIN_TO_DOT_FILE((GstBin*)pipeline, GST_DEBUG_GRAPH_SHOW_ALL, "pipeline");
467 #endif
468  }
469 
470  return result;
471 }
472 
473 bool gstreamer::Playbin::seek(const std::chrono::microseconds& ms)
474 {
475  is_seeking = true;
476  return gst_element_seek_simple(
477  pipeline,
478  GST_FORMAT_TIME,
479  (GstSeekFlags)(GST_SEEK_FLAG_FLUSH | GST_SEEK_FLAG_KEY_UNIT),
480  ms.count() * 1000);
481 }
482 
484 {
485  if (not video_sink || not is_mir_video_sink())
486  throw std::runtime_error
487  {
488  "Missing video sink or video sink does not support query of width and height."
489  };
490 
491  // Initialize to default value prior to querying actual values from the sink.
492  uint32_t video_width = 0, video_height = 0;
493  g_object_get (video_sink, "height", &video_height, nullptr);
494  g_object_get (video_sink, "width", &video_width, nullptr);
495  // TODO(tvoss): We should probably check here if width and height are valid.
497  {
500  };
501 }
502 
503 std::string gstreamer::Playbin::get_file_content_type(const std::string& uri) const
504 {
505  if (uri.empty())
506  return std::string();
507 
508  std::string filename(uri);
509  std::cout << "filename: " << filename << std::endl;
510  size_t pos = uri.find("file://");
511  if (pos != std::string::npos)
512  filename = uri.substr(pos + 7, std::string::npos);
513  else
514  // Anything other than a file, for now claim that the type
515  // is both audio and video.
516  // FIXME: implement true net stream sampling and get the type from GstCaps
517  return std::string("audio/video/");
518 
519 
520  GError *error = nullptr;
521  std::unique_ptr<GFile, void(*)(void *)> file(
522  g_file_new_for_path(filename.c_str()), g_object_unref);
523  std::unique_ptr<GFileInfo, void(*)(void *)> info(
524  g_file_query_info(
525  file.get(), G_FILE_ATTRIBUTE_STANDARD_FAST_CONTENT_TYPE ","
526  G_FILE_ATTRIBUTE_ETAG_VALUE, G_FILE_QUERY_INFO_NONE,
527  /* cancellable */ NULL, &error),
528  g_object_unref);
529  if (!info)
530  {
531  std::string error_str(error->message);
532  g_error_free(error);
533 
534  std::cout << "Failed to query the URI for the presence of video content: "
535  << error_str << std::endl;
536  return std::string();
537  }
538 
539  std::string content_type(g_file_info_get_attribute_string(
540  info.get(), G_FILE_ATTRIBUTE_STANDARD_FAST_CONTENT_TYPE));
541 
542  return content_type;
543 }
544 
545 bool gstreamer::Playbin::is_audio_file(const std::string& uri) const
546 {
547  if (uri.empty())
548  return false;
549 
550  if (get_file_content_type(uri).find("audio/") == 0)
551  {
552  std::cout << "Found audio content" << std::endl;
553  return true;
554  }
555 
556  return false;
557 }
558 
559 bool gstreamer::Playbin::is_video_file(const std::string& uri) const
560 {
561  if (uri.empty())
562  return false;
563 
564  if (get_file_content_type(uri).find("video/") == 0)
565  {
566  std::cout << "Found video content" << std::endl;
567  return true;
568  }
569 
570  return false;
571 }
572 
574 {
575  return file_type;
576 }
static std::string get_audio_role_str(core::ubuntu::media::Player::AudioStreamRole audio_role)
Definition: playbin.cpp:274
bool set_state_and_wait(GstState new_state)
Definition: playbin.cpp:414
static void source_setup(GstElement *, GstElement *source, gpointer user_data)
Definition: playbin.cpp:78
void setup_source(GstElement *source)
Definition: playbin.cpp:381
std::tuple< Height, Width > Dimensions
Height and Width of a video.
Definition: dimensions.h:139
core::ubuntu::media::video::Dimensions get_video_dimensions() const
Definition: playbin.cpp:483
GstMessageType type
Definition: bus.h:181
bool seek(const std::chrono::microseconds &ms)
Definition: playbin.cpp:473
uint64_t duration() const
Definition: playbin.cpp:357
void reset_pipeline()
Definition: playbin.cpp:149
static const std::string & pipeline_name()
Definition: playbin.cpp:66
void set_lifetime(core::ubuntu::media::Player::Lifetime)
Definition: playbin.cpp:332
struct gstreamer::Playbin::@12 signals
void set_audio_stream_role(core::ubuntu::media::Player::AudioStreamRole new_audio_role)
Definition: playbin.cpp:311
std::map< std::string, std::string > HeadersType
Definition: player.h:48
void on_new_message(const Bus::Message &message)
Definition: playbin.cpp:168
core::ubuntu::media::Player::Orientation orientation_lut(const gchar *orientation)
Definition: playbin.cpp:296
gstreamer::Bus & message_bus()
Definition: playbin.cpp:213
void set_uri(const std::string &uri, const core::ubuntu::media::Player::HeadersType &headers)
Definition: playbin.cpp:366
core::Signal< void > about_to_finish
Definition: playbin.h:121
std::string uri() const
Definition: playbin.cpp:403
void setup_pipeline_for_audio_video()
Definition: playbin.cpp:218
union gstreamer::Bus::Message::Detail detail
GstElement * pipeline
Definition: playbin.h:109
std::string get_file_content_type(const std::string &uri) const
Definition: playbin.cpp:503
struct gstreamer::Bus::Message::Detail::Tag tag
bool is_audio_file(const std::string &uri) const
Definition: playbin.cpp:545
bool is_video_file(const std::string &uri) const
Definition: playbin.cpp:559
struct gstreamer::Bus::Message::Detail::ErrorWarningInfo error_warning_info
void set_volume(double new_volume)
Definition: playbin.cpp:268
void create_video_sink(uint32_t texture_id)
Definition: playbin.cpp:258
uint64_t position() const
Definition: playbin.cpp:337
IntWrapper is a type-safe integer that allows for encoding/enforcing semantics by means of tags...
Definition: dimensions.h:68
MediaFileType media_file_type() const
Definition: playbin.cpp:573
struct gstreamer::Bus::Message::Detail::StateChanged state_changed