24 #include <gst/pbutils/missing-plugins.h> 26 #include <hybris/media/surface_texture_client_hybris.h> 27 #include <hybris/media/media_codec_layer.h> 32 #include <sys/socket.h> 38 static const char *PULSE_SINK =
"pulsesink";
39 static const char *HYBRIS_SINK =
"hybrissink";
40 static const char *MIR_SINK =
"mirsink";
44 void gstreamer::Playbin::setup_video_sink_for_buffer_streaming()
46 IGBPWrapperHybris igbp;
47 SurfaceTextureClientHybris stc;
49 GstStructure *structure;
52 case core::ubuntu::media::AVBackend::Backend::hybris:
55 igbp = decoding_service_get_igraphicbufferproducer();
56 stc = surface_texture_client_create_by_igbp(igbp);
59 surface_texture_client_set_hardware_rendering(stc, TRUE);
61 context = gst_context_new(
"gst.mir.MirContext", TRUE);
62 structure = gst_context_writable_structure(context);
63 gst_structure_set(structure,
"gst_mir_context", G_TYPE_POINTER, stc, NULL);
66 gst_element_set_context(pipeline, context);
68 case core::ubuntu::media::AVBackend::Backend::mir:
70 connect_to_consumer();
73 g_object_set (G_OBJECT (video_sink),
"export-buffers", TRUE,
nullptr);
75 case core::ubuntu::media::AVBackend::Backend::none:
77 throw core::ubuntu::media::Player::Errors::
78 OutOfProcessBufferStreamingNotSupported{};
82 bool gstreamer::Playbin::is_supported_video_sink(
void)
const 84 if (video_sink_name == HYBRIS_SINK || video_sink_name == MIR_SINK)
101 static const std::string s{
"playbin"};
107 auto thiz =
static_cast<Playbin*
>(user_data);
115 if (user_data ==
nullptr)
118 static_cast<Playbin*
>(user_data)->setup_source(source);
122 : pipeline(gst_element_factory_make(
"playbin", pipeline_name().c_str())),
132 std::placeholders::_1))),
151 throw std::runtime_error(
"Could not create pipeline for playbin.");
193 print_refs(*
this,
"gstreamer::Playbin::~Playbin pipeline");
202 if (sock_consumer != -1) {
203 close(sock_consumer);
208 print_refs(*
this,
"gstreamer::Playbin::~Playbin pipeline");
214 MH_INFO(
"Client died, resetting pipeline");
230 const auto ret = gst_element_set_state(
pipeline, GST_STATE_NULL);
233 case GST_STATE_CHANGE_FAILURE:
234 MH_WARNING(
"Failed to reset the pipeline state. Client reconnect may not function properly.");
236 case GST_STATE_CHANGE_NO_PREROLL:
237 case GST_STATE_CHANGE_SUCCESS:
238 case GST_STATE_CHANGE_ASYNC:
241 MH_WARNING(
"Failed to reset the pipeline state. Client reconnect may not function properly.");
248 if (sock_consumer != -1) {
249 close(sock_consumer);
254 void gstreamer::Playbin::process_missing_plugin_message(GstMessage *message)
256 gchar *desc = gst_missing_plugin_message_get_description(message);
260 const GstStructure *msg_data = gst_message_get_structure(message);
261 if (g_strcmp0(
"decoder", gst_structure_get_string(msg_data,
"type")) != 0)
265 if (!gst_structure_get(msg_data,
"detail", GST_TYPE_CAPS, &caps, NULL)) {
270 GstStructure *caps_data = gst_caps_get_structure(caps, 0);
276 const gchar *mime = gst_structure_get_name(caps_data);
277 if (strstr(mime,
"audio"))
279 else if (strstr(mime,
"video"))
282 MH_ERROR(
"Missing decoder for %s", mime);
287 const GstStructure *msg_data = gst_message_get_structure(message);
288 const gchar *struct_name = gst_structure_get_name(msg_data);
290 if (g_strcmp0(
"buffer-export-data", struct_name) == 0)
294 if (!gst_structure_get(msg_data,
295 "fd", G_TYPE_INT, &fd,
296 "width", G_TYPE_INT, &meta.
width,
297 "height", G_TYPE_INT, &meta.
height,
298 "fourcc", G_TYPE_INT, &meta.
fourcc,
299 "stride", G_TYPE_INT, &meta.
stride,
300 "offset", G_TYPE_INT, &meta.
offset,
303 MH_ERROR(
"Bad buffer-export-data message: mirsink version mismatch?");
307 send_buffer_data(fd, &meta,
sizeof meta);
309 else if (g_strcmp0(
"frame-ready", struct_name) == 0)
315 MH_ERROR(
"Unknown GST_MESSAGE_ELEMENT with struct %s", struct_name);
321 switch (message.
type)
323 case GST_MESSAGE_ERROR:
326 case GST_MESSAGE_WARNING:
329 case GST_MESSAGE_INFO:
332 case GST_MESSAGE_STATE_CHANGED:
333 if (message.
source ==
"playbin") {
339 case GST_MESSAGE_ELEMENT:
340 if (gst_is_missing_plugin_message(message.
message))
341 process_missing_plugin_message(message.
message);
345 case GST_MESSAGE_TAG:
348 if (gst_tag_list_get_string(message.
detail.
tag.
tag_list,
"image-orientation", &orientation))
352 g_free (orientation);
358 case GST_MESSAGE_ASYNC_DONE:
366 case GST_MESSAGE_EOS:
369 case GST_MESSAGE_BUFFERING:
385 g_object_get (
pipeline,
"flags", &flags,
nullptr);
389 g_object_set (
pipeline,
"flags", flags,
nullptr);
391 const char *asink_name = ::getenv(
"CORE_UBUNTU_MEDIA_SERVICE_AUDIO_SINK_NAME");
393 if (asink_name ==
nullptr)
394 asink_name = PULSE_SINK;
396 audio_sink = gst_element_factory_make (asink_name,
"audio-sink");
400 MH_ERROR(
"Error trying to create audio sink %s", asink_name);
402 const char *vsink_name = ::getenv(
"CORE_UBUNTU_MEDIA_SERVICE_VIDEO_SINK_NAME");
404 if (vsink_name ==
nullptr) {
405 if (backend == core::ubuntu::media::AVBackend::Backend::hybris)
406 vsink_name = HYBRIS_SINK;
407 else if (backend == core::ubuntu::media::AVBackend::Backend::mir)
408 vsink_name = MIR_SINK;
412 video_sink_name = vsink_name;
413 video_sink = gst_element_factory_make (vsink_name,
"video-sink");
417 MH_ERROR(
"Error trying to create video sink %s", vsink_name);
425 "No video sink configured for the current pipeline" 428 setup_video_sink_for_buffer_streaming();
433 g_object_set (
pipeline,
"volume", new_volume, NULL);
441 case media::Player::AudioStreamRole::alarm:
444 case media::Player::AudioStreamRole::alert:
447 case media::Player::AudioStreamRole::multimedia:
450 case media::Player::AudioStreamRole::phone:
461 if (g_strcmp0(orientation,
"rotate-0") == 0)
462 return media::Player::Orientation::rotate0;
463 else if (g_strcmp0(orientation,
"rotate-90") == 0)
464 return media::Player::Orientation::rotate90;
465 else if (g_strcmp0(orientation,
"rotate-180") == 0)
466 return media::Player::Orientation::rotate180;
467 else if (g_strcmp0(orientation,
"rotate-270") == 0)
468 return media::Player::Orientation::rotate270;
470 return media::Player::Orientation::rotate0;
477 MH_INFO(
"Audio stream role: %s", role_str);
479 GstStructure *props = gst_structure_from_string (role_str.c_str(), NULL);
480 if (
audio_sink !=
nullptr && props !=
nullptr)
482 g_object_set (
audio_sink,
"stream-properties", props, NULL);
486 MH_WARNING(
"Couldn't set audio stream role - couldn't get audio_sink from pipeline");
489 gst_structure_free (props);
500 gst_element_query_position (
pipeline, GST_FORMAT_TIME, &pos);
514 return static_cast<uint64_t
>(pos);
520 gst_element_query_duration (
pipeline, GST_FORMAT_TIME, &dur);
523 return static_cast<uint64_t
>(dur);
527 const std::string&
uri,
529 bool do_pipeline_reset)
531 gchar *current_uri =
nullptr;
532 g_object_get(
pipeline,
"current-uri", ¤t_uri, NULL);
537 if (current_uri and do_pipeline_reset)
540 std::string tmp_uri{uri};
542 if (uri_check->is_local_file())
544 if (uri_check->is_encoded())
548 MH_DEBUG(
"File URI was encoded, now decoded: %s", tmp_uri);
553 g_object_set(
pipeline,
"uri", tmp_uri.c_str(), NULL);
570 if (g_object_class_find_property(G_OBJECT_GET_CLASS(source),
571 "cookies") != NULL) {
572 gchar ** cookies = g_strsplit(
request_headers[
"Cookie"].c_str(),
";", 0);
573 g_object_set(source,
"cookies", cookies, NULL);
579 if (g_object_class_find_property(G_OBJECT_GET_CLASS(source),
580 "user-agent") != NULL) {
581 g_object_set(source,
"user-agent",
request_headers[
"User-Agent"].c_str(), NULL);
588 gchar* data =
nullptr;
589 g_object_get(
pipeline,
"current-uri", &data,
nullptr);
591 std::string result((data ==
nullptr ?
"" : data));
600 auto thiz =
static_cast<Playbin*
>(user_data);
601 if (thiz and thiz->pipeline)
602 gst_element_set_state(thiz->pipeline, thiz->current_new_state);
609 static const std::chrono::nanoseconds state_change_timeout
614 std::chrono::milliseconds{5000}
618 GstState current, pending;
625 MH_DEBUG(
"Requested state change in main thread context.");
627 GstState current, pending;
628 result = GST_STATE_CHANGE_SUCCESS == gst_element_get_state(
632 state_change_timeout.count());
636 const auto ret = gst_element_set_state(
pipeline, new_state);
638 MH_DEBUG(
"Requested state change not using main thread context.");
642 case GST_STATE_CHANGE_FAILURE:
643 result =
false;
break;
644 case GST_STATE_CHANGE_NO_PREROLL:
645 case GST_STATE_CHANGE_SUCCESS:
646 result =
true;
break;
647 case GST_STATE_CHANGE_ASYNC:
648 result = GST_STATE_CHANGE_SUCCESS == gst_element_get_state(
652 state_change_timeout.count());
659 if (result && new_state == GST_STATE_PLAYING)
668 catch (
const std::exception& e)
670 MH_WARNING(
"Problem querying video dimensions: %s", e.what());
674 MH_WARNING(
"Problem querying video dimensions.");
677 #ifdef DEBUG_GST_PIPELINE 678 MH_DEBUG(
"Dumping pipeline dot file");
679 GST_DEBUG_BIN_TO_DOT_FILE((GstBin*)
pipeline, GST_DEBUG_GRAPH_SHOW_ALL,
"pipeline");
689 return gst_element_seek_simple(
692 (GstSeekFlags)(GST_SEEK_FLAG_FLUSH | GST_SEEK_FLAG_KEY_UNIT),
698 if (not
video_sink || not is_supported_video_sink())
699 throw std::runtime_error
701 "Missing video sink or video sink does not support query of width and height." 705 int video_width = 0, video_height = 0;
708 GstIterator *iter = gst_element_iterate_pads(
video_sink);
710 gst_iterator_next(iter, &item) == GST_ITERATOR_OK;
711 g_value_unset(&item))
713 GstPad *pad = GST_PAD(g_value_get_object(&item));
714 GstCaps *caps = gst_pad_get_current_caps(pad);
717 const GstStructure *s = gst_caps_get_structure(caps, 0);
718 gst_structure_get_int(s,
"width", &video_width);
719 gst_structure_get_int(s,
"height", &video_height);
720 MH_DEBUG(
"Video dimensions are %d x %d", video_width, video_height);
722 gst_caps_unref(caps);
725 gst_iterator_free(iter);
740 signals.on_video_dimensions_changed(new_dimensions);
745 GError *error =
nullptr;
748 std::unique_ptr<GFile, void(*)(void *)> file(
749 g_file_new_for_uri(uri.c_str()), g_object_unref);
750 std::unique_ptr<GFileInfo, void(*)(void *)> info(
752 file.get(), G_FILE_ATTRIBUTE_STANDARD_FAST_CONTENT_TYPE
"," 753 G_FILE_ATTRIBUTE_ETAG_VALUE, G_FILE_QUERY_INFO_NONE,
757 return std::string();
759 std::string content_type = g_file_info_get_attribute_string(
760 info.get(), G_FILE_ATTRIBUTE_STANDARD_FAST_CONTENT_TYPE);
761 if (content_type.empty())
762 return std::string();
764 if (content_type ==
"application/octet-stream")
766 std::unique_ptr<GFileInfo, void(*)(void *)> full_info(
767 g_file_query_info(file.get(), G_FILE_ATTRIBUTE_STANDARD_CONTENT_TYPE,
768 G_FILE_QUERY_INFO_NONE,
769 NULL, &error),g_object_unref);
772 return std::string();
774 content_type = g_file_info_get_attribute_string(
775 full_info.get(), G_FILE_ATTRIBUTE_STANDARD_CONTENT_TYPE);
776 if (content_type.empty())
777 return std::string();
785 return std::string();
787 std::string encoded_uri;
789 gchar *uri_scheme = g_uri_parse_scheme(uri.c_str());
791 if (uri_scheme and strlen(uri_scheme) > 0 and uri_check->is_encoded())
793 MH_DEBUG(
"Is a URI and is already percent encoded");
797 else if (uri_scheme and strlen(uri_scheme) > 0 and !uri_check->is_encoded())
799 MH_DEBUG(
"Is a URI and is not already percent encoded");
800 gchar *encoded = g_uri_escape_string(uri.c_str(),
806 return std::string();
808 encoded_uri.assign(encoded);
813 GError *error =
nullptr;
814 MH_DEBUG(
"Is a path and is not already percent encoded");
815 gchar *str = g_filename_to_uri(uri.c_str(),
nullptr, &error);
819 return std::string();
821 encoded_uri.assign(str);
823 if (error !=
nullptr)
825 MH_WARNING(
"Failed to get actual track content type: %s", error->message);
829 return std::string(
"audio/video/");
831 gchar *escaped = g_uri_escape_string(encoded_uri.c_str(),
837 return std::string();
839 encoded_uri.assign(escaped);
851 return std::string();
853 gchar *decoded_gchar = g_uri_unescape_string(uri.c_str(),
nullptr);
855 return std::string();
857 const std::string decoded{decoded_gchar};
858 g_free(decoded_gchar);
865 return std::string();
867 const std::string encoded_uri{
encode_uri(uri)};
870 if (content_type.empty())
872 MH_WARNING(
"Failed to get actual track content type");
873 return std::string(
"audio/video/");
876 MH_INFO(
"Found content type: %s", content_type);
888 MH_INFO(
"Found audio content");
902 MH_INFO(
"Found video content");
933 bool gstreamer::Playbin::connect_to_consumer(
void)
935 static const char *local_socket =
"media-hub-server";
936 static const char *consumer_socket =
"media-consumer";
939 struct sockaddr_un local, remote;
941 if (sock_consumer != -1) {
943 close(sock_consumer);
946 if ((sock_consumer = socket(AF_UNIX, SOCK_DGRAM, 0)) == -1)
948 MH_ERROR(
"Cannot create socket: %s (%d)", strerror(errno), errno);
953 ostringstream local_ss;
954 local_ss << local_socket << key;
955 local.sun_family = AF_UNIX;
956 local.sun_path[0] =
'\0';
957 strcpy(local.sun_path + 1, local_ss.str().c_str());
958 len =
sizeof(local.sun_family) + local_ss.str().length() + 1;
959 if (bind(sock_consumer, (
struct sockaddr *) &local, len) == -1)
961 MH_ERROR(
"Cannot bind socket: %s (%d)", strerror(errno), errno);
962 close(sock_consumer);
968 ostringstream remote_ss;
969 remote_ss << consumer_socket << key;
970 remote.sun_family = AF_UNIX;
971 remote.sun_path[0] =
'\0';
972 strcpy(remote.sun_path + 1, remote_ss.str().c_str());
973 len =
sizeof(remote.sun_family) + remote_ss.str().length() + 1;
974 if (connect(sock_consumer, (
struct sockaddr *) &remote, len) == -1)
976 MH_ERROR(
"Cannot connect to consumer: %s (%d)", strerror(errno), errno);
977 close(sock_consumer);
982 MH_DEBUG(
"Connected to buffer consumer socket");
987 void gstreamer::Playbin::send_buffer_data(
int fd,
void *data,
size_t len)
990 char buf[CMSG_SPACE(
sizeof fd)]{};
991 struct cmsghdr *cmsg;
992 struct iovec io = { .iov_base = data, .iov_len = len };
996 msg.msg_control = buf;
997 msg.msg_controllen =
sizeof buf;
999 cmsg = CMSG_FIRSTHDR(&msg);
1000 cmsg->cmsg_level = SOL_SOCKET;
1001 cmsg->cmsg_type = SCM_RIGHTS;
1002 cmsg->cmsg_len = CMSG_LEN(
sizeof fd);
1004 memmove(CMSG_DATA(cmsg), &fd,
sizeof fd);
1006 msg.msg_controllen = cmsg->cmsg_len;
1008 if (sendmsg(sock_consumer, &msg, 0) < 0)
1009 MH_ERROR(
"Failed to send dma_buf fd to consumer: %s (%d)",
1010 strerror(errno), errno);
1013 void gstreamer::Playbin::send_frame_ready(
void)
1015 const char ready =
'r';
1017 if (send (sock_consumer, &ready,
sizeof ready, 0) == -1)
1018 MH_ERROR(
"Error when sending frame ready flag to client: %s (%d)",
1019 strerror(errno), errno);
static std::string get_audio_role_str(core::ubuntu::media::Player::AudioStreamRole audio_role)
MediaFileType media_file_type() const
core::ubuntu::media::video::Dimensions get_video_dimensions() const
bool is_audio_file(const std::string &uri) const
static void source_setup(GstElement *, GstElement *source, gpointer user_data)
void setup_source(GstElement *source)
void set_uri(const std::string &uri, const core::ubuntu::media::Player::HeadersType &headers, bool do_pipeline_reset=true)
void process_message_element(GstMessage *message)
static gboolean set_state_in_main_thread(gpointer user_data)
bool seek(const std::chrono::microseconds &ms)
core::Connection on_new_message_connection_async
static const std::string & pipeline_name()
std::string file_info_from_uri(const std::string &uri) const
Playbin(const core::ubuntu::media::Player::PlayerKey key)
void emit_video_dimensions_changed_if_changed(const core::ubuntu::media::video::Dimensions &new_dimensions)
void set_lifetime(core::ubuntu::media::Player::Lifetime)
struct gstreamer::Playbin::@12 signals
void set_audio_stream_role(core::ubuntu::media::Player::AudioStreamRole new_audio_role)
bool is_missing_audio_codec
GstState current_new_state
std::string get_file_content_type(const std::string &uri) const
core::ubuntu::media::Player::Orientation orientation_lut(const gchar *orientation)
uint64_t position() const
struct gstreamer::Bus::Message::Detail::@1 buffering
gstreamer::Bus & message_bus()
core::Signal< void > about_to_finish
bool is_video_file(const std::string &uri) const
void setup_pipeline_for_audio_video()
bool set_state_and_wait(GstState new_state, bool use_main_thread=false)
std::string decode_uri(const std::string &uri) const
union gstreamer::Bus::Message::Detail detail
gulong source_setup_handler_id
core::ubuntu::media::video::Dimensions cached_video_dimensions
struct gstreamer::Bus::Message::Detail::Tag tag
boost::flyweight< std::string > source
bool can_play_streams() const
struct gstreamer::Bus::Message::Detail::ErrorWarningInfo error_warning_info
void set_volume(double new_volume)
void on_new_message_async(const Bus::Message &message)
void create_video_sink(uint32_t texture_id)
gulong about_to_finish_handler_id
std::string encode_uri(const std::string &uri) const
bool is_missing_video_codec
core::Signal< Message > on_new_message_async
core::ubuntu::media::Player::HeadersType request_headers
uint64_t duration() const
struct gstreamer::Bus::Message::Detail::StateChanged state_changed
uint64_t previous_position
core::ubuntu::media::Player::Lifetime player_lifetime