diff --git a/README.md b/README.md index 86c530644..c5596ff13 100755 --- a/README.md +++ b/README.md @@ -259,7 +259,7 @@ The `features`, `compare`, `release` and `performance` of SRS. 1. Support NGINX-RTMP style EXEC, read [#367][bug #367]. 1. Support NGINX-RTMP style dvr control module, read [#459][bug #459]. 1. Support HTTP Security Raw Api, read [#459][bug #459], [#470][bug #470], [#319][bug #319]. -1. [dev]Support Integration with Kafka/Spark Big-Data system, read [#467][bug #467]. +1. Support Integration with Kafka/Spark Big-Data system, read [#467][bug #467]. 1. [plan]Support Origin Cluster for Load Balance and Fault Tolarence, read [#464][bug #464], [RTMP 302][bug #92]. 1. [plan]Support H.265, push RTMP with H.265, delivery in HLS, read [#465][bug #465]. 1. [plan]Support MPEG-DASH, the future streaming protocol, read [#299][bug #299]. @@ -386,6 +386,7 @@ Remark: ### History +* v3.0, 2015-10-23, fix [#467][bug #467], support write log to kafka. 3.0.6 * v3.0, 2015-10-20, fix [#502][bug #502], support snapshot with http-callback or transcoder. 3.0.5 * v3.0, 2015-09-19, support amf0 and json to convert with each other. * v3.0, 2015-09-19, json objects support dumps to string. @@ -1279,6 +1280,7 @@ Winlin [bug #466]: https://github.com/simple-rtmp-server/srs/issues/466 [bug #468]: https://github.com/simple-rtmp-server/srs/issues/468 [bug #502]: https://github.com/simple-rtmp-server/srs/issues/502 +[bug #467]: https://github.com/simple-rtmp-server/srs/issues/467 [bug #xxxxxxx]: https://github.com/simple-rtmp-server/srs/issues/xxxxxxx [r2.0a2]: https://github.com/simple-rtmp-server/srs/releases/tag/v2.0-a2 diff --git a/trunk/conf/full.conf b/trunk/conf/full.conf index 1ebc28790..dcffd69d9 100644 --- a/trunk/conf/full.conf +++ b/trunk/conf/full.conf @@ -242,8 +242,11 @@ kafka { enabled off; # the broker list, broker is # and use space to specify multple brokers. - # for exampl, 127.0.0.1:9092 127.0.0.1:9093 + # for example, 127.0.0.1:9092 127.0.0.1:9093 brokers 127.0.0.1:9092; + # the kafka topic to use. + # default: srs + topic srs; } ############################################################################################# diff --git a/trunk/src/app/srs_app_caster_flv.cpp b/trunk/src/app/srs_app_caster_flv.cpp index 6e77ba07d..f8258f896 100644 --- a/trunk/src/app/srs_app_caster_flv.cpp +++ b/trunk/src/app/srs_app_caster_flv.cpp @@ -42,6 +42,7 @@ using namespace std; #include #include #include +#include #define SRS_HTTP_FLV_STREAM_BUFFER 4096 @@ -117,20 +118,13 @@ int SrsAppCasterFlv::serve_http(ISrsHttpResponseWriter* w, ISrsHttpMessage* r) SrsDynamicHttpConn::SrsDynamicHttpConn(IConnectionManager* cm, st_netfd_t fd, SrsHttpServeMux* m) : SrsHttpConn(cm, fd, m) { - - req = NULL; - io = NULL; - client = NULL; - stfd = NULL; - stream_id = 0; - + sdk = new SrsSimpleRtmpClient(); pprint = SrsPithyPrint::create_caster(); } SrsDynamicHttpConn::~SrsDynamicHttpConn() { - close(); - + srs_freep(sdk); srs_freep(pprint); } @@ -176,7 +170,7 @@ int SrsDynamicHttpConn::proxy(ISrsHttpResponseWriter* w, ISrsHttpMessage* r, std } ret = do_proxy(rr, &dec); - close(); + sdk->close(); return ret; } @@ -185,14 +179,22 @@ int SrsDynamicHttpConn::do_proxy(ISrsHttpResponseReader* rr, SrsFlvDecoder* dec) { int ret = ERROR_SUCCESS; + int64_t cto = SRS_CONSTS_RTMP_TIMEOUT_US; + int64_t sto = SRS_CONSTS_RTMP_PULSE_TIMEOUT_US; + if ((ret = sdk->connect(output, cto, sto)) != ERROR_SUCCESS) { + srs_error("flv: connect %s failed, cto=%"PRId64", sto=%"PRId64". ret=%d", output.c_str(), cto, sto, ret); + return ret; + } + + if ((ret = sdk->publish()) != ERROR_SUCCESS) { + srs_error("flv: publish failed. ret=%d", ret); + return ret; + } + char pps[4]; while (!rr->eof()) { pprint->elapse(); - if ((ret = connect()) != ERROR_SUCCESS) { - return ret; - } - char type; int32_t size; u_int32_t time; @@ -212,13 +214,23 @@ int SrsDynamicHttpConn::do_proxy(ISrsHttpResponseReader* rr, SrsFlvDecoder* dec) return ret; } - if ((ret = rtmp_write_packet(type, time, data, size)) != ERROR_SUCCESS) { + SrsSharedPtrMessage* msg = NULL; + if ((ret = sdk->rtmp_create_msg(type, time, data, size, &msg)) != ERROR_SUCCESS) { + return ret; + } + + // TODO: FIXME: for post flv, reconnect when error. + if ((ret = sdk->send_and_free_message(msg)) != ERROR_SUCCESS) { if (!srs_is_client_gracefully_close(ret)) { srs_error("flv: proxy rtmp packet failed. ret=%d", ret); } return ret; } + if (pprint->can_print()) { + srs_trace("flv: send msg %d age=%d, dts=%d, size=%d", type, pprint->age(), time, size); + } + if ((ret = dec->read_previous_tag_size(pps)) != ERROR_SUCCESS) { if (!srs_is_client_gracefully_close(ret)) { srs_error("flv: proxy tag header pps failed. ret=%d", ret); @@ -230,153 +242,6 @@ int SrsDynamicHttpConn::do_proxy(ISrsHttpResponseReader* rr, SrsFlvDecoder* dec) return ret; } -int SrsDynamicHttpConn::rtmp_write_packet(char type, u_int32_t timestamp, char* data, int size) -{ - int ret = ERROR_SUCCESS; - - SrsSharedPtrMessage* msg = NULL; - - if ((ret = srs_rtmp_create_msg(type, timestamp, data, size, stream_id, &msg)) != ERROR_SUCCESS) { - srs_error("flv: create shared ptr msg failed. ret=%d", ret); - return ret; - } - srs_assert(msg); - - if (pprint->can_print()) { - srs_trace("flv: send msg %s age=%d, dts=%"PRId64", size=%d", - msg->is_audio()? "A":msg->is_video()? "V":"N", pprint->age(), msg->timestamp, msg->size); - } - - // send out encoded msg. - if ((ret = client->send_and_free_message(msg, stream_id)) != ERROR_SUCCESS) { - return ret; - } - - return ret; -} - -int SrsDynamicHttpConn::connect() -{ - int ret = ERROR_SUCCESS; - - // when ok, ignore. - // TODO: FIXME: should reconnect when disconnected. - if (io || client) { - return ret; - } - - // parse uri - if (!req) { - req = new SrsRequest(); - - size_t pos = string::npos; - string uri = req->tcUrl = output; - - // tcUrl, stream - if ((pos = uri.rfind("/")) != string::npos) { - req->stream = uri.substr(pos + 1); - req->tcUrl = uri = uri.substr(0, pos); - } - - srs_discovery_tc_url(req->tcUrl, - req->schema, req->host, req->vhost, req->app, req->port, - req->param); - } - - // connect host. - if ((ret = srs_socket_connect(req->host, ::atoi(req->port.c_str()), ST_UTIME_NO_TIMEOUT, &stfd)) != ERROR_SUCCESS) { - srs_error("mpegts: connect server %s:%s failed. ret=%d", req->host.c_str(), req->port.c_str(), ret); - return ret; - } - io = new SrsStSocket(stfd); - client = new SrsRtmpClient(io); - - client->set_recv_timeout(SRS_CONSTS_RTMP_RECV_TIMEOUT_US); - client->set_send_timeout(SRS_CONSTS_RTMP_SEND_TIMEOUT_US); - - // connect to vhost/app - if ((ret = client->handshake()) != ERROR_SUCCESS) { - srs_error("mpegts: handshake with server failed. ret=%d", ret); - return ret; - } - if ((ret = connect_app(req->host, req->port)) != ERROR_SUCCESS) { - srs_error("mpegts: connect with server failed. ret=%d", ret); - return ret; - } - if ((ret = client->create_stream(stream_id)) != ERROR_SUCCESS) { - srs_error("mpegts: connect with server failed, stream_id=%d. ret=%d", stream_id, ret); - return ret; - } - - // publish. - if ((ret = client->publish(req->stream, stream_id)) != ERROR_SUCCESS) { - srs_error("mpegts: publish failed, stream=%s, stream_id=%d. ret=%d", - req->stream.c_str(), stream_id, ret); - return ret; - } - - return ret; -} - -// TODO: FIXME: refine the connect_app. -int SrsDynamicHttpConn::connect_app(string ep_server, string ep_port) -{ - int ret = ERROR_SUCCESS; - - // args of request takes the srs info. - if (req->args == NULL) { - req->args = SrsAmf0Any::object(); - } - - // notify server the edge identity, - // @see https://github.com/simple-rtmp-server/srs/issues/147 - SrsAmf0Object* data = req->args; - data->set("srs_sig", SrsAmf0Any::str(RTMP_SIG_SRS_KEY)); - data->set("srs_server", SrsAmf0Any::str(RTMP_SIG_SRS_KEY" "RTMP_SIG_SRS_VERSION" ("RTMP_SIG_SRS_URL_SHORT")")); - data->set("srs_license", SrsAmf0Any::str(RTMP_SIG_SRS_LICENSE)); - data->set("srs_role", SrsAmf0Any::str(RTMP_SIG_SRS_ROLE)); - data->set("srs_url", SrsAmf0Any::str(RTMP_SIG_SRS_URL)); - data->set("srs_version", SrsAmf0Any::str(RTMP_SIG_SRS_VERSION)); - data->set("srs_site", SrsAmf0Any::str(RTMP_SIG_SRS_WEB)); - data->set("srs_email", SrsAmf0Any::str(RTMP_SIG_SRS_EMAIL)); - data->set("srs_copyright", SrsAmf0Any::str(RTMP_SIG_SRS_COPYRIGHT)); - data->set("srs_primary", SrsAmf0Any::str(RTMP_SIG_SRS_PRIMARY)); - data->set("srs_authors", SrsAmf0Any::str(RTMP_SIG_SRS_AUTHROS)); - // for edge to directly get the id of client. - data->set("srs_pid", SrsAmf0Any::number(getpid())); - data->set("srs_id", SrsAmf0Any::number(_srs_context->get_id())); - - // local ip of edge - std::vector ips = srs_get_local_ipv4_ips(); - assert(_srs_config->get_stats_network() < (int)ips.size()); - std::string local_ip = ips[_srs_config->get_stats_network()]; - data->set("srs_server_ip", SrsAmf0Any::str(local_ip.c_str())); - - // generate the tcUrl - std::string param = ""; - std::string tc_url = srs_generate_tc_url(ep_server, req->vhost, req->app, ep_port, param); - - // upnode server identity will show in the connect_app of client. - // @see https://github.com/simple-rtmp-server/srs/issues/160 - // the debug_srs_upnode is config in vhost and default to true. - bool debug_srs_upnode = _srs_config->get_debug_srs_upnode(req->vhost); - if ((ret = client->connect_app(req->app, tc_url, req, debug_srs_upnode)) != ERROR_SUCCESS) { - srs_error("mpegts: connect with server failed, tcUrl=%s, dsu=%d. ret=%d", - tc_url.c_str(), debug_srs_upnode, ret); - return ret; - } - - return ret; -} - -void SrsDynamicHttpConn::close() -{ - srs_freep(client); - srs_freep(io); - srs_freep(req); - srs_close_stfd(stfd); -} - SrsHttpFileReader::SrsHttpFileReader(ISrsHttpResponseReader* h) { http = h; diff --git a/trunk/src/app/srs_app_caster_flv.hpp b/trunk/src/app/srs_app_caster_flv.hpp index cf0d11f86..9dd0f8550 100644 --- a/trunk/src/app/srs_app_caster_flv.hpp +++ b/trunk/src/app/srs_app_caster_flv.hpp @@ -43,6 +43,8 @@ class SrsRequest; class SrsPithyPrint; class ISrsHttpResponseReader; class SrsFlvDecoder; +class SrsTcpClient; +class SrsSimpleRtmpClient; #include #include @@ -84,12 +86,7 @@ class SrsDynamicHttpConn : public SrsHttpConn private: std::string output; SrsPithyPrint* pprint; -private: - SrsRequest* req; - st_netfd_t stfd; - SrsStSocket* io; - SrsRtmpClient* client; - int stream_id; + SrsSimpleRtmpClient* sdk; public: SrsDynamicHttpConn(IConnectionManager* cm, st_netfd_t fd, SrsHttpServeMux* m); virtual ~SrsDynamicHttpConn(); @@ -99,14 +96,6 @@ public: virtual int proxy(ISrsHttpResponseWriter* w, ISrsHttpMessage* r, std::string o); private: virtual int do_proxy(ISrsHttpResponseReader* rr, SrsFlvDecoder* dec); - virtual int rtmp_write_packet(char type, u_int32_t timestamp, char* data, int size); -private: - // connect to rtmp output url. - // @remark ignore when not connected, reconnect when disconnected. - virtual int connect(); - virtual int connect_app(std::string ep_server, std::string ep_port); - // close the connected io and rtmp to ready to be re-connect. - virtual void close(); }; /** diff --git a/trunk/src/app/srs_app_config.cpp b/trunk/src/app/srs_app_config.cpp index f0de5ac7a..477a3eca6 100644 --- a/trunk/src/app/srs_app_config.cpp +++ b/trunk/src/app/srs_app_config.cpp @@ -2128,6 +2128,8 @@ int SrsConfig::global_to_json(SrsJsonObject* obj) sobj->set(sdir->name, sdir->dumps_arg0_to_boolean()); } else if (sdir->name == "brokers") { sobj->set(sdir->name, sdir->dumps_args()); + } else if (sdir->name == "topic") { + sobj->set(sdir->name, sdir->dumps_arg0_to_str()); } } obj->set(dir->name, sobj); @@ -3546,7 +3548,7 @@ int SrsConfig::check_config() SrsConfDirective* conf = root->get("kafka"); for (int i = 0; conf && i < (int)conf->directives.size(); i++) { string n = conf->at(i)->name; - if (n != "enabled" && n != "brokers") { + if (n != "enabled" && n != "brokers" && n != "topic") { ret = ERROR_SYSTEM_CONFIG_INVALID; srs_error("unsupported kafka directive %s, ret=%d", n.c_str(), ret); return ret; @@ -4298,6 +4300,23 @@ SrsConfDirective* SrsConfig::get_kafka_brokers() return conf; } +string SrsConfig::get_kafka_topic() +{ + static string DEFAULT = "srs"; + + SrsConfDirective* conf = root->get("kafka"); + if (!conf) { + return DEFAULT; + } + + conf = conf->get("topic"); + if (!conf || conf->arg0().empty()) { + return DEFAULT; + } + + return conf->arg0(); +} + SrsConfDirective* SrsConfig::get_vhost(string vhost, bool try_default_vhost) { srs_assert(root); diff --git a/trunk/src/app/srs_app_config.hpp b/trunk/src/app/srs_app_config.hpp index 48cbd59e3..662187b1d 100644 --- a/trunk/src/app/srs_app_config.hpp +++ b/trunk/src/app/srs_app_config.hpp @@ -638,6 +638,10 @@ public: * get the broker list, each is format in . */ virtual SrsConfDirective* get_kafka_brokers(); + /** + * get the kafka topic to use for srs. + */ + virtual std::string get_kafka_topic(); // vhost specified section public: /** diff --git a/trunk/src/app/srs_app_dvr.cpp b/trunk/src/app/srs_app_dvr.cpp index 3b0971af8..4f161208f 100644 --- a/trunk/src/app/srs_app_dvr.cpp +++ b/trunk/src/app/srs_app_dvr.cpp @@ -107,7 +107,7 @@ int SrsFlvSegment::open(bool use_tmp_file) bool fresh_flv_file = !srs_path_exists(path); // create dir first. - std::string dir = path.substr(0, path.rfind("/")); + std::string dir = srs_path_dirname(path); if ((ret = srs_create_dir_recursively(dir)) != ERROR_SUCCESS) { srs_error("create dir=%s failed. ret=%d", dir.c_str(), ret); return ret; @@ -410,7 +410,7 @@ string SrsFlvSegment::generate_path() std::string path_config = _srs_config->get_dvr_path(req->vhost); // add [stream].[timestamp].flv as filename for dir - if (path_config.find(".flv") != path_config.length() - 4) { + if (!srs_string_ends_with(path_config, ".flv")) { path_config += "/[stream].[timestamp].flv"; } diff --git a/trunk/src/app/srs_app_edge.cpp b/trunk/src/app/srs_app_edge.cpp index 2ac192bb5..d859e91dd 100644 --- a/trunk/src/app/srs_app_edge.cpp +++ b/trunk/src/app/srs_app_edge.cpp @@ -45,6 +45,7 @@ using namespace std; #include #include #include +#include // when error, edge ingester sleep for a while and retry. #define SRS_EDGE_INGESTER_SLEEP_US (int64_t)(1*1000*1000LL) @@ -63,13 +64,11 @@ using namespace std; SrsEdgeIngester::SrsEdgeIngester() { - io = NULL; - kbps = new SrsKbps(); - client = NULL; - _edge = NULL; - _req = NULL; - stream_id = 0; - stfd = NULL; + source = NULL; + edge = NULL; + req = NULL; + + sdk = new SrsSimpleRtmpClient(); lb = new SrsLbRoundRobin(); pthread = new SrsReusableThread2("edge-igs", this, SRS_EDGE_INGESTER_SLEEP_US); } @@ -78,18 +77,18 @@ SrsEdgeIngester::~SrsEdgeIngester() { stop(); + srs_freep(sdk); srs_freep(lb); srs_freep(pthread); - srs_freep(kbps); } -int SrsEdgeIngester::initialize(SrsSource* source, SrsPlayEdge* edge, SrsRequest* req) +int SrsEdgeIngester::initialize(SrsSource* s, SrsPlayEdge* e, SrsRequest* r) { int ret = ERROR_SUCCESS; - _source = source; - _edge = edge; - _req = req; + source = s; + edge = e; + req = r; return ret; } @@ -98,7 +97,7 @@ int SrsEdgeIngester::start() { int ret = ERROR_SUCCESS; - if ((ret = _source->on_publish()) != ERROR_SUCCESS) { + if ((ret = source->on_publish()) != ERROR_SUCCESS) { srs_error("edge pull stream then publish to edge failed. ret=%d", ret); return ret; } @@ -109,15 +108,10 @@ int SrsEdgeIngester::start() void SrsEdgeIngester::stop() { pthread->stop(); - - close_underlayer_socket(); - - srs_freep(client); - srs_freep(io); - kbps->set_io(NULL, NULL); + sdk->close(); // notice to unpublish. - _source->on_unpublish(); + source->on_unpublish(); } string SrsEdgeIngester::get_curr_origin() @@ -129,39 +123,47 @@ int SrsEdgeIngester::cycle() { int ret = ERROR_SUCCESS; - _source->on_source_id_changed(_srs_context->get_id()); + source->on_source_id_changed(_srs_context->get_id()); - std::string ep_server; - int ep_port; - if ((ret = connect_server(ep_server, ep_port)) != ERROR_SUCCESS) { - return ret; + std::string url; + if (true) { + SrsConfDirective* conf = _srs_config->get_vhost_edge_origin(req->vhost); + + // @see https://github.com/simple-rtmp-server/srs/issues/79 + // when origin is error, for instance, server is shutdown, + // then user remove the vhost then reload, the conf is empty. + if (!conf) { + ret = ERROR_EDGE_VHOST_REMOVED; + srs_warn("vhost %s removed. ret=%d", req->vhost.c_str(), ret); + return ret; + } + + // select the origin. + std::string server = lb->select(conf->args); + int port = SRS_CONSTS_RTMP_DEFAULT_PORT; + srs_parse_hostport(server, server, port); + + // support vhost tranform for edge, + // @see https://github.com/simple-rtmp-server/srs/issues/372 + std::string vhost = _srs_config->get_vhost_edge_transform_vhost(req->vhost); + vhost = srs_string_replace(vhost, "[vhost]", req->vhost); + + url = srs_generate_rtmp_url(server, port, vhost, req->app, req->stream); } - srs_assert(client); - - client->set_recv_timeout(SRS_CONSTS_RTMP_RECV_TIMEOUT_US); - client->set_send_timeout(SRS_CONSTS_RTMP_SEND_TIMEOUT_US); - - SrsRequest* req = _req; - if ((ret = client->handshake()) != ERROR_SUCCESS) { - srs_error("handshake with server failed. ret=%d", ret); - return ret; - } - if ((ret = connect_app(ep_server, ep_port)) != ERROR_SUCCESS) { - return ret; - } - if ((ret = client->create_stream(stream_id)) != ERROR_SUCCESS) { - srs_error("connect with server failed, stream_id=%d. ret=%d", stream_id, ret); + int64_t cto = SRS_EDGE_INGESTER_TIMEOUT_US; + int64_t sto = SRS_CONSTS_RTMP_PULSE_TIMEOUT_US; + if ((ret = sdk->connect(url, cto, sto)) != ERROR_SUCCESS) { + srs_error("edge pull %s failed, cto=%"PRId64", sto=%"PRId64". ret=%d", url.c_str(), cto, sto, ret); return ret; } - if ((ret = client->play(req->stream, stream_id)) != ERROR_SUCCESS) { - srs_error("connect with server failed, stream=%s, stream_id=%d. ret=%d", - req->stream.c_str(), stream_id, ret); + if ((ret = sdk->play()) != ERROR_SUCCESS) { + srs_error("edge pull %s stream failed. ret=%d", url.c_str(), ret); return ret; } - if ((ret = _edge->on_ingest_play()) != ERROR_SUCCESS) { + if ((ret = edge->on_ingest_play()) != ERROR_SUCCESS) { return ret; } @@ -178,27 +180,20 @@ int SrsEdgeIngester::ingest() { int ret = ERROR_SUCCESS; - client->set_recv_timeout(SRS_EDGE_INGESTER_TIMEOUT_US); - SrsPithyPrint* pprint = SrsPithyPrint::create_edge(); SrsAutoFree(SrsPithyPrint, pprint); - + while (!pthread->interrupted()) { pprint->elapse(); // pithy print if (pprint->can_print()) { - kbps->sample(); - srs_trace("<- "SRS_CONSTS_LOG_EDGE_PLAY - " time=%"PRId64", okbps=%d,%d,%d, ikbps=%d,%d,%d", - pprint->age(), - kbps->get_send_kbps(), kbps->get_send_kbps_30s(), kbps->get_send_kbps_5m(), - kbps->get_recv_kbps(), kbps->get_recv_kbps_30s(), kbps->get_recv_kbps_5m()); + sdk->kbps_sample(SRS_CONSTS_LOG_EDGE_PLAY, pprint->age()); } - + // read from client. SrsCommonMessage* msg = NULL; - if ((ret = client->recv_message(&msg)) != ERROR_SUCCESS) { + if ((ret = sdk->recv_message(&msg)) != ERROR_SUCCESS) { if (!srs_is_client_gracefully_close(ret)) { srs_error("pull origin server message failed. ret=%d", ret); } @@ -213,68 +208,6 @@ int SrsEdgeIngester::ingest() return ret; } } - - return ret; -} - -// TODO: FIXME: refine the connect_app. -int SrsEdgeIngester::connect_app(string ep_server, int ep_port) -{ - int ret = ERROR_SUCCESS; - - SrsRequest* req = _req; - - // args of request takes the srs info. - if (req->args == NULL) { - req->args = SrsAmf0Any::object(); - } - - // notify server the edge identity, - // @see https://github.com/simple-rtmp-server/srs/issues/147 - SrsAmf0Object* data = req->args; - data->set("srs_sig", SrsAmf0Any::str(RTMP_SIG_SRS_KEY)); - data->set("srs_server", SrsAmf0Any::str(RTMP_SIG_SRS_SERVER)); - data->set("srs_license", SrsAmf0Any::str(RTMP_SIG_SRS_LICENSE)); - data->set("srs_role", SrsAmf0Any::str(RTMP_SIG_SRS_ROLE)); - data->set("srs_url", SrsAmf0Any::str(RTMP_SIG_SRS_URL)); - data->set("srs_version", SrsAmf0Any::str(RTMP_SIG_SRS_VERSION)); - data->set("srs_site", SrsAmf0Any::str(RTMP_SIG_SRS_WEB)); - data->set("srs_email", SrsAmf0Any::str(RTMP_SIG_SRS_EMAIL)); - data->set("srs_copyright", SrsAmf0Any::str(RTMP_SIG_SRS_COPYRIGHT)); - data->set("srs_primary", SrsAmf0Any::str(RTMP_SIG_SRS_PRIMARY)); - data->set("srs_authors", SrsAmf0Any::str(RTMP_SIG_SRS_AUTHROS)); - // for edge to directly get the id of client. - data->set("srs_pid", SrsAmf0Any::number(getpid())); - data->set("srs_id", SrsAmf0Any::number(_srs_context->get_id())); - - // local ip of edge - std::vector ips = srs_get_local_ipv4_ips(); - assert(_srs_config->get_stats_network() < (int)ips.size()); - std::string local_ip = ips[_srs_config->get_stats_network()]; - data->set("srs_server_ip", SrsAmf0Any::str(local_ip.c_str())); - - // support vhost tranform for edge, - // @see https://github.com/simple-rtmp-server/srs/issues/372 - std::string vhost = _srs_config->get_vhost_edge_transform_vhost(req->vhost); - vhost = srs_string_replace(vhost, "[vhost]", req->vhost); - // generate the tcUrl - std::string param = ""; - std::string tc_url = srs_generate_tc_url(ep_server, vhost, req->app, ep_port, param); - srs_trace("edge ingest from %s:%d at %s", ep_server.c_str(), ep_port, tc_url.c_str()); - - // replace the tcUrl in request, - // which will replace the tc_url in client.connect_app(). - req->tcUrl = tc_url; - - // upnode server identity will show in the connect_app of client. - // @see https://github.com/simple-rtmp-server/srs/issues/160 - // the debug_srs_upnode is config in vhost and default to true. - bool debug_srs_upnode = _srs_config->get_debug_srs_upnode(req->vhost); - if ((ret = client->connect_app(req->app, tc_url, req, debug_srs_upnode)) != ERROR_SUCCESS) { - srs_error("connect with server failed, tcUrl=%s, dsu=%d. ret=%d", - tc_url.c_str(), debug_srs_upnode, ret); - return ret; - } return ret; } @@ -282,8 +215,6 @@ int SrsEdgeIngester::connect_app(string ep_server, int ep_port) int SrsEdgeIngester::process_publish_message(SrsCommonMessage* msg) { int ret = ERROR_SUCCESS; - - SrsSource* source = _source; // process audio packet if (msg->header.is_audio()) { @@ -313,7 +244,7 @@ int SrsEdgeIngester::process_publish_message(SrsCommonMessage* msg) // process onMetaData if (msg->header.is_amf0_data() || msg->header.is_amf3_data()) { SrsPacket* pkt = NULL; - if ((ret = client->decode_message(msg, &pkt)) != ERROR_SUCCESS) { + if ((ret = sdk->decode_message(msg, &pkt)) != ERROR_SUCCESS) { srs_error("decode onMetaData message failed. ret=%d", ret); return ret; } @@ -336,82 +267,26 @@ int SrsEdgeIngester::process_publish_message(SrsCommonMessage* msg) return ret; } -void SrsEdgeIngester::close_underlayer_socket() -{ - srs_close_stfd(stfd); -} - -int SrsEdgeIngester::connect_server(string& ep_server, int& ep_port) -{ - int ret = ERROR_SUCCESS; - - // reopen - close_underlayer_socket(); - - SrsConfDirective* conf = _srs_config->get_vhost_edge_origin(_req->vhost); - - // @see https://github.com/simple-rtmp-server/srs/issues/79 - // when origin is error, for instance, server is shutdown, - // then user remove the vhost then reload, the conf is empty. - if (!conf) { - ret = ERROR_EDGE_VHOST_REMOVED; - srs_warn("vhost %s removed. ret=%d", _req->vhost.c_str(), ret); - return ret; - } - - // select the origin. - if (true) { - std::string server = lb->select(conf->args); - ep_port = SRS_CONSTS_RTMP_DEFAULT_PORT; - srs_parse_hostport(server, ep_server, ep_port); - } - - // open socket. - int64_t timeout = SRS_EDGE_INGESTER_TIMEOUT_US; - if ((ret = srs_socket_connect(ep_server, ep_port, timeout, &stfd)) != ERROR_SUCCESS) { - srs_warn("edge pull failed, stream=%s, tcUrl=%s to server=%s, port=%d, timeout=%"PRId64", ret=%d", - _req->stream.c_str(), _req->tcUrl.c_str(), ep_server.c_str(), ep_port, timeout, ret); - return ret; - } - - srs_freep(client); - srs_freep(io); - - srs_assert(stfd); - io = new SrsStSocket(stfd); - client = new SrsRtmpClient(io); - - kbps->set_io(io, io); - - srs_trace("edge pull connected, url=%s/%s, server=%s:%d", - _req->tcUrl.c_str(), _req->stream.c_str(), ep_server.c_str(), ep_port); - - return ret; -} - SrsEdgeForwarder::SrsEdgeForwarder() { - io = NULL; - kbps = new SrsKbps(); - client = NULL; - _edge = NULL; - _req = NULL; + edge = NULL; + req = NULL; + send_error_code = ERROR_SUCCESS; + + sdk = new SrsSimpleRtmpClient(); lb = new SrsLbRoundRobin(); - stream_id = 0; - stfd = NULL; pthread = new SrsReusableThread2("edge-fwr", this, SRS_EDGE_FORWARDER_SLEEP_US); queue = new SrsMessageQueue(); - send_error_code = ERROR_SUCCESS; } SrsEdgeForwarder::~SrsEdgeForwarder() { stop(); + srs_freep(sdk); srs_freep(lb); srs_freep(pthread); srs_freep(queue); - srs_freep(kbps); } void SrsEdgeForwarder::set_queue_size(double queue_size) @@ -419,13 +294,13 @@ void SrsEdgeForwarder::set_queue_size(double queue_size) return queue->set_queue_size(queue_size); } -int SrsEdgeForwarder::initialize(SrsSource* source, SrsPublishEdge* edge, SrsRequest* req) +int SrsEdgeForwarder::initialize(SrsSource* s, SrsPublishEdge* e, SrsRequest* r) { int ret = ERROR_SUCCESS; - _source = source; - _edge = edge; - _req = req; + source = s; + edge = e; + req = r; return ret; } @@ -434,36 +309,37 @@ int SrsEdgeForwarder::start() { int ret = ERROR_SUCCESS; + // reset the error code. send_error_code = ERROR_SUCCESS; - std::string ep_server; - int ep_port; - if ((ret = connect_server(ep_server, ep_port)) != ERROR_SUCCESS) { - return ret; + std::string url; + if (true) { + SrsConfDirective* conf = _srs_config->get_vhost_edge_origin(req->vhost); + srs_assert(conf); + + // select the origin. + std::string server = lb->select(conf->args); + int port = SRS_CONSTS_RTMP_DEFAULT_PORT; + srs_parse_hostport(server, server, port); + + // support vhost tranform for edge, + // @see https://github.com/simple-rtmp-server/srs/issues/372 + std::string vhost = _srs_config->get_vhost_edge_transform_vhost(req->vhost); + vhost = srs_string_replace(vhost, "[vhost]", req->vhost); + + url = srs_generate_rtmp_url(server, port, vhost, req->app, req->stream); } - srs_assert(client); - - client->set_recv_timeout(SRS_CONSTS_RTMP_RECV_TIMEOUT_US); - client->set_send_timeout(SRS_CONSTS_RTMP_SEND_TIMEOUT_US); - - SrsRequest* req = _req; - if ((ret = client->handshake()) != ERROR_SUCCESS) { - srs_error("handshake with server failed. ret=%d", ret); - return ret; - } - if ((ret = connect_app(ep_server, ep_port)) != ERROR_SUCCESS) { - srs_error("connect with server failed. ret=%d", ret); - return ret; - } - if ((ret = client->create_stream(stream_id)) != ERROR_SUCCESS) { - srs_error("connect with server failed, stream_id=%d. ret=%d", stream_id, ret); + // open socket. + int64_t cto = SRS_EDGE_FORWARDER_TIMEOUT_US; + int64_t sto = SRS_CONSTS_RTMP_TIMEOUT_US; + if ((ret = sdk->connect(url, cto, sto)) != ERROR_SUCCESS) { + srs_warn("edge push %s failed, cto=%"PRId64", sto=%"PRId64". ret=%d", url.c_str(), cto, sto, ret); return ret; } - if ((ret = client->publish(req->stream, stream_id)) != ERROR_SUCCESS) { - srs_error("publish failed, stream=%s, stream_id=%d. ret=%d", - req->stream.c_str(), stream_id, ret); + if ((ret = sdk->publish()) != ERROR_SUCCESS) { + srs_error("edge push publish failed. ret=%d", ret); return ret; } @@ -473,14 +349,8 @@ int SrsEdgeForwarder::start() void SrsEdgeForwarder::stop() { pthread->stop(); - - close_underlayer_socket(); - + sdk->close(); queue->clear(); - - srs_freep(client); - srs_freep(io); - kbps->set_io(NULL, NULL); } #define SYS_MAX_EDGE_SEND_MSGS 128 @@ -488,7 +358,7 @@ int SrsEdgeForwarder::cycle() { int ret = ERROR_SUCCESS; - client->set_recv_timeout(SRS_CONSTS_RTMP_PULSE_TIMEOUT_US); + sdk->set_recv_timeout(SRS_CONSTS_RTMP_PULSE_TIMEOUT_US); SrsPithyPrint* pprint = SrsPithyPrint::create_edge(); SrsAutoFree(SrsPithyPrint, pprint); @@ -504,7 +374,7 @@ int SrsEdgeForwarder::cycle() // read from client. if (true) { SrsCommonMessage* msg = NULL; - ret = client->recv_message(&msg); + ret = sdk->recv_message(&msg); srs_verbose("edge loop recv message. ret=%d", ret); if (ret != ERROR_SUCCESS && ret != ERROR_SOCKET_TIMEOUT) { @@ -528,22 +398,17 @@ int SrsEdgeForwarder::cycle() // pithy print if (pprint->can_print()) { - kbps->sample(); - srs_trace("-> "SRS_CONSTS_LOG_EDGE_PUBLISH - " time=%"PRId64", msgs=%d, okbps=%d,%d,%d, ikbps=%d,%d,%d", - pprint->age(), count, - kbps->get_send_kbps(), kbps->get_send_kbps_30s(), kbps->get_send_kbps_5m(), - kbps->get_recv_kbps(), kbps->get_recv_kbps_30s(), kbps->get_recv_kbps_5m()); + sdk->kbps_sample(SRS_CONSTS_LOG_EDGE_PUBLISH, pprint->age(), count); } // ignore when no messages. if (count <= 0) { - srs_verbose("no packets to push."); + srs_verbose("edge no packets to push."); continue; } // sendout messages, all messages are freed by send_and_free_messages(). - if ((ret = client->send_and_free_messages(msgs.msgs, count, stream_id)) != ERROR_SUCCESS) { + if ((ret = sdk->send_and_free_messages(msgs.msgs, count)) != ERROR_SUCCESS) { srs_error("edge publish push message to server failed. ret=%d", ret); return ret; } @@ -578,7 +443,7 @@ int SrsEdgeForwarder::proxy(SrsCommonMessage* msg) } srs_verbose("initialize shared ptr msg success."); - copy.stream_id = stream_id; + copy.stream_id = sdk->sid(); if ((ret = queue->enqueue(copy.copy())) != ERROR_SUCCESS) { srs_error("enqueue edge publish msg failed. ret=%d", ret); } @@ -586,114 +451,6 @@ int SrsEdgeForwarder::proxy(SrsCommonMessage* msg) return ret; } -void SrsEdgeForwarder::close_underlayer_socket() -{ - srs_close_stfd(stfd); -} - -int SrsEdgeForwarder::connect_server(string& ep_server, int& ep_port) -{ - int ret = ERROR_SUCCESS; - - // reopen - close_underlayer_socket(); - - SrsConfDirective* conf = _srs_config->get_vhost_edge_origin(_req->vhost); - srs_assert(conf); - - // select the origin. - if (true) { - std::string server = lb->select(conf->args); - ep_port = SRS_CONSTS_RTMP_DEFAULT_PORT; - srs_parse_hostport(server, ep_server, ep_port); - } - - // open socket. - int64_t timeout = SRS_EDGE_FORWARDER_TIMEOUT_US; - if ((ret = srs_socket_connect(ep_server, ep_port, timeout, &stfd)) != ERROR_SUCCESS) { - srs_warn("edge push failed, stream=%s, tcUrl=%s to server=%s, port=%d, timeout=%"PRId64", ret=%d", - _req->stream.c_str(), _req->tcUrl.c_str(), ep_server.c_str(), ep_port, timeout, ret); - return ret; - } - - srs_freep(client); - srs_freep(io); - - srs_assert(stfd); - io = new SrsStSocket(stfd); - client = new SrsRtmpClient(io); - - kbps->set_io(io, io); - - // open socket. - srs_trace("edge push connected, stream=%s, tcUrl=%s to server=%s, port=%d", - _req->stream.c_str(), _req->tcUrl.c_str(), ep_server.c_str(), ep_port); - - return ret; -} - -// TODO: FIXME: refine the connect_app. -int SrsEdgeForwarder::connect_app(string ep_server, int ep_port) -{ - int ret = ERROR_SUCCESS; - - SrsRequest* req = _req; - - // args of request takes the srs info. - if (req->args == NULL) { - req->args = SrsAmf0Any::object(); - } - - // notify server the edge identity, - // @see https://github.com/simple-rtmp-server/srs/issues/147 - SrsAmf0Object* data = req->args; - data->set("srs_sig", SrsAmf0Any::str(RTMP_SIG_SRS_KEY)); - data->set("srs_server", SrsAmf0Any::str(RTMP_SIG_SRS_SERVER)); - data->set("srs_license", SrsAmf0Any::str(RTMP_SIG_SRS_LICENSE)); - data->set("srs_role", SrsAmf0Any::str(RTMP_SIG_SRS_ROLE)); - data->set("srs_url", SrsAmf0Any::str(RTMP_SIG_SRS_URL)); - data->set("srs_version", SrsAmf0Any::str(RTMP_SIG_SRS_VERSION)); - data->set("srs_site", SrsAmf0Any::str(RTMP_SIG_SRS_WEB)); - data->set("srs_email", SrsAmf0Any::str(RTMP_SIG_SRS_EMAIL)); - data->set("srs_copyright", SrsAmf0Any::str(RTMP_SIG_SRS_COPYRIGHT)); - data->set("srs_primary", SrsAmf0Any::str(RTMP_SIG_SRS_PRIMARY)); - data->set("srs_authors", SrsAmf0Any::str(RTMP_SIG_SRS_AUTHROS)); - // for edge to directly get the id of client. - data->set("srs_pid", SrsAmf0Any::number(getpid())); - data->set("srs_id", SrsAmf0Any::number(_srs_context->get_id())); - - // local ip of edge - std::vector ips = srs_get_local_ipv4_ips(); - assert(_srs_config->get_stats_network() < (int)ips.size()); - std::string local_ip = ips[_srs_config->get_stats_network()]; - data->set("srs_server_ip", SrsAmf0Any::str(local_ip.c_str())); - - // support vhost tranform for edge, - // @see https://github.com/simple-rtmp-server/srs/issues/372 - std::string vhost = _srs_config->get_vhost_edge_transform_vhost(req->vhost); - vhost = srs_string_replace(vhost, "[vhost]", req->vhost); - // generate the tcUrl - std::string param = ""; - std::string tc_url = srs_generate_tc_url(ep_server, vhost, req->app, ep_port, param); - srs_trace("edge forward to %s:%d at %s", ep_server.c_str(), ep_port, tc_url.c_str()); - - // replace the tcUrl in request, - // which will replace the tc_url in client.connect_app(). - req->tcUrl = tc_url; - - // upnode server identity will show in the connect_app of client. - // @see https://github.com/simple-rtmp-server/srs/issues/160 - // the debug_srs_upnode is config in vhost and default to true. - bool debug_srs_upnode = _srs_config->get_debug_srs_upnode(req->vhost); - if ((ret = client->connect_app(req->app, tc_url, req, debug_srs_upnode)) != ERROR_SUCCESS) { - srs_error("connect with server failed, tcUrl=%s, dsu=%d. ret=%d", - tc_url.c_str(), debug_srs_upnode, ret); - return ret; - } - - return ret; -} - SrsPlayEdge::SrsPlayEdge() { state = SrsEdgeStateInit; diff --git a/trunk/src/app/srs_app_edge.hpp b/trunk/src/app/srs_app_edge.hpp index 3f05ce925..708a83f84 100644 --- a/trunk/src/app/srs_app_edge.hpp +++ b/trunk/src/app/srs_app_edge.hpp @@ -47,6 +47,8 @@ class SrsMessageQueue; class ISrsProtocolReaderWriter; class SrsKbps; class SrsLbRoundRobin; +class SrsTcpClient; +class SrsSimpleRtmpClient; /** * the state of edge, auto machine @@ -79,22 +81,17 @@ enum SrsEdgeUserState class SrsEdgeIngester : public ISrsReusableThread2Handler { private: - int stream_id; -private: - SrsSource* _source; - SrsPlayEdge* _edge; - SrsRequest* _req; + SrsSource* source; + SrsPlayEdge* edge; + SrsRequest* req; SrsReusableThread2* pthread; - st_netfd_t stfd; - ISrsProtocolReaderWriter* io; - SrsKbps* kbps; - SrsRtmpClient* client; + SrsSimpleRtmpClient* sdk; SrsLbRoundRobin* lb; public: SrsEdgeIngester(); virtual ~SrsEdgeIngester(); public: - virtual int initialize(SrsSource* source, SrsPlayEdge* edge, SrsRequest* req); + virtual int initialize(SrsSource* s, SrsPlayEdge* e, SrsRequest* r); virtual int start(); virtual void stop(); virtual std::string get_curr_origin(); @@ -103,9 +100,6 @@ public: virtual int cycle(); private: virtual int ingest(); - virtual void close_underlayer_socket(); - virtual int connect_server(std::string& ep_server, int& ep_port); - virtual int connect_app(std::string ep_server, int ep_port); virtual int process_publish_message(SrsCommonMessage* msg); }; @@ -115,16 +109,11 @@ private: class SrsEdgeForwarder : public ISrsReusableThread2Handler { private: - int stream_id; -private: - SrsSource* _source; - SrsPublishEdge* _edge; - SrsRequest* _req; + SrsSource* source; + SrsPublishEdge* edge; + SrsRequest* req; SrsReusableThread2* pthread; - st_netfd_t stfd; - ISrsProtocolReaderWriter* io; - SrsKbps* kbps; - SrsRtmpClient* client; + SrsSimpleRtmpClient* sdk; SrsLbRoundRobin* lb; /** * we must ensure one thread one fd principle, @@ -143,7 +132,7 @@ public: public: virtual void set_queue_size(double queue_size); public: - virtual int initialize(SrsSource* source, SrsPublishEdge* edge, SrsRequest* req); + virtual int initialize(SrsSource* s, SrsPublishEdge* e, SrsRequest* r); virtual int start(); virtual void stop(); // interface ISrsReusableThread2Handler @@ -151,10 +140,6 @@ public: virtual int cycle(); public: virtual int proxy(SrsCommonMessage* msg); -private: - virtual void close_underlayer_socket(); - virtual int connect_server(std::string& ep_server, int& ep_port); - virtual int connect_app(std::string ep_server, int ep_port); }; /** diff --git a/trunk/src/app/srs_app_ffmpeg.cpp b/trunk/src/app/srs_app_ffmpeg.cpp index cb28f0479..79f262916 100644 --- a/trunk/src/app/srs_app_ffmpeg.cpp +++ b/trunk/src/app/srs_app_ffmpeg.cpp @@ -219,7 +219,7 @@ int SrsFFMPEG::initialize_transcode(SrsConfDirective* engine) // for not rtmp input, donot append the iformat, // for example, "-f flv" before "-i udp://192.168.1.252:2222" // @see https://github.com/simple-rtmp-server/srs/issues/290 - if (input.find("rtmp://") != 0) { + if (!srs_string_starts_with(input, "rtmp://")) { iformat = ""; } diff --git a/trunk/src/app/srs_app_forward.cpp b/trunk/src/app/srs_app_forward.cpp index 21383b570..c2882be55 100644 --- a/trunk/src/app/srs_app_forward.cpp +++ b/trunk/src/app/srs_app_forward.cpp @@ -45,51 +45,47 @@ using namespace std; #include #include #include +#include // when error, forwarder sleep for a while and retry. #define SRS_FORWARDER_SLEEP_US (int64_t)(3*1000*1000LL) -SrsForwarder::SrsForwarder(SrsSource* _source) +SrsForwarder::SrsForwarder(SrsSource* s) { - source = _source; + source = s; - _req = NULL; - io = NULL; - client = NULL; - stfd = NULL; - kbps = new SrsKbps(); - stream_id = 0; + req = NULL; + sh_video = sh_audio = NULL; + sdk = new SrsSimpleRtmpClient(); pthread = new SrsReusableThread2("forward", this, SRS_FORWARDER_SLEEP_US); queue = new SrsMessageQueue(); jitter = new SrsRtmpJitter(); - - sh_video = sh_audio = NULL; } SrsForwarder::~SrsForwarder() { on_unpublish(); + srs_freep(sdk); srs_freep(pthread); srs_freep(queue); srs_freep(jitter); - srs_freep(kbps); srs_freep(sh_video); srs_freep(sh_audio); } -int SrsForwarder::initialize(SrsRequest* req, string ep_forward) +int SrsForwarder::initialize(SrsRequest* r, string ep) { int ret = ERROR_SUCCESS; // it's ok to use the request object, // SrsSource already copy it and never delete it. - _req = req; + req = r; // the ep(endpoint) to forward to - _ep_forward = ep_forward; + ep_forward = ep; return ret; } @@ -103,12 +99,17 @@ int SrsForwarder::on_publish() { int ret = ERROR_SUCCESS; - SrsRequest* req = _req; - // discovery the server port and tcUrl from req and ep_forward. - int port; - std::string server, tc_url; - discovery_ep(server, port, tc_url); + std::string server; + std::string tcUrl; + int port = SRS_CONSTS_RTMP_DEFAULT_PORT; + if (true) { + // parse host:port from hostport. + srs_parse_hostport(ep_forward, server, port); + + // generate tcUrl + tcUrl = srs_generate_tc_url(server, req->vhost, req->app, port, req->param); + } // dead loop check std::string source_ep = "rtmp://"; @@ -119,7 +120,7 @@ int SrsForwarder::on_publish() source_ep += req->vhost; std::string dest_ep = "rtmp://"; - if (_ep_forward == SRS_CONSTS_LOCALHOST) { + if (ep_forward == SRS_CONSTS_LOCALHOST) { dest_ep += req->host; } else { dest_ep += server; @@ -136,7 +137,7 @@ int SrsForwarder::on_publish() return ret; } srs_trace("start forward %s to %s, tcUrl=%s, stream=%s", - source_ep.c_str(), dest_ep.c_str(), tc_url.c_str(), + source_ep.c_str(), dest_ep.c_str(), tcUrl.c_str(), req->stream.c_str()); if ((ret = pthread->start()) != ERROR_SUCCESS) { @@ -151,12 +152,7 @@ int SrsForwarder::on_publish() void SrsForwarder::on_unpublish() { pthread->stop(); - - close_underlayer_socket(); - - srs_freep(client); - srs_freep(io); - kbps->set_io(NULL, NULL); + sdk->close(); } int SrsForwarder::on_meta_data(SrsSharedPtrMessage* shared_metadata) @@ -230,32 +226,26 @@ int SrsForwarder::cycle() { int ret = ERROR_SUCCESS; - std::string ep_server; - int ep_port; - if ((ret = connect_server(ep_server, ep_port)) != ERROR_SUCCESS) { - return ret; + std::string url; + if (true) { + std::string server; + int port = SRS_CONSTS_RTMP_DEFAULT_PORT; + + // parse host:port from hostport. + srs_parse_hostport(ep_forward, server, port); + + // generate url + url = srs_generate_rtmp_url(server, port, req->vhost, req->app, req->stream); } - srs_assert(client); - - client->set_recv_timeout(SRS_CONSTS_RTMP_RECV_TIMEOUT_US); - client->set_send_timeout(SRS_CONSTS_RTMP_SEND_TIMEOUT_US); - if ((ret = client->handshake()) != ERROR_SUCCESS) { - srs_error("handshake with server failed. ret=%d", ret); - return ret; - } - if ((ret = connect_app(ep_server, ep_port)) != ERROR_SUCCESS) { - srs_error("connect with server failed. ret=%d", ret); - return ret; - } - if ((ret = client->create_stream(stream_id)) != ERROR_SUCCESS) { - srs_error("connect with server failed, stream_id=%d. ret=%d", stream_id, ret); + int64_t cto = SRS_FORWARDER_SLEEP_US; + int64_t sto = SRS_CONSTS_RTMP_TIMEOUT_US; + if ((ret = sdk->connect(url, cto, sto)) != ERROR_SUCCESS) { + srs_warn("forward failed, url=%s, cto=%"PRId64", sto=%"PRId64". ret=%d", url.c_str(), cto, sto, ret); return ret; } - if ((ret = client->publish(_req->stream, stream_id)) != ERROR_SUCCESS) { - srs_error("connect with server failed, stream_name=%s, stream_id=%d. ret=%d", - _req->stream.c_str(), stream_id, ret); + if ((ret = sdk->publish()) != ERROR_SUCCESS) { return ret; } @@ -271,115 +261,12 @@ int SrsForwarder::cycle() return ret; } -void SrsForwarder::close_underlayer_socket() -{ - srs_close_stfd(stfd); -} - -void SrsForwarder::discovery_ep(string& server, int& port, string& tc_url) -{ - SrsRequest* req = _req; - - port = SRS_CONSTS_RTMP_DEFAULT_PORT; - srs_parse_hostport(_ep_forward, server, port); - - // generate tcUrl - tc_url = srs_generate_tc_url(server, req->vhost, req->app, port, req->param); -} - -int SrsForwarder::connect_server(string& ep_server, int& ep_port) -{ - int ret = ERROR_SUCCESS; - - // reopen - close_underlayer_socket(); - - // discovery the server port and tcUrl from req and ep_forward. - string tc_url; - discovery_ep(ep_server, ep_port, tc_url); - - // open socket. - int64_t timeout = SRS_FORWARDER_SLEEP_US; - if ((ret = srs_socket_connect(ep_server, ep_port, timeout, &stfd)) != ERROR_SUCCESS) { - srs_warn("forward failed, stream=%s, tcUrl=%s to server=%s, port=%d, timeout=%"PRId64", ret=%d", - _req->stream.c_str(), _req->tcUrl.c_str(), ep_server.c_str(), ep_port, timeout, ret); - return ret; - } - - srs_freep(client); - srs_freep(io); - - srs_assert(stfd); - io = new SrsStSocket(stfd); - client = new SrsRtmpClient(io); - - kbps->set_io(io, io); - - srs_trace("forward connected, stream=%s, tcUrl=%s to server=%s, port=%d", - _req->stream.c_str(), _req->tcUrl.c_str(), ep_server.c_str(), ep_port); - - return ret; -} - -// TODO: FIXME: refine the connect_app. -int SrsForwarder::connect_app(string ep_server, int ep_port) -{ - int ret = ERROR_SUCCESS; - - SrsRequest* req = _req; - - // args of request takes the srs info. - if (req->args == NULL) { - req->args = SrsAmf0Any::object(); - } - - // notify server the edge identity, - // @see https://github.com/simple-rtmp-server/srs/issues/147 - SrsAmf0Object* data = req->args; - data->set("srs_sig", SrsAmf0Any::str(RTMP_SIG_SRS_KEY)); - data->set("srs_server", SrsAmf0Any::str(RTMP_SIG_SRS_SERVER)); - data->set("srs_license", SrsAmf0Any::str(RTMP_SIG_SRS_LICENSE)); - data->set("srs_role", SrsAmf0Any::str(RTMP_SIG_SRS_ROLE)); - data->set("srs_url", SrsAmf0Any::str(RTMP_SIG_SRS_URL)); - data->set("srs_version", SrsAmf0Any::str(RTMP_SIG_SRS_VERSION)); - data->set("srs_site", SrsAmf0Any::str(RTMP_SIG_SRS_WEB)); - data->set("srs_email", SrsAmf0Any::str(RTMP_SIG_SRS_EMAIL)); - data->set("srs_copyright", SrsAmf0Any::str(RTMP_SIG_SRS_COPYRIGHT)); - data->set("srs_primary", SrsAmf0Any::str(RTMP_SIG_SRS_PRIMARY)); - data->set("srs_authors", SrsAmf0Any::str(RTMP_SIG_SRS_AUTHROS)); - // for edge to directly get the id of client. - data->set("srs_pid", SrsAmf0Any::number(getpid())); - data->set("srs_id", SrsAmf0Any::number(_srs_context->get_id())); - - // local ip of edge - std::vector ips = srs_get_local_ipv4_ips(); - assert(_srs_config->get_stats_network() < (int)ips.size()); - std::string local_ip = ips[_srs_config->get_stats_network()]; - data->set("srs_server_ip", SrsAmf0Any::str(local_ip.c_str())); - - // generate the tcUrl - std::string param = ""; - std::string tc_url = srs_generate_tc_url(ep_server, req->vhost, req->app, ep_port, param); - - // upnode server identity will show in the connect_app of client. - // @see https://github.com/simple-rtmp-server/srs/issues/160 - // the debug_srs_upnode is config in vhost and default to true. - bool debug_srs_upnode = _srs_config->get_debug_srs_upnode(req->vhost); - if ((ret = client->connect_app(req->app, tc_url, req, debug_srs_upnode)) != ERROR_SUCCESS) { - srs_error("connect with server failed, tcUrl=%s, dsu=%d. ret=%d", - tc_url.c_str(), debug_srs_upnode, ret); - return ret; - } - - return ret; -} - #define SYS_MAX_FORWARD_SEND_MSGS 128 int SrsForwarder::forward() { int ret = ERROR_SUCCESS; - client->set_recv_timeout(SRS_CONSTS_RTMP_PULSE_TIMEOUT_US); + sdk->set_recv_timeout(SRS_CONSTS_RTMP_PULSE_TIMEOUT_US); SrsPithyPrint* pprint = SrsPithyPrint::create_forwarder(); SrsAutoFree(SrsPithyPrint, pprint); @@ -389,13 +276,13 @@ int SrsForwarder::forward() // update sequence header // TODO: FIXME: maybe need to zero the sequence header timestamp. if (sh_video) { - if ((ret = client->send_and_free_message(sh_video->copy(), stream_id)) != ERROR_SUCCESS) { + if ((ret = sdk->send_and_free_message(sh_video->copy())) != ERROR_SUCCESS) { srs_error("forwarder send sh_video to server failed. ret=%d", ret); return ret; } } if (sh_audio) { - if ((ret = client->send_and_free_message(sh_audio->copy(), stream_id)) != ERROR_SUCCESS) { + if ((ret = sdk->send_and_free_message(sh_audio->copy())) != ERROR_SUCCESS) { srs_error("forwarder send sh_audio to server failed. ret=%d", ret); return ret; } @@ -407,7 +294,7 @@ int SrsForwarder::forward() // read from client. if (true) { SrsCommonMessage* msg = NULL; - ret = client->recv_message(&msg); + ret = sdk->recv_message(&msg); srs_verbose("play loop recv message. ret=%d", ret); if (ret != ERROR_SUCCESS && ret != ERROR_SOCKET_TIMEOUT) { @@ -428,12 +315,7 @@ int SrsForwarder::forward() // pithy print if (pprint->can_print()) { - kbps->sample(); - srs_trace("-> "SRS_CONSTS_LOG_FOWARDER - " time=%"PRId64", msgs=%d, okbps=%d,%d,%d, ikbps=%d,%d,%d", - pprint->age(), count, - kbps->get_send_kbps(), kbps->get_send_kbps_30s(), kbps->get_send_kbps_5m(), - kbps->get_recv_kbps(), kbps->get_recv_kbps_30s(), kbps->get_recv_kbps_5m()); + sdk->kbps_sample(SRS_CONSTS_LOG_FOWARDER, pprint->age(), count); } // ignore when no messages. @@ -443,7 +325,7 @@ int SrsForwarder::forward() } // sendout messages, all messages are freed by send_and_free_messages(). - if ((ret = client->send_and_free_messages(msgs.msgs, count, stream_id)) != ERROR_SUCCESS) { + if ((ret = sdk->send_and_free_messages(msgs.msgs, count)) != ERROR_SUCCESS) { srs_error("forwarder messages to server failed. ret=%d", ret); return ret; } diff --git a/trunk/src/app/srs_app_forward.hpp b/trunk/src/app/srs_app_forward.hpp index 5842f0ac6..487cd2b83 100644 --- a/trunk/src/app/srs_app_forward.hpp +++ b/trunk/src/app/srs_app_forward.hpp @@ -43,6 +43,7 @@ class SrsRtmpClient; class SrsRequest; class SrsSource; class SrsKbps; +class SrsSimpleRtmpClient; /** * forward the stream to other servers. @@ -52,17 +53,13 @@ class SrsForwarder : public ISrsReusableThread2Handler { private: // the ep to forward, server[:port]. - std::string _ep_forward; - SrsRequest* _req; - int stream_id; + std::string ep_forward; + SrsRequest* req; private: - st_netfd_t stfd; SrsReusableThread2* pthread; private: SrsSource* source; - ISrsProtocolReaderWriter* io; - SrsKbps* kbps; - SrsRtmpClient* client; + SrsSimpleRtmpClient* sdk; SrsRtmpJitter* jitter; SrsMessageQueue* queue; /** @@ -75,7 +72,7 @@ public: SrsForwarder(SrsSource* _source); virtual ~SrsForwarder(); public: - virtual int initialize(SrsRequest* req, std::string ep_forward); + virtual int initialize(SrsRequest* r, std::string ep); virtual void set_queue_size(double queue_size); public: virtual int on_publish(); @@ -99,10 +96,6 @@ public: public: virtual int cycle(); private: - virtual void close_underlayer_socket(); - virtual void discovery_ep(std::string& server, int& port, std::string& tc_url); - virtual int connect_server(std::string& ep_server, int& ep_port); - virtual int connect_app(std::string ep_server, int ep_port); virtual int forward(); }; diff --git a/trunk/src/app/srs_app_http_client.cpp b/trunk/src/app/srs_app_http_client.cpp index 08f4b2b0c..1bf9ff185 100644 --- a/trunk/src/app/srs_app_http_client.cpp +++ b/trunk/src/app/srs_app_http_client.cpp @@ -39,9 +39,7 @@ using namespace std; SrsHttpClient::SrsHttpClient() { - connected = false; - stfd = NULL; - skt = NULL; + transport = new SrsTcpClient(); parser = NULL; timeout_us = 0; port = 0; @@ -50,6 +48,8 @@ SrsHttpClient::SrsHttpClient() SrsHttpClient::~SrsHttpClient() { disconnect(); + + srs_freep(transport); srs_freep(parser); } @@ -102,7 +102,7 @@ int SrsHttpClient::post(string path, string req, ISrsHttpMessage** ppmsg) << req; std::string data = ss.str(); - if ((ret = skt->write((void*)data.c_str(), data.length(), NULL)) != ERROR_SUCCESS) { + if ((ret = transport->write((void*)data.c_str(), data.length(), NULL)) != ERROR_SUCCESS) { // disconnect when error. disconnect(); @@ -111,7 +111,7 @@ int SrsHttpClient::post(string path, string req, ISrsHttpMessage** ppmsg) } ISrsHttpMessage* msg = NULL; - if ((ret = parser->parse_message(skt, NULL, &msg)) != ERROR_SUCCESS) { + if ((ret = parser->parse_message(transport, NULL, &msg)) != ERROR_SUCCESS) { srs_error("parse http post response failed. ret=%d", ret); return ret; } @@ -148,7 +148,7 @@ int SrsHttpClient::get(string path, std::string req, ISrsHttpMessage** ppmsg) << req; std::string data = ss.str(); - if ((ret = skt->write((void*)data.c_str(), data.length(), NULL)) != ERROR_SUCCESS) { + if ((ret = transport->write((void*)data.c_str(), data.length(), NULL)) != ERROR_SUCCESS) { // disconnect when error. disconnect(); @@ -157,7 +157,7 @@ int SrsHttpClient::get(string path, std::string req, ISrsHttpMessage** ppmsg) } ISrsHttpMessage* msg = NULL; - if ((ret = parser->parse_message(skt, NULL, &msg)) != ERROR_SUCCESS) { + if ((ret = parser->parse_message(transport, NULL, &msg)) != ERROR_SUCCESS) { srs_error("parse http post response failed. ret=%d", ret); return ret; } @@ -171,37 +171,30 @@ int SrsHttpClient::get(string path, std::string req, ISrsHttpMessage** ppmsg) void SrsHttpClient::disconnect() { - connected = false; - - srs_close_stfd(stfd); - srs_freep(skt); + transport->close(); } int SrsHttpClient::connect() { int ret = ERROR_SUCCESS; - if (connected) { + if (transport->connected()) { return ret; } disconnect(); // open socket. - if ((ret = srs_socket_connect(host, port, timeout_us, &stfd)) != ERROR_SUCCESS) { + if ((ret = transport->connect(host, port, timeout_us)) != ERROR_SUCCESS) { srs_warn("http client failed, server=%s, port=%d, timeout=%"PRId64", ret=%d", host.c_str(), port, timeout_us, ret); return ret; } srs_info("connect to server success. server=%s, port=%d", host.c_str(), port); - srs_assert(!skt); - skt = new SrsStSocket(stfd); - connected = true; - // set the recv/send timeout in us. - skt->set_recv_timeout(timeout_us); - skt->set_send_timeout(timeout_us); + transport->set_recv_timeout(timeout_us); + transport->set_send_timeout(timeout_us); return ret; } diff --git a/trunk/src/app/srs_app_http_client.hpp b/trunk/src/app/srs_app_http_client.hpp index b05def7ca..c905277ee 100644 --- a/trunk/src/app/srs_app_http_client.hpp +++ b/trunk/src/app/srs_app_http_client.hpp @@ -49,9 +49,7 @@ class SrsStSocket; class SrsHttpClient { private: - bool connected; - st_netfd_t stfd; - SrsStSocket* skt; + SrsTcpClient* transport; SrsHttpParser* parser; private: int64_t timeout_us; diff --git a/trunk/src/app/srs_app_http_conn.cpp b/trunk/src/app/srs_app_http_conn.cpp index a64a953ab..786e99dd0 100644 --- a/trunk/src/app/srs_app_http_conn.cpp +++ b/trunk/src/app/srs_app_http_conn.cpp @@ -303,7 +303,7 @@ int SrsHttpResponseWriter::send_header(char* data, int size) return skt->write((void*)buf.c_str(), buf.length(), NULL); } -SrsHttpResponseReader::SrsHttpResponseReader(SrsHttpMessage* msg, SrsStSocket* io) +SrsHttpResponseReader::SrsHttpResponseReader(SrsHttpMessage* msg, ISrsProtocolReaderWriter* io) { skt = io; owner = msg; @@ -494,7 +494,7 @@ int SrsHttpResponseReader::read_specified(char* data, int nb_data, int* nb_read) return ret; } -SrsHttpMessage::SrsHttpMessage(SrsStSocket* io, SrsConnection* c) : ISrsHttpMessage() +SrsHttpMessage::SrsHttpMessage(ISrsProtocolReaderWriter* io, SrsConnection* c) : ISrsHttpMessage() { conn = c; chunked = false; @@ -572,12 +572,7 @@ int SrsHttpMessage::update(string url, bool allow_jsonp, http_parser* header, Sr } // parse ext. - _ext = _uri->get_path(); - if ((pos = _ext.rfind(".")) != string::npos) { - _ext = _ext.substr(pos); - } else { - _ext = ""; - } + _ext = srs_path_filext(_uri->get_path()); // parse jsonp request message. if (allow_jsonp) { @@ -816,23 +811,23 @@ SrsRequest* SrsHttpMessage::to_request(string vhost) { SrsRequest* req = new SrsRequest(); - req->app = _uri->get_path(); - size_t pos = string::npos; - if ((pos = req->app.rfind("/")) != string::npos) { - req->stream = req->app.substr(pos + 1); - req->app = req->app.substr(0, pos); - } - if ((pos = req->stream.rfind(".")) != string::npos) { - req->stream = req->stream.substr(0, pos); - } + // http path, for instance, /live/livestream.flv, parse to + // app: /live + // stream: livestream.flv + srs_parse_rtmp_url(_uri->get_path(), req->app, req->stream); - req->tcUrl = "rtmp://" + vhost + req->app; + // trim the start slash, for instance, /live to live + req->app = srs_string_trim_start(req->app, "/"); + + // remove the extension, for instance, livestream.flv to livestream + req->stream = srs_path_filename(req->stream); + + // generate others. + req->tcUrl = "rtmp://" + vhost + "/" + req->app; req->pageUrl = get_request_header("Referer"); req->objectEncoding = 0; - srs_discovery_tc_url(req->tcUrl, - req->schema, req->host, req->vhost, req->app, req->port, - req->param); + srs_discovery_tc_url(req->tcUrl, req->schema, req->host, req->vhost, req->app, req->port, req->param); req->strip(); return req; @@ -875,7 +870,7 @@ int SrsHttpParser::initialize(enum http_parser_type type, bool allow_jsonp) return ret; } -int SrsHttpParser::parse_message(SrsStSocket* skt, SrsConnection* conn, ISrsHttpMessage** ppmsg) +int SrsHttpParser::parse_message(ISrsProtocolReaderWriter* io, SrsConnection* conn, ISrsHttpMessage** ppmsg) { *ppmsg = NULL; @@ -892,7 +887,7 @@ int SrsHttpParser::parse_message(SrsStSocket* skt, SrsConnection* conn, ISrsHttp header_parsed = 0; // do parse - if ((ret = parse_message_imp(skt)) != ERROR_SUCCESS) { + if ((ret = parse_message_imp(io)) != ERROR_SUCCESS) { if (!srs_is_client_gracefully_close(ret)) { srs_error("parse http msg failed. ret=%d", ret); } @@ -900,7 +895,7 @@ int SrsHttpParser::parse_message(SrsStSocket* skt, SrsConnection* conn, ISrsHttp } // create msg - SrsHttpMessage* msg = new SrsHttpMessage(skt, conn); + SrsHttpMessage* msg = new SrsHttpMessage(io, conn); // initalize http msg, parse url. if ((ret = msg->update(url, jsonp, &header, buffer, headers)) != ERROR_SUCCESS) { @@ -915,7 +910,7 @@ int SrsHttpParser::parse_message(SrsStSocket* skt, SrsConnection* conn, ISrsHttp return ret; } -int SrsHttpParser::parse_message_imp(SrsStSocket* skt) +int SrsHttpParser::parse_message_imp(ISrsProtocolReaderWriter* io) { int ret = ERROR_SUCCESS; @@ -949,7 +944,7 @@ int SrsHttpParser::parse_message_imp(SrsStSocket* skt) // when nothing parsed, read more to parse. if (nparsed == 0) { // when requires more, only grow 1bytes, but the buffer will cache more. - if ((ret = buffer->grow(skt, buffer->size() + 1)) != ERROR_SUCCESS) { + if ((ret = buffer->grow(io, buffer->size() + 1)) != ERROR_SUCCESS) { if (!srs_is_client_gracefully_close(ret)) { srs_error("read body from server failed. ret=%d", ret); } @@ -1114,19 +1109,19 @@ int SrsHttpUri::initialize(string _url) return ret; } -const char* SrsHttpUri::get_url() +string SrsHttpUri::get_url() { - return url.data(); + return url; } -const char* SrsHttpUri::get_schema() +string SrsHttpUri::get_schema() { - return schema.data(); + return schema; } -const char* SrsHttpUri::get_host() +string SrsHttpUri::get_host() { - return host.data(); + return host; } int SrsHttpUri::get_port() @@ -1134,14 +1129,14 @@ int SrsHttpUri::get_port() return port; } -const char* SrsHttpUri::get_path() +string SrsHttpUri::get_path() { - return path.data(); + return path; } -const char* SrsHttpUri::get_query() +string SrsHttpUri::get_query() { - return query.data(); + return query; } string SrsHttpUri::get_uri_field(string uri, http_parser_url* hp_u, http_parser_url_fields field) diff --git a/trunk/src/app/srs_app_http_conn.hpp b/trunk/src/app/srs_app_http_conn.hpp index 64f826bc5..2fab8765d 100644 --- a/trunk/src/app/srs_app_http_conn.hpp +++ b/trunk/src/app/srs_app_http_conn.hpp @@ -120,7 +120,7 @@ public: class SrsHttpResponseReader : virtual public ISrsHttpResponseReader { private: - SrsStSocket* skt; + ISrsProtocolReaderWriter* skt; SrsHttpMessage* owner; SrsFastStream* buffer; bool is_eof; @@ -131,7 +131,7 @@ private: // already read total bytes. int64_t nb_total_read; public: - SrsHttpResponseReader(SrsHttpMessage* msg, SrsStSocket* io); + SrsHttpResponseReader(SrsHttpMessage* msg, ISrsProtocolReaderWriter* io); virtual ~SrsHttpResponseReader(); public: /** @@ -208,7 +208,7 @@ private: // the method in QueryString will override the HTTP method. std::string jsonp_method; public: - SrsHttpMessage(SrsStSocket* io, SrsConnection* c); + SrsHttpMessage(ISrsProtocolReaderWriter* io, SrsConnection* c); virtual ~SrsHttpMessage(); public: /** @@ -332,12 +332,12 @@ public: * or error and *ppmsg must be NULL. * @remark, if success, *ppmsg always NOT-NULL, *ppmsg always is_complete(). */ - virtual int parse_message(SrsStSocket* skt, SrsConnection* conn, ISrsHttpMessage** ppmsg); + virtual int parse_message(ISrsProtocolReaderWriter* io, SrsConnection* conn, ISrsHttpMessage** ppmsg); private: /** * parse the HTTP message to member field: msg. */ - virtual int parse_message_imp(SrsStSocket* skt); + virtual int parse_message_imp(ISrsProtocolReaderWriter* io); private: static int on_message_begin(http_parser* parser); static int on_headers_complete(http_parser* parser); @@ -369,12 +369,12 @@ public: */ virtual int initialize(std::string _url); public: - virtual const char* get_url(); - virtual const char* get_schema(); - virtual const char* get_host(); + virtual std::string get_url(); + virtual std::string get_schema(); + virtual std::string get_host(); virtual int get_port(); - virtual const char* get_path(); - virtual const char* get_query(); + virtual std::string get_path(); + virtual std::string get_query(); private: /** * get the parsed url field. diff --git a/trunk/src/app/srs_app_http_static.cpp b/trunk/src/app/srs_app_http_static.cpp index 18a3869a8..6304d864e 100644 --- a/trunk/src/app/srs_app_http_static.cpp +++ b/trunk/src/app/srs_app_http_static.cpp @@ -242,7 +242,7 @@ int SrsHttpStaticServer::initialize() mount = srs_string_replace(mount, SRS_CONSTS_RTMP_DEFAULT_VHOST"/", "/"); // the dir mount must always ends with "/" - if (mount != "/" && mount.rfind("/") != mount.length() - 1) { + if (mount != "/" && !srs_string_ends_with(mount, "/")) { mount += "/"; } diff --git a/trunk/src/app/srs_app_http_stream.cpp b/trunk/src/app/srs_app_http_stream.cpp index 204e9eb62..3f3c4f7f1 100644 --- a/trunk/src/app/srs_app_http_stream.cpp +++ b/trunk/src/app/srs_app_http_stream.cpp @@ -611,11 +611,7 @@ SrsLiveEntry::SrsLiveEntry(std::string m, bool h) req = NULL; source = NULL; - std::string ext; - size_t pos = string::npos; - if ((pos = m.rfind(".")) != string::npos) { - ext = m.substr(pos); - } + std::string ext = srs_path_filext(m); _is_flv = (ext == ".flv"); _is_ts = (ext == ".ts"); _is_mp3 = (ext == ".mp3"); @@ -1319,10 +1315,7 @@ string SrsHttpStreamServer::hls_mount_generate(SrsRequest* r, string uri, string std::string mount = tmpl; // the ts is relative from the m3u8, the same start dir. - size_t pos = string::npos; - if ((pos = mount.rfind("/")) != string::npos) { - mount = mount.substr(0, pos); - } + mount = srs_path_dirname(mount); // replace the vhost variable mount = srs_string_replace(mount, "[vhost]", r->vhost); diff --git a/trunk/src/app/srs_app_ingest.cpp b/trunk/src/app/srs_app_ingest.cpp index fab6647cf..512aa7132 100644 --- a/trunk/src/app/srs_app_ingest.cpp +++ b/trunk/src/app/srs_app_ingest.cpp @@ -35,6 +35,7 @@ using namespace std; #include #include #include +#include // when error, ingester sleep for a while and retry. // ingest never sleep a long time, for we must start the stream ASAP. @@ -354,17 +355,9 @@ int SrsIngester::initialize_ffmpeg(SrsFFMPEG* ffmpeg, SrsConfDirective* vhost, S } // find the app and stream in rtmp url - std::string url = output; std::string app, stream; - size_t pos = std::string::npos; - if ((pos = url.rfind("/")) != std::string::npos) { - stream = url.substr(pos + 1); - url = url.substr(0, pos); - } - if ((pos = url.rfind("/")) != std::string::npos) { - app = url.substr(pos + 1); - url = url.substr(0, pos); - } + srs_parse_rtmp_url(output, app, stream); + size_t pos; if ((pos = app.rfind("?")) != std::string::npos) { app = app.substr(0, pos); } diff --git a/trunk/src/app/srs_app_kafka.cpp b/trunk/src/app/srs_app_kafka.cpp index 9eb821941..89491f742 100644 --- a/trunk/src/app/srs_app_kafka.cpp +++ b/trunk/src/app/srs_app_kafka.cpp @@ -32,33 +32,328 @@ using namespace std; #include #include #include +#include #include +#include +#include +#include #ifdef SRS_AUTO_KAFKA +#define SRS_KAKFA_CYCLE_INTERVAL_MS 3000 +#define SRS_KAFKA_PRODUCER_TIMEOUT 30000 +#define SRS_KAFKA_PRODUCER_AGGREGATE_SIZE 1 + +std::string srs_kafka_metadata_summary(SrsKafkaTopicMetadataResponse* metadata) +{ + vector bs; + for (int i = 0; i < metadata->brokers.size(); i++) { + SrsKafkaBroker* broker = metadata->brokers.at(i); + + string hostport = srs_int2str(broker->node_id) + "/" + broker->host.to_str(); + if (broker->port > 0) { + hostport += ":" + srs_int2str(broker->port); + } + + bs.push_back(hostport); + } + + vector ps; + for (int i = 0; i < metadata->metadatas.size(); i++) { + SrsKafkaTopicMetadata* topic = metadata->metadatas.at(i); + + for (int j = 0; j < topic->metadatas.size(); j++) { + string desc = "topic=" + topic->name.to_str(); + + SrsKafkaPartitionMetadata* partition = topic->metadatas.at(j); + + desc += "?partition=" + srs_int2str(partition->partition_id); + desc += "&leader=" + srs_int2str(partition->leader); + + vector replicas = srs_kafka_array2vector(&partition->replicas); + desc += "&replicas=" + srs_join_vector_string(replicas, ","); + + ps.push_back(desc); + } + } + + std::stringstream ss; + ss << "brokers=" << srs_join_vector_string(bs, ","); + ss << ", " << srs_join_vector_string(ps, ", "); + + return ss.str(); +} + +std::string srs_kafka_summary_partitions(const vector& partitions) +{ + vector ret; + + vector::const_iterator it; + for (it = partitions.begin(); it != partitions.end(); ++it) { + SrsKafkaPartition* partition = *it; + + string desc = "tcp://"; + desc += partition->host + ":" + srs_int2str(partition->port); + desc += "?broker=" + srs_int2str(partition->broker); + desc += "&partition=" + srs_int2str(partition->id); + ret.push_back(desc); + } + + return srs_join_vector_string(ret, ", "); +} + +void srs_kafka_metadata2connector(string topic_name, SrsKafkaTopicMetadataResponse* metadata, vector& partitions) +{ + for (int i = 0; i < metadata->metadatas.size(); i++) { + SrsKafkaTopicMetadata* topic = metadata->metadatas.at(i); + + for (int j = 0; j < topic->metadatas.size(); j++) { + SrsKafkaPartitionMetadata* partition = topic->metadatas.at(j); + + SrsKafkaPartition* p = new SrsKafkaPartition(); + + p->topic = topic_name; + p->id = partition->partition_id; + p->broker = partition->leader; + + for (int i = 0; i < metadata->brokers.size(); i++) { + SrsKafkaBroker* broker = metadata->brokers.at(i); + if (broker->node_id == p->broker) { + p->host = broker->host.to_str(); + p->port = broker->port; + break; + } + } + + partitions.push_back(p); + } + } +} + +SrsKafkaPartition::SrsKafkaPartition() +{ + id = broker = 0; + port = SRS_CONSTS_KAFKA_DEFAULT_PORT; + + transport = new SrsTcpClient(); + kafka = new SrsKafkaClient(transport); +} + +SrsKafkaPartition::~SrsKafkaPartition() +{ + srs_freep(kafka); + srs_freep(transport); +} + +string SrsKafkaPartition::hostport() +{ + if (ep.empty()) { + ep = host + ":" + srs_int2str(port); + } + + return ep; +} + +int SrsKafkaPartition::connect() +{ + int ret = ERROR_SUCCESS; + + if (transport->connected()) { + return ret; + } + + int64_t timeout = SRS_KAFKA_PRODUCER_TIMEOUT * 1000; + if ((ret = transport->connect(host, port, timeout)) != ERROR_SUCCESS) { + srs_error("connect to %s partition=%d failed, timeout=%"PRId64". ret=%d", hostport().c_str(), id, timeout, ret); + return ret; + } + + srs_trace("connect at %s, partition=%d, broker=%d", hostport().c_str(), id, broker); + + return ret; +} + +int SrsKafkaPartition::flush(SrsKafkaPartitionCache* pc) +{ + return kafka->write_messages(topic, id, *pc); +} + +SrsKafkaMessage::SrsKafkaMessage(SrsKafkaProducer* p, int k, SrsJsonObject* j) +{ + producer = p; + key = k; + obj = j; +} + +SrsKafkaMessage::~SrsKafkaMessage() +{ + srs_freep(obj); +} + +int SrsKafkaMessage::call() +{ + int ret = producer->send(key, obj); + + // the obj is manged by producer now. + obj = NULL; + + return ret; +} + +string SrsKafkaMessage::to_string() +{ + return "kafka"; +} + +SrsKafkaCache::SrsKafkaCache() +{ + count = 0; + nb_partitions = 0; +} + +SrsKafkaCache::~SrsKafkaCache() +{ + map::iterator it; + for (it = cache.begin(); it != cache.end(); ++it) { + SrsKafkaPartitionCache* pc = it->second; + + for (vector::iterator it2 = pc->begin(); it2 != pc->end(); ++it2) { + SrsJsonObject* obj = *it2; + srs_freep(obj); + } + pc->clear(); + + srs_freep(pc); + } + cache.clear(); +} + +void SrsKafkaCache::append(int key, SrsJsonObject* obj) +{ + count++; + + int partition = 0; + if (nb_partitions > 0) { + partition = key % nb_partitions; + } + + SrsKafkaPartitionCache* pc = NULL; + map::iterator it = cache.find(partition); + if (it == cache.end()) { + pc = new SrsKafkaPartitionCache(); + cache[partition] = pc; + } else { + pc = it->second; + } + + pc->push_back(obj); +} + +int SrsKafkaCache::size() +{ + return count; +} + +bool SrsKafkaCache::fetch(int* pkey, SrsKafkaPartitionCache** ppc) +{ + map::iterator it; + for (it = cache.begin(); it != cache.end(); ++it) { + int32_t key = it->first; + SrsKafkaPartitionCache* pc = it->second; + + if (!pc->empty()) { + *pkey = (int)key; + *ppc = pc; + return true; + } + } + + return false; +} + +int SrsKafkaCache::flush(SrsKafkaPartition* partition, int key, SrsKafkaPartitionCache* pc) +{ + int ret = ERROR_SUCCESS; + + // ensure the key exists. + srs_assert (cache.find(key) != cache.end()); + + // the cache is vector, which is continous store. + // we remember the messages we have written and clear it when completed. + int nb_msgs = (int)pc->size(); + if (pc->empty()) { + return ret; + } + + // connect transport. + if ((ret = partition->connect()) != ERROR_SUCCESS) { + srs_error("kafka connect to partition failed. ret=%d", ret); + return ret; + } + + // write the json objects. + if ((ret = partition->flush(pc)) != ERROR_SUCCESS) { + srs_error("kafka write messages failed. ret=%d", ret); + return ret; + } + + // free all wrote messages. + for (vector::iterator it = pc->begin(); it != pc->end(); ++it) { + SrsJsonObject* obj = *it; + srs_freep(obj); + } + + // remove the messages from cache. + if (pc->size() == nb_msgs) { + pc->clear(); + } else { + pc->erase(pc->begin(), pc->begin() + nb_msgs); + } + + return ret; +} + +ISrsKafkaCluster::ISrsKafkaCluster() +{ +} + +ISrsKafkaCluster::~ISrsKafkaCluster() +{ +} + SrsKafkaProducer::SrsKafkaProducer() { - lb = new SrsLbRoundRobin(); + metadata_ok = false; + metadata_expired = st_cond_new(); + + lock = st_mutex_new(); + pthread = new SrsReusableThread("kafka", this, SRS_KAKFA_CYCLE_INTERVAL_MS * 1000); worker = new SrsAsyncCallWorker(); + cache = new SrsKafkaCache(); + + lb = new SrsLbRoundRobin(); } SrsKafkaProducer::~SrsKafkaProducer() { + clear_metadata(); + srs_freep(lb); + srs_freep(worker); + srs_freep(pthread); + srs_freep(cache); + + st_mutex_destroy(lock); + st_cond_destroy(metadata_expired); } int SrsKafkaProducer::initialize() { int ret = ERROR_SUCCESS; - // when kafka enabled, request metadata when startup. - if ((ret = request_metadata()) != ERROR_SUCCESS) { - srs_error("request kafka metadata failed. ret=%d", ret); - return ret; - } - - srs_info("initialize kafka producer ok."); + enabled = _srs_config->get_kafka_enabled(); + srs_info("initialize kafka ok, enabled=%d.", enabled); return ret; } @@ -68,43 +363,257 @@ int SrsKafkaProducer::start() int ret = ERROR_SUCCESS; if ((ret = worker->start()) != ERROR_SUCCESS) { - srs_error("start kafka failed. ret=%d", ret); + srs_error("start kafka worker failed. ret=%d", ret); return ret; } - srs_info("kafka worker ok"); + if ((ret = pthread->start()) != ERROR_SUCCESS) { + srs_error("start kafka thread failed. ret=%d", ret); + } + + refresh_metadata(); return ret; } void SrsKafkaProducer::stop() { + pthread->stop(); worker->stop(); } +int SrsKafkaProducer::send(int key, SrsJsonObject* obj) +{ + int ret = ERROR_SUCCESS; + + // cache the json object. + cache->append(key, obj); + + // too few messages, ignore. + if (cache->size() < SRS_KAFKA_PRODUCER_AGGREGATE_SIZE) { + return ret; + } + + // too many messages, warn user. + if (cache->size() > SRS_KAFKA_PRODUCER_AGGREGATE_SIZE * 10) { + srs_warn("kafka cache too many messages: %d", cache->size()); + } + + // sync with backgound metadata worker. + st_mutex_lock(lock); + + // flush message when metadata is ok. + if (metadata_ok) { + ret = flush(); + } + + st_mutex_unlock(lock); + + return ret; +} + +int SrsKafkaProducer::on_client(int key, SrsListenerType type, string ip) +{ + int ret = ERROR_SUCCESS; + + if (!enabled) { + return ret; + } + + SrsJsonObject* obj = SrsJsonAny::object(); + + obj->set("msg", SrsJsonAny::str("accept")); + obj->set("type", SrsJsonAny::integer(type)); + obj->set("ip", SrsJsonAny::str(ip.c_str())); + + return worker->execute(new SrsKafkaMessage(this, key, obj)); +} + +int SrsKafkaProducer::on_close(int key) +{ + int ret = ERROR_SUCCESS; + + if (!enabled) { + return ret; + } + + SrsJsonObject* obj = SrsJsonAny::object(); + + obj->set("msg", SrsJsonAny::str("close")); + + return worker->execute(new SrsKafkaMessage(this, key, obj)); +} + +int SrsKafkaProducer::cycle() +{ + int ret = ERROR_SUCCESS; + + if ((ret = do_cycle()) != ERROR_SUCCESS) { + srs_warn("ignore kafka error. ret=%d", ret); + } + + return ret; +} + +int SrsKafkaProducer::on_before_cycle() +{ + // wait for the metadata expired. + // when metadata is ok, wait for it expired. + if (metadata_ok) { + st_cond_wait(metadata_expired); + } + + // request to lock to acquire the socket. + st_mutex_lock(lock); + + return ERROR_SUCCESS; +} + +int SrsKafkaProducer::on_end_cycle() +{ + st_mutex_unlock(lock); + + return ERROR_SUCCESS; +} + +void SrsKafkaProducer::clear_metadata() +{ + vector::iterator it; + + for (it = partitions.begin(); it != partitions.end(); ++it) { + SrsKafkaPartition* partition = *it; + srs_freep(partition); + } + + partitions.clear(); +} + +int SrsKafkaProducer::do_cycle() +{ + int ret = ERROR_SUCCESS; + + // ignore when disabled. + if (!enabled) { + return ret; + } + + // when kafka enabled, request metadata when startup. + if ((ret = request_metadata()) != ERROR_SUCCESS) { + srs_error("request kafka metadata failed. ret=%d", ret); + return ret; + } + + return ret; +} + int SrsKafkaProducer::request_metadata() { int ret = ERROR_SUCCESS; - bool enabled = _srs_config->get_kafka_enabled(); + // ignore when disabled. if (!enabled) { return ret; } + // select one broker to connect to. SrsConfDirective* brokers = _srs_config->get_kafka_brokers(); if (!brokers) { srs_warn("ignore for empty brokers."); return ret; } - srs_assert(!brokers->args.empty()); - std::string broker = lb->select(brokers->args); + SrsTcpClient* transport = new SrsTcpClient(); + SrsAutoFree(SrsTcpClient, transport); + SrsKafkaClient* kafka = new SrsKafkaClient(transport); + SrsAutoFree(SrsKafkaClient, kafka); + + std::string server; + int port = SRS_CONSTS_KAFKA_DEFAULT_PORT; + if (true) { + srs_assert(!brokers->args.empty()); + std::string broker = lb->select(brokers->args); + srs_parse_endpoint(broker, server, port); + } + + std::string topic = _srs_config->get_kafka_topic(); if (true) { std::string senabled = srs_bool2switch(enabled); std::string sbrokers = srs_join_vector_string(brokers->args, ","); - srs_trace("kafka ok, enabled:%s, brokers:%s, current:[%d]%s", - senabled.c_str(), sbrokers.c_str(), lb->current(), broker.c_str()); + srs_trace("kafka request enabled:%s, brokers:%s, current:[%d]%s:%d, topic:%s", + senabled.c_str(), sbrokers.c_str(), lb->current(), server.c_str(), port, topic.c_str()); + } + + // reconnect to kafka server. + if ((ret = transport->connect(server, port, SRS_CONSTS_KAFKA_TIMEOUT_US)) != ERROR_SUCCESS) { + srs_error("kafka connect %s:%d failed. ret=%d", server.c_str(), port, ret); + return ret; + } + + // do fetch medata from broker. + SrsKafkaTopicMetadataResponse* metadata = NULL; + if ((ret = kafka->fetch_metadata(topic, &metadata)) != ERROR_SUCCESS) { + srs_error("kafka fetch metadata failed. ret=%d", ret); + return ret; + } + SrsAutoFree(SrsKafkaTopicMetadataResponse, metadata); + + // we may need to request multiple times. + // for example, the first time to create a none-exists topic, then query metadata. + if (!metadata->metadatas.empty()) { + SrsKafkaTopicMetadata* topic = metadata->metadatas.at(0); + if (topic->metadatas.empty()) { + srs_warn("topic %s metadata empty, retry.", topic->name.to_str().c_str()); + return ret; + } + } + + // show kafka metadata. + string summary = srs_kafka_metadata_summary(metadata); + srs_trace("kafka metadata: %s", summary.c_str()); + + // generate the partition info. + srs_kafka_metadata2connector(topic, metadata, partitions); + srs_trace("kafka connector: %s", srs_kafka_summary_partitions(partitions).c_str()); + + // update the total partition for cache. + cache->nb_partitions = (int)partitions.size(); + + metadata_ok = true; + + return ret; +} + +void SrsKafkaProducer::refresh_metadata() +{ + clear_metadata(); + + metadata_ok = false; + st_cond_signal(metadata_expired); + srs_trace("kafka async refresh metadata in background"); +} + +int SrsKafkaProducer::flush() +{ + int ret = ERROR_SUCCESS; + + // flush all available partition caches. + while (true) { + int key = -1; + SrsKafkaPartitionCache* pc = NULL; + + // all flushed, or no kafka partition to write to. + if (!cache->fetch(&key, &pc) || partitions.empty()) { + break; + } + + // flush specified partition. + srs_assert(key >= 0 && pc); + SrsKafkaPartition* partition = partitions.at(key % partitions.size()); + if ((ret = cache->flush(partition, key, pc)) != ERROR_SUCCESS) { + srs_error("flush partition failed. ret=%d", ret); + return ret; + } } return ret; diff --git a/trunk/src/app/srs_app_kafka.hpp b/trunk/src/app/srs_app_kafka.hpp index 02dc39724..e7508b249 100644 --- a/trunk/src/app/srs_app_kafka.hpp +++ b/trunk/src/app/srs_app_kafka.hpp @@ -29,16 +29,142 @@ */ #include +#include +#include + class SrsLbRoundRobin; class SrsAsyncCallWorker; +class SrsTcpClient; +class SrsKafkaClient; +class SrsJsonObject; +class SrsKafkaProducer; + +#include +#include +#include #ifdef SRS_AUTO_KAFKA +/** + * the partition messages cache. + */ +typedef std::vector SrsKafkaPartitionCache; + +/** + * the kafka partition info. + */ +struct SrsKafkaPartition +{ +private: + std::string ep; + SrsTcpClient* transport; + SrsKafkaClient* kafka; +public: + int id; + std::string topic; + // leader. + int broker; + std::string host; + int port; +public: + SrsKafkaPartition(); + virtual ~SrsKafkaPartition(); +public: + virtual std::string hostport(); + virtual int connect(); + virtual int flush(SrsKafkaPartitionCache* pc); +}; + +/** + * the following is all types of kafka messages. + */ +class SrsKafkaMessage : public ISrsAsyncCallTask +{ +private: + SrsKafkaProducer* producer; + int key; + SrsJsonObject* obj; +public: + SrsKafkaMessage(SrsKafkaProducer* p, int k, SrsJsonObject* j); + virtual ~SrsKafkaMessage(); +// interface ISrsAsyncCallTask +public: + virtual int call(); + virtual std::string to_string(); +}; + +/** + * a message cache for kafka. + */ +class SrsKafkaCache +{ +public: + // the total partitions, + // for the key to map to the parition by key%nb_partitions. + int nb_partitions; +private: + // total messages for all partitions. + int count; + // key is the partition id, value is the message set to write to this partition. + // @remark, when refresh metadata, the partition will increase, + // so maybe some message will dispatch to new partition. + std::map< int32_t, SrsKafkaPartitionCache*> cache; +public: + SrsKafkaCache(); + virtual ~SrsKafkaCache(); +public: + virtual void append(int key, SrsJsonObject* obj); + virtual int size(); + /** + * fetch out a available partition cache. + * @return true when got a key and pc; otherwise, false. + */ + virtual bool fetch(int* pkey, SrsKafkaPartitionCache** ppc); + /** + * flush the specified partition cache. + */ + virtual int flush(SrsKafkaPartition* partition, int key, SrsKafkaPartitionCache* pc); +}; + +/** + * the kafka cluster interface. + */ +class ISrsKafkaCluster +{ +public: + ISrsKafkaCluster(); + virtual ~ISrsKafkaCluster(); +public: + /** + * when got any client connect to SRS, notify kafka. + * @param key the partition map key, the client id or hash(ip). + * @param type the type of client. + * @param ip the peer ip of client. + */ + virtual int on_client(int key, SrsListenerType type, std::string ip) = 0; + /** + * when client close or disconnect for error. + * @param key the partition map key, the client id or hash(ip). + */ + virtual int on_close(int key) = 0; +}; + /** * the kafka producer used to save log to kafka cluster. */ -class SrsKafkaProducer +class SrsKafkaProducer : virtual public ISrsReusableThreadHandler, virtual public ISrsKafkaCluster { +private: + // TODO: FIXME: support reload. + bool enabled; + st_mutex_t lock; + SrsReusableThread* pthread; +private: + bool metadata_ok; + st_cond_t metadata_expired; +public: + std::vector partitions; + SrsKafkaCache* cache; private: SrsLbRoundRobin* lb; SrsAsyncCallWorker* worker; @@ -49,8 +175,31 @@ public: virtual int initialize(); virtual int start(); virtual void stop(); +// internal: for worker to call task to send object. +public: + /** + * send json object to kafka cluster. + * the producer will aggregate message and send in kafka message set. + * @param key the key to map to the partition, user can use cid or hash. + * @param obj the json object; user must never free it again. + */ + virtual int send(int key, SrsJsonObject* obj); +// interface ISrsKafkaCluster +public: + virtual int on_client(int key, SrsListenerType type, std::string ip); + virtual int on_close(int key); +// interface ISrsReusableThreadHandler +public: + virtual int cycle(); + virtual int on_before_cycle(); + virtual int on_end_cycle(); private: + virtual void clear_metadata(); + virtual int do_cycle(); virtual int request_metadata(); + // set the metadata to invalid and refresh it. + virtual void refresh_metadata(); + virtual int flush(); }; #endif diff --git a/trunk/src/app/srs_app_mpegts_udp.cpp b/trunk/src/app/srs_app_mpegts_udp.cpp index 313279b11..0d437166c 100644 --- a/trunk/src/app/srs_app_mpegts_udp.cpp +++ b/trunk/src/app/srs_app_mpegts_udp.cpp @@ -49,6 +49,7 @@ using namespace std; #include #include #include +#include SrsMpegtsQueue::SrsMpegtsQueue() { @@ -132,10 +133,7 @@ SrsMpegtsOverUdp::SrsMpegtsOverUdp(SrsConfDirective* c) output = _srs_config->get_stream_caster_output(c); req = NULL; - io = NULL; - client = NULL; - stfd = NULL; - stream_id = 0; + sdk = new SrsSimpleRtmpClient(); avc = new SrsRawH264Stream(); aac = new SrsRawAacStream(); @@ -150,6 +148,7 @@ SrsMpegtsOverUdp::~SrsMpegtsOverUdp() { close(); + srs_freep(sdk); srs_freep(buffer); srs_freep(stream); srs_freep(context); @@ -345,8 +344,8 @@ int SrsMpegtsOverUdp::on_ts_video(SrsTsMessage* msg, SrsBuffer* avs) } // ts tbn to flv tbn. - u_int32_t dts = msg->dts / 90; - u_int32_t pts = msg->dts / 90; + u_int32_t dts = (u_int32_t)(msg->dts / 90); + u_int32_t pts = (u_int32_t)(msg->dts / 90); // send each frame. while (!avs->empty()) { @@ -504,7 +503,7 @@ int SrsMpegtsOverUdp::on_ts_audio(SrsTsMessage* msg, SrsBuffer* avs) } // ts tbn to flv tbn. - u_int32_t dts = msg->dts / 90; + u_int32_t dts = (u_int32_t)(msg->dts / 90); // send each frame. while (!avs->empty()) { @@ -565,8 +564,8 @@ int SrsMpegtsOverUdp::rtmp_write_packet(char type, u_int32_t timestamp, char* da int ret = ERROR_SUCCESS; SrsSharedPtrMessage* msg = NULL; - - if ((ret = srs_rtmp_create_msg(type, timestamp, data, size, stream_id, &msg)) != ERROR_SUCCESS) { + + if ((ret = sdk->rtmp_create_msg(type, timestamp, data, size, &msg)) != ERROR_SUCCESS) { srs_error("mpegts: create shared ptr msg failed. ret=%d", ret); return ret; } @@ -590,7 +589,7 @@ int SrsMpegtsOverUdp::rtmp_write_packet(char type, u_int32_t timestamp, char* da } // send out encoded msg. - if ((ret = client->send_and_free_message(msg, stream_id)) != ERROR_SUCCESS) { + if ((ret = sdk->send_and_free_message(msg)) != ERROR_SUCCESS) { return ret; } } @@ -604,120 +603,29 @@ int SrsMpegtsOverUdp::connect() // when ok, ignore. // TODO: FIXME: should reconnect when disconnected. - if (io || client) { + if (sdk->connected()) { return ret; } - // parse uri - if (!req) { - req = new SrsRequest(); - - size_t pos = string::npos; - string uri = req->tcUrl = output; - - // tcUrl, stream - if ((pos = uri.rfind("/")) != string::npos) { - req->stream = uri.substr(pos + 1); - req->tcUrl = uri = uri.substr(0, pos); - } - - srs_discovery_tc_url(req->tcUrl, - req->schema, req->host, req->vhost, req->app, req->port, - req->param); - } - - // connect host. - if ((ret = srs_socket_connect(req->host, ::atoi(req->port.c_str()), ST_UTIME_NO_TIMEOUT, &stfd)) != ERROR_SUCCESS) { - srs_error("mpegts: connect server %s:%s failed. ret=%d", req->host.c_str(), req->port.c_str(), ret); - return ret; - } - io = new SrsStSocket(stfd); - client = new SrsRtmpClient(io); - - client->set_recv_timeout(SRS_CONSTS_RTMP_RECV_TIMEOUT_US); - client->set_send_timeout(SRS_CONSTS_RTMP_SEND_TIMEOUT_US); - - // connect to vhost/app - if ((ret = client->handshake()) != ERROR_SUCCESS) { - srs_error("mpegts: handshake with server failed. ret=%d", ret); - return ret; - } - if ((ret = connect_app(req->host, req->port)) != ERROR_SUCCESS) { - srs_error("mpegts: connect with server failed. ret=%d", ret); - return ret; - } - if ((ret = client->create_stream(stream_id)) != ERROR_SUCCESS) { - srs_error("mpegts: connect with server failed, stream_id=%d. ret=%d", stream_id, ret); + int64_t cto = SRS_CONSTS_RTMP_TIMEOUT_US; + int64_t sto = SRS_CONSTS_RTMP_PULSE_TIMEOUT_US; + if ((ret = sdk->connect(output, cto, sto)) != ERROR_SUCCESS) { + srs_error("mpegts: connect %s failed, cto=%"PRId64", sto=%"PRId64". ret=%d", output.c_str(), cto, sto, ret); return ret; } - // publish. - if ((ret = client->publish(req->stream, stream_id)) != ERROR_SUCCESS) { - srs_error("mpegts: publish failed, stream=%s, stream_id=%d. ret=%d", - req->stream.c_str(), stream_id, ret); + if ((ret = sdk->publish()) != ERROR_SUCCESS) { + srs_error("mpegts: publish failed. ret=%d", ret); return ret; } return ret; } -// TODO: FIXME: refine the connect_app. -int SrsMpegtsOverUdp::connect_app(string ep_server, string ep_port) -{ - int ret = ERROR_SUCCESS; - - // args of request takes the srs info. - if (req->args == NULL) { - req->args = SrsAmf0Any::object(); - } - - // notify server the edge identity, - // @see https://github.com/simple-rtmp-server/srs/issues/147 - SrsAmf0Object* data = req->args; - data->set("srs_sig", SrsAmf0Any::str(RTMP_SIG_SRS_KEY)); - data->set("srs_server", SrsAmf0Any::str(RTMP_SIG_SRS_KEY" "RTMP_SIG_SRS_VERSION" ("RTMP_SIG_SRS_URL_SHORT")")); - data->set("srs_license", SrsAmf0Any::str(RTMP_SIG_SRS_LICENSE)); - data->set("srs_role", SrsAmf0Any::str(RTMP_SIG_SRS_ROLE)); - data->set("srs_url", SrsAmf0Any::str(RTMP_SIG_SRS_URL)); - data->set("srs_version", SrsAmf0Any::str(RTMP_SIG_SRS_VERSION)); - data->set("srs_site", SrsAmf0Any::str(RTMP_SIG_SRS_WEB)); - data->set("srs_email", SrsAmf0Any::str(RTMP_SIG_SRS_EMAIL)); - data->set("srs_copyright", SrsAmf0Any::str(RTMP_SIG_SRS_COPYRIGHT)); - data->set("srs_primary", SrsAmf0Any::str(RTMP_SIG_SRS_PRIMARY)); - data->set("srs_authors", SrsAmf0Any::str(RTMP_SIG_SRS_AUTHROS)); - // for edge to directly get the id of client. - data->set("srs_pid", SrsAmf0Any::number(getpid())); - data->set("srs_id", SrsAmf0Any::number(_srs_context->get_id())); - - // local ip of edge - std::vector ips = srs_get_local_ipv4_ips(); - assert(_srs_config->get_stats_network() < (int)ips.size()); - std::string local_ip = ips[_srs_config->get_stats_network()]; - data->set("srs_server_ip", SrsAmf0Any::str(local_ip.c_str())); - - // generate the tcUrl - std::string param = ""; - std::string tc_url = srs_generate_tc_url(ep_server, req->vhost, req->app, ep_port, param); - - // upnode server identity will show in the connect_app of client. - // @see https://github.com/simple-rtmp-server/srs/issues/160 - // the debug_srs_upnode is config in vhost and default to true. - bool debug_srs_upnode = _srs_config->get_debug_srs_upnode(req->vhost); - if ((ret = client->connect_app(req->app, tc_url, req, debug_srs_upnode)) != ERROR_SUCCESS) { - srs_error("mpegts: connect with server failed, tcUrl=%s, dsu=%d. ret=%d", - tc_url.c_str(), debug_srs_upnode, ret); - return ret; - } - - return ret; -} - void SrsMpegtsOverUdp::close() { - srs_freep(client); - srs_freep(io); srs_freep(req); - srs_close_stfd(stfd); + sdk->close(); } #endif diff --git a/trunk/src/app/srs_app_mpegts_udp.hpp b/trunk/src/app/srs_app_mpegts_udp.hpp index 400e8627e..e9de14c00 100644 --- a/trunk/src/app/srs_app_mpegts_udp.hpp +++ b/trunk/src/app/srs_app_mpegts_udp.hpp @@ -48,6 +48,7 @@ class SrsSharedPtrMessage; class SrsRawAacStream; struct SrsRawAacStreamCodec; class SrsPithyPrint; +class SrsSimpleRtmpClient; #include #include @@ -86,10 +87,7 @@ private: std::string output; private: SrsRequest* req; - st_netfd_t stfd; - SrsStSocket* io; - SrsRtmpClient* client; - int stream_id; + SrsSimpleRtmpClient* sdk; private: SrsRawH264Stream* avc; std::string h264_sps; @@ -126,7 +124,6 @@ private: // connect to rtmp output url. // @remark ignore when not connected, reconnect when disconnected. virtual int connect(); - virtual int connect_app(std::string ep_server, std::string ep_port); // close the connected io and rtmp to ready to be re-connect. virtual void close(); }; diff --git a/trunk/src/app/srs_app_rtmp_conn.cpp b/trunk/src/app/srs_app_rtmp_conn.cpp index 5bc2d0dd2..777f50c61 100644 --- a/trunk/src/app/srs_app_rtmp_conn.cpp +++ b/trunk/src/app/srs_app_rtmp_conn.cpp @@ -55,6 +55,7 @@ using namespace std; #include #include #include +#include // when stream is busy, for example, streaming is already // publishing, when a new client to request to publish, @@ -76,10 +77,252 @@ using namespace std; // when edge timeout, retry next. #define SRS_EDGE_TOKEN_TRAVERSE_TIMEOUT_US (int64_t)(3*1000*1000LL) +SrsSimpleRtmpClient::SrsSimpleRtmpClient() +{ + req = NULL; + client = NULL; + kbps = new SrsKbps(); + + transport = new SrsTcpClient(); + stream_id = 0; +} + +SrsSimpleRtmpClient::~SrsSimpleRtmpClient() +{ + close(); + + srs_freep(kbps); + srs_freep(transport); + + srs_freep(client); + kbps->set_io(NULL, NULL); +} + +int SrsSimpleRtmpClient::connect(string url, int64_t connect_timeout, int64_t stream_timeout) +{ + int ret = ERROR_SUCCESS; + + // when ok, ignore. + // TODO: FIXME: should reconnect when disconnected. + if (transport->connected()) { + return ret; + } + + // parse uri + srs_freep(req); + req = new SrsRequest(); + srs_parse_rtmp_url(url, req->tcUrl, req->stream); + srs_discovery_tc_url(req->tcUrl, req->schema, req->host, req->vhost, req->app, req->port, req->param); + + // connect host. + if ((ret = transport->connect(req->host, req->port, connect_timeout)) != ERROR_SUCCESS) { + return ret; + } + + srs_freep(client); + client = new SrsRtmpClient(transport); + + kbps->set_io(transport, transport); + + client->set_recv_timeout(stream_timeout); + client->set_send_timeout(stream_timeout); + + // connect to vhost/app + if ((ret = client->handshake()) != ERROR_SUCCESS) { + srs_error("sdk: handshake with server failed. ret=%d", ret); + return ret; + } + if ((ret = connect_app()) != ERROR_SUCCESS) { + srs_error("sdk: connect with server failed. ret=%d", ret); + return ret; + } + if ((ret = client->create_stream(stream_id)) != ERROR_SUCCESS) { + srs_error("sdk: connect with server failed, stream_id=%d. ret=%d", stream_id, ret); + return ret; + } + + return ret; +} + +int SrsSimpleRtmpClient::connect_app() +{ + int ret = ERROR_SUCCESS; + + // args of request takes the srs info. + if (req->args == NULL) { + req->args = SrsAmf0Any::object(); + } + + // notify server the edge identity, + // @see https://github.com/simple-rtmp-server/srs/issues/147 + SrsAmf0Object* data = req->args; + data->set("srs_sig", SrsAmf0Any::str(RTMP_SIG_SRS_KEY)); + data->set("srs_server", SrsAmf0Any::str(RTMP_SIG_SRS_SERVER)); + data->set("srs_license", SrsAmf0Any::str(RTMP_SIG_SRS_LICENSE)); + data->set("srs_role", SrsAmf0Any::str(RTMP_SIG_SRS_ROLE)); + data->set("srs_url", SrsAmf0Any::str(RTMP_SIG_SRS_URL)); + data->set("srs_version", SrsAmf0Any::str(RTMP_SIG_SRS_VERSION)); + data->set("srs_site", SrsAmf0Any::str(RTMP_SIG_SRS_WEB)); + data->set("srs_email", SrsAmf0Any::str(RTMP_SIG_SRS_EMAIL)); + data->set("srs_copyright", SrsAmf0Any::str(RTMP_SIG_SRS_COPYRIGHT)); + data->set("srs_primary", SrsAmf0Any::str(RTMP_SIG_SRS_PRIMARY)); + data->set("srs_authors", SrsAmf0Any::str(RTMP_SIG_SRS_AUTHROS)); + // for edge to directly get the id of client. + data->set("srs_pid", SrsAmf0Any::number(getpid())); + data->set("srs_id", SrsAmf0Any::number(_srs_context->get_id())); + + // local ip of edge + std::vector ips = srs_get_local_ipv4_ips(); + assert(_srs_config->get_stats_network() < (int)ips.size()); + std::string local_ip = ips[_srs_config->get_stats_network()]; + data->set("srs_server_ip", SrsAmf0Any::str(local_ip.c_str())); + + // generate the tcUrl + std::string param = ""; + std::string target_vhost = req->vhost; + std::string tc_url = srs_generate_tc_url(req->host, req->vhost, req->app, req->port, param); + + // replace the tcUrl in request, + // which will replace the tc_url in client.connect_app(). + req->tcUrl = tc_url; + + // upnode server identity will show in the connect_app of client. + // @see https://github.com/simple-rtmp-server/srs/issues/160 + // the debug_srs_upnode is config in vhost and default to true. + bool debug_srs_upnode = _srs_config->get_debug_srs_upnode(req->vhost); + if ((ret = client->connect_app(req->app, tc_url, req, debug_srs_upnode)) != ERROR_SUCCESS) { + srs_error("sdk: connect with server failed, tcUrl=%s, dsu=%d. ret=%d", + tc_url.c_str(), debug_srs_upnode, ret); + return ret; + } + + return ret; +} + +bool SrsSimpleRtmpClient::connected() +{ + return transport->connected(); +} + +void SrsSimpleRtmpClient::close() +{ + transport->close(); + + srs_freep(client); + srs_freep(req); +} + +int SrsSimpleRtmpClient::publish() +{ + int ret = ERROR_SUCCESS; + + // publish. + if ((ret = client->publish(req->stream, stream_id)) != ERROR_SUCCESS) { + srs_error("sdk: publish failed, stream=%s, stream_id=%d. ret=%d", + req->stream.c_str(), stream_id, ret); + return ret; + } + + return ret; +} + +int SrsSimpleRtmpClient::play() +{ + int ret = ERROR_SUCCESS; + + if ((ret = client->play(req->stream, stream_id)) != ERROR_SUCCESS) { + srs_error("connect with server failed, stream=%s, stream_id=%d. ret=%d", + req->stream.c_str(), stream_id, ret); + return ret; + } + + return ret; +} + +void SrsSimpleRtmpClient::kbps_sample(const char* label, int64_t age) +{ + kbps->sample(); + + int sr = kbps->get_send_kbps(); + int sr30s = kbps->get_send_kbps_30s(); + int sr5m = kbps->get_send_kbps_5m(); + int rr = kbps->get_recv_kbps(); + int rr30s = kbps->get_recv_kbps_30s(); + int rr5m = kbps->get_recv_kbps_5m(); + + srs_trace("<- %s time=%"PRId64", okbps=%d,%d,%d, ikbps=%d,%d,%d", age, sr, sr30s, sr5m, rr, rr30s, rr5m); +} + +void SrsSimpleRtmpClient::kbps_sample(const char* label, int64_t age, int msgs) +{ + kbps->sample(); + + int sr = kbps->get_send_kbps(); + int sr30s = kbps->get_send_kbps_30s(); + int sr5m = kbps->get_send_kbps_5m(); + int rr = kbps->get_recv_kbps(); + int rr30s = kbps->get_recv_kbps_30s(); + int rr5m = kbps->get_recv_kbps_5m(); + + srs_trace("<- %s time=%"PRId64", msgs=%d, okbps=%d,%d,%d, ikbps=%d,%d,%d", age, msgs, sr, sr30s, sr5m, rr, rr30s, rr5m); +} + +int SrsSimpleRtmpClient::sid() +{ + return stream_id; +} + +int SrsSimpleRtmpClient::rtmp_create_msg(char type, u_int32_t timestamp, char* data, int size, SrsSharedPtrMessage** pmsg) +{ + *pmsg = NULL; + + int ret = ERROR_SUCCESS; + + if ((ret = srs_rtmp_create_msg(type, timestamp, data, size, stream_id, pmsg)) != ERROR_SUCCESS) { + srs_error("sdk: create shared ptr msg failed. ret=%d", ret); + return ret; + } + + return ret; +} + +int SrsSimpleRtmpClient::recv_message(SrsCommonMessage** pmsg) +{ + return client->recv_message(pmsg); +} + +int SrsSimpleRtmpClient::decode_message(SrsCommonMessage* msg, SrsPacket** ppacket) +{ + return client->decode_message(msg, ppacket); +} + +int SrsSimpleRtmpClient::send_and_free_messages(SrsSharedPtrMessage** msgs, int nb_msgs) +{ + return client->send_and_free_messages(msgs, nb_msgs, stream_id); +} + +int SrsSimpleRtmpClient::send_and_free_message(SrsSharedPtrMessage* msg) +{ + return client->send_and_free_message(msg, stream_id); +} + +void SrsSimpleRtmpClient::set_recv_timeout(int64_t timeout) +{ + transport->set_recv_timeout(timeout); +} + +#ifdef SRS_AUTO_KAFKA +SrsRtmpConn::SrsRtmpConn(SrsServer* svr, ISrsKafkaCluster* k, st_netfd_t c) +#else SrsRtmpConn::SrsRtmpConn(SrsServer* svr, st_netfd_t c) +#endif : SrsConnection(svr, c) { server = svr; +#ifdef SRS_AUTO_KAFKA + kafka = k; +#endif + req = new SrsRequest(); res = new SrsResponse(); skt = new SrsStSocket(c); @@ -131,9 +374,17 @@ int SrsRtmpConn::do_cycle() int ret = ERROR_SUCCESS; srs_trace("RTMP client ip=%s", ip.c_str()); + + // notify kafka cluster. +#ifdef SRS_AUTO_KAFKA + if ((ret = kafka->on_client(srs_id(), SrsListenerRtmpStream, ip)) != ERROR_SUCCESS) { + srs_error("kafka handler on_client failed. ret=%d", ret); + return ret; + } +#endif - rtmp->set_recv_timeout(SRS_CONSTS_RTMP_RECV_TIMEOUT_US); - rtmp->set_send_timeout(SRS_CONSTS_RTMP_SEND_TIMEOUT_US); + rtmp->set_recv_timeout(SRS_CONSTS_RTMP_TIMEOUT_US); + rtmp->set_send_timeout(SRS_CONSTS_RTMP_TIMEOUT_US); if ((ret = rtmp->handshake()) != ERROR_SUCCESS) { srs_error("rtmp handshake failed. ret=%d", ret); @@ -460,8 +711,8 @@ int SrsRtmpConn::stream_service_cycle() srs_info("security check ok"); // client is identified, set the timeout to service timeout. - rtmp->set_recv_timeout(SRS_CONSTS_RTMP_RECV_TIMEOUT_US); - rtmp->set_send_timeout(SRS_CONSTS_RTMP_SEND_TIMEOUT_US); + rtmp->set_recv_timeout(SRS_CONSTS_RTMP_TIMEOUT_US); + rtmp->set_send_timeout(SRS_CONSTS_RTMP_TIMEOUT_US); // find a source to serve. SrsSource* source = SrsSource::fetch(req); @@ -1229,11 +1480,13 @@ int SrsRtmpConn::check_edge_token_traverse_auth() srs_assert(req); - st_netfd_t stsock = NULL; + SrsTcpClient* transport = new SrsTcpClient(); + SrsAutoFree(SrsTcpClient, transport); + vector args = _srs_config->get_vhost_edge_origin(req->vhost)->args; for (int i = 0; i < (int)args.size(); i++) { string hostport = args.at(i); - if ((ret = connect_server(hostport, &stsock)) == ERROR_SUCCESS) { + if ((ret = connect_server(hostport, transport)) == ERROR_SUCCESS) { break; } } @@ -1242,20 +1495,13 @@ int SrsRtmpConn::check_edge_token_traverse_auth() return ret; } - srs_assert(stsock); - SrsStSocket* io = new SrsStSocket(stsock); - SrsRtmpClient* client = new SrsRtmpClient(io); + SrsRtmpClient* client = new SrsRtmpClient(transport); + SrsAutoFree(SrsRtmpClient, client); - ret = do_token_traverse_auth(client); - - srs_freep(client); - srs_freep(io); - srs_close_stfd(stsock); - - return ret; + return do_token_traverse_auth(client); } -int SrsRtmpConn::connect_server(string hostport, st_netfd_t* pstsock) +int SrsRtmpConn::connect_server(string hostport, SrsTcpClient* transport) { int ret = ERROR_SUCCESS; @@ -1268,16 +1514,14 @@ int SrsRtmpConn::connect_server(string hostport, st_netfd_t* pstsock) srs_parse_hostport(hostport, server, port); // open socket. - st_netfd_t stsock = NULL; int64_t timeout = SRS_EDGE_TOKEN_TRAVERSE_TIMEOUT_US; - if ((ret = srs_socket_connect(server, port, timeout, &stsock)) != ERROR_SUCCESS) { + if ((ret = transport->connect(server, port, timeout)) != ERROR_SUCCESS) { srs_warn("edge token traverse failed, tcUrl=%s to server=%s, port=%d, timeout=%"PRId64", ret=%d", req->tcUrl.c_str(), server.c_str(), port, timeout, ret); return ret; } srs_info("edge token auth connected, url=%s/%s, server=%s:%d", req->tcUrl.c_str(), req->stream.c_str(), server.c_str(), port); - *pstsock = stsock; return ret; } @@ -1287,8 +1531,8 @@ int SrsRtmpConn::do_token_traverse_auth(SrsRtmpClient* client) srs_assert(client); - client->set_recv_timeout(SRS_CONSTS_RTMP_RECV_TIMEOUT_US); - client->set_send_timeout(SRS_CONSTS_RTMP_SEND_TIMEOUT_US); + client->set_recv_timeout(SRS_CONSTS_RTMP_TIMEOUT_US); + client->set_send_timeout(SRS_CONSTS_RTMP_TIMEOUT_US); if ((ret = client->handshake()) != ERROR_SUCCESS) { srs_error("handshake with server failed. ret=%d", ret); @@ -1311,6 +1555,13 @@ int SrsRtmpConn::on_disconnect() int ret = ERROR_SUCCESS; http_hooks_on_close(); + +#ifdef SRS_AUTO_KAFKA + if ((ret = kafka->on_close(srs_id())) != ERROR_SUCCESS) { + srs_error("notify kafka failed. ret=%d", ret); + return ret; + } +#endif // TODO: implements it. diff --git a/trunk/src/app/srs_app_rtmp_conn.hpp b/trunk/src/app/srs_app_rtmp_conn.hpp index 005315aa4..29fea6ee4 100644 --- a/trunk/src/app/srs_app_rtmp_conn.hpp +++ b/trunk/src/app/srs_app_rtmp_conn.hpp @@ -56,10 +56,53 @@ class SrsQueueRecvThread; class SrsPublishRecvThread; class SrsSecurity; class ISrsWakable; +class SrsCommonMessage; +class SrsPacket; +#ifdef SRS_AUTO_KAFKA +class ISrsKafkaCluster; +#endif /** -* the client provides the main logic control for RTMP clients. -*/ + * the simple rtmp client stub, use SrsRtmpClient and provides high level APIs. + */ +class SrsSimpleRtmpClient +{ +private: + SrsRequest* req; + SrsTcpClient* transport; + SrsRtmpClient* client; + SrsKbps* kbps; + int stream_id; +public: + SrsSimpleRtmpClient(); + virtual ~SrsSimpleRtmpClient(); +public: + virtual int connect(std::string url, int64_t connect_timeout, int64_t stream_timeout); +private: + virtual int connect_app(); +public: + virtual bool connected(); + virtual void close(); +public: + virtual int publish(); + virtual int play(); + virtual void kbps_sample(const char* label, int64_t age); + virtual void kbps_sample(const char* label, int64_t age, int msgs); + virtual int sid(); +public: + virtual int rtmp_create_msg(char type, u_int32_t timestamp, char* data, int size, SrsSharedPtrMessage** pmsg); +public: + virtual int recv_message(SrsCommonMessage** pmsg); + virtual int decode_message(SrsCommonMessage* msg, SrsPacket** ppacket); + virtual int send_and_free_messages(SrsSharedPtrMessage** msgs, int nb_msgs); + virtual int send_and_free_message(SrsSharedPtrMessage* msg); +public: + virtual void set_recv_timeout(int64_t timeout); +}; + +/** + * the client provides the main logic control for RTMP clients. + */ class SrsRtmpConn : public virtual SrsConnection, public virtual ISrsReloadHandler { // for the thread to directly access any field of connection. @@ -95,8 +138,16 @@ private: int publish_normal_timeout; // whether enable the tcp_nodelay. bool tcp_nodelay; + // the kafka cluster +#ifdef SRS_AUTO_KAFKA + ISrsKafkaCluster* kafka; +#endif public: +#ifdef SRS_AUTO_KAFKA + SrsRtmpConn(SrsServer* svr, ISrsKafkaCluster* k, st_netfd_t c); +#else SrsRtmpConn(SrsServer* svr, st_netfd_t c); +#endif virtual ~SrsRtmpConn(); public: virtual void dispose(); @@ -134,7 +185,7 @@ private: virtual void set_sock_options(); private: virtual int check_edge_token_traverse_auth(); - virtual int connect_server(std::string hostport, st_netfd_t* pstsock); + virtual int connect_server(std::string hostport, SrsTcpClient* transport); virtual int do_token_traverse_auth(SrsRtmpClient* client); /** * when the connection disconnect, call this method. diff --git a/trunk/src/app/srs_app_rtsp.cpp b/trunk/src/app/srs_app_rtsp.cpp index c0c6b38e1..e40539b8c 100644 --- a/trunk/src/app/srs_app_rtsp.cpp +++ b/trunk/src/app/srs_app_rtsp.cpp @@ -42,6 +42,7 @@ using namespace std; #include #include #include +#include #ifdef SRS_AUTO_STREAM_CASTER @@ -167,7 +168,7 @@ int SrsRtspJitter::correct(int64_t& ts) previous_timestamp = ts; } - delta = srs_max(0, ts - previous_timestamp); + delta = srs_max(0, (int)(ts - previous_timestamp)); if (delta > 90000) { delta = 0; } @@ -195,9 +196,7 @@ SrsRtspConn::SrsRtspConn(SrsRtspCaster* c, st_netfd_t fd, std::string o) trd = new SrsOneCycleThread("rtsp", this); req = NULL; - io = NULL; - client = NULL; - stream_id = 0; + sdk = new SrsSimpleRtmpClient(); vjitter = new SrsRtspJitter(); ajitter = new SrsRtspJitter(); @@ -218,8 +217,7 @@ SrsRtspConn::~SrsRtspConn() srs_freep(skt); srs_freep(rtsp); - srs_freep(client); - srs_freep(io); + srs_freep(sdk); srs_freep(req); srs_freep(vjitter); @@ -254,7 +252,7 @@ int SrsRtspConn::do_cycle() srs_info("rtsp: got rtsp request"); if (req->is_options()) { - SrsRtspOptionsResponse* res = new SrsRtspOptionsResponse(req->seq); + SrsRtspOptionsResponse* res = new SrsRtspOptionsResponse((int)req->seq); res->session = session; if ((ret = rtsp->send_message(res)) != ERROR_SUCCESS) { if (!srs_is_client_gracefully_close(ret)) { @@ -270,10 +268,7 @@ int SrsRtspConn::do_cycle() if ((pos = rtsp_tcUrl.rfind(".sdp")) != string::npos) { rtsp_tcUrl = rtsp_tcUrl.substr(0, pos); } - if ((pos = rtsp_tcUrl.rfind("/")) != string::npos) { - rtsp_stream = rtsp_tcUrl.substr(pos + 1); - rtsp_tcUrl = rtsp_tcUrl.substr(0, pos); - } + srs_parse_rtmp_url(rtsp_tcUrl, rtsp_tcUrl, rtsp_stream); srs_assert(req->sdp); video_id = ::atoi(req->sdp->video_stream_id.c_str()); @@ -291,7 +286,7 @@ int SrsRtspConn::do_cycle() audio_sample_rate, audio_channel, rtsp_tcUrl.c_str(), rtsp_stream.c_str() ); - SrsRtspResponse* res = new SrsRtspResponse(req->seq); + SrsRtspResponse* res = new SrsRtspResponse((int)req->seq); res->session = session; if ((ret = rtsp->send_message(res)) != ERROR_SUCCESS) { if (!srs_is_client_gracefully_close(ret)) { @@ -331,7 +326,7 @@ int SrsRtspConn::do_cycle() session = "O9EaZ4bf"; // TODO: FIXME: generate session id. } - SrsRtspSetupResponse* res = new SrsRtspSetupResponse(req->seq); + SrsRtspSetupResponse* res = new SrsRtspSetupResponse((int)req->seq); res->client_port_min = req->transport->client_port_min; res->client_port_max = req->transport->client_port_max; res->local_port_min = lpm; @@ -344,7 +339,7 @@ int SrsRtspConn::do_cycle() return ret; } } else if (req->is_record()) { - SrsRtspResponse* res = new SrsRtspResponse(req->seq); + SrsRtspResponse* res = new SrsRtspResponse((int)req->seq); res->session = session; if ((ret = rtsp->send_message(res)) != ERROR_SUCCESS) { if (!srs_is_client_gracefully_close(ret)) { @@ -437,7 +432,11 @@ int SrsRtspConn::on_rtp_video(SrsRtpPacket* pkt, int64_t dts, int64_t pts) return ret; } - if ((ret = write_h264_ipb_frame(pkt->payload->bytes(), pkt->payload->length(), dts / 90, pts / 90)) != ERROR_SUCCESS) { + char* bytes = pkt->payload->bytes(); + int length = pkt->payload->length(); + u_int32_t fdts = (u_int32_t)(dts / 90); + u_int32_t fpts = (u_int32_t)(pts / 90); + if ((ret = write_h264_ipb_frame(bytes, length, fdts, fpts)) != ERROR_SUCCESS) { return ret; } @@ -479,7 +478,7 @@ int SrsRtspConn::kickoff_audio_cache(SrsRtpPacket* pkt, int64_t dts) int nb_frame = acache->audio_samples->sample_units[i].size; int64_t timestamp = (acache->dts + delta * i) / 90; acodec->aac_packet_type = 1; - if ((ret = write_audio_raw_frame(frame, nb_frame, acodec, timestamp)) != ERROR_SUCCESS) { + if ((ret = write_audio_raw_frame(frame, nb_frame, acodec, (u_int32_t)timestamp)) != ERROR_SUCCESS) { return ret; } } @@ -500,7 +499,7 @@ int SrsRtspConn::write_sequence_header() int64_t dts = vjitter->timestamp() / 90; // send video sps/pps - if ((ret = write_h264_sps_pps(dts, dts)) != ERROR_SUCCESS) { + if ((ret = write_h264_sps_pps((u_int32_t)dts, (u_int32_t)dts)) != ERROR_SUCCESS) { return ret; } @@ -538,7 +537,7 @@ int SrsRtspConn::write_sequence_header() break; }; - if ((ret = write_audio_raw_frame((char*)sh.data(), (int)sh.length(), acodec, dts)) != ERROR_SUCCESS) { + if ((ret = write_audio_raw_frame((char*)sh.data(), (int)sh.length(), acodec, (u_int32_t)dts)) != ERROR_SUCCESS) { return ret; } } @@ -625,14 +624,14 @@ int SrsRtspConn::rtmp_write_packet(char type, u_int32_t timestamp, char* data, i SrsSharedPtrMessage* msg = NULL; - if ((ret = srs_rtmp_create_msg(type, timestamp, data, size, stream_id, &msg)) != ERROR_SUCCESS) { + if ((ret = sdk->rtmp_create_msg(type, timestamp, data, size, &msg)) != ERROR_SUCCESS) { srs_error("rtsp: create shared ptr msg failed. ret=%d", ret); return ret; } srs_assert(msg); // send out encoded msg. - if ((ret = client->send_and_free_message(msg, stream_id)) != ERROR_SUCCESS) { + if ((ret = sdk->send_and_free_message(msg)) != ERROR_SUCCESS) { return ret; } @@ -645,122 +644,41 @@ int SrsRtspConn::connect() int ret = ERROR_SUCCESS; // when ok, ignore. - if (io || client) { + // TODO: FIXME: support reconnect. + if (sdk->connected()) { return ret; } - // parse uri + // generate rtmp url to connect to. + std::string url; if (!req) { - req = new SrsRequest(); - - std::string schema, host, vhost, app, port, param; + std::string schema, host, vhost, app, param; + int port; srs_discovery_tc_url(rtsp_tcUrl, schema, host, vhost, app, port, param); // generate output by template. std::string output = output_template; output = srs_string_replace(output, "[app]", app); output = srs_string_replace(output, "[stream]", rtsp_stream); - - size_t pos = string::npos; - string uri = req->tcUrl = output; - - // tcUrl, stream - if ((pos = uri.rfind("/")) != string::npos) { - req->stream = uri.substr(pos + 1); - req->tcUrl = uri = uri.substr(0, pos); - } - - srs_discovery_tc_url(req->tcUrl, - req->schema, req->host, req->vhost, req->app, req->port, - req->param); } // connect host. - if ((ret = srs_socket_connect(req->host, ::atoi(req->port.c_str()), ST_UTIME_NO_TIMEOUT, &stfd)) != ERROR_SUCCESS) { - srs_error("rtsp: connect server %s:%s failed. ret=%d", req->host.c_str(), req->port.c_str(), ret); - return ret; - } - io = new SrsStSocket(stfd); - client = new SrsRtmpClient(io); - - client->set_recv_timeout(SRS_CONSTS_RTMP_RECV_TIMEOUT_US); - client->set_send_timeout(SRS_CONSTS_RTMP_SEND_TIMEOUT_US); - - // connect to vhost/app - if ((ret = client->handshake()) != ERROR_SUCCESS) { - srs_error("rtsp: handshake with server failed. ret=%d", ret); - return ret; - } - if ((ret = connect_app(req->host, req->port)) != ERROR_SUCCESS) { - srs_error("rtsp: connect with server failed. ret=%d", ret); - return ret; - } - if ((ret = client->create_stream(stream_id)) != ERROR_SUCCESS) { - srs_error("rtsp: connect with server failed, stream_id=%d. ret=%d", stream_id, ret); + int64_t cto = SRS_CONSTS_RTMP_TIMEOUT_US; + int64_t sto = SRS_CONSTS_RTMP_PULSE_TIMEOUT_US; + if ((ret = sdk->connect(url, cto, sto)) != ERROR_SUCCESS) { + srs_error("rtsp: connect %s failed, cto=%"PRId64", sto=%"PRId64". ret=%d", url.c_str(), cto, sto, ret); return ret; } // publish. - if ((ret = client->publish(req->stream, stream_id)) != ERROR_SUCCESS) { - srs_error("rtsp: publish failed, stream=%s, stream_id=%d. ret=%d", - req->stream.c_str(), stream_id, ret); + if ((ret = sdk->publish()) != ERROR_SUCCESS) { + srs_error("rtsp: publish %s failed. ret=%d", url.c_str(), ret); return ret; } return write_sequence_header(); } -// TODO: FIXME: refine the connect_app. -int SrsRtspConn::connect_app(string ep_server, string ep_port) -{ - int ret = ERROR_SUCCESS; - - // args of request takes the srs info. - if (req->args == NULL) { - req->args = SrsAmf0Any::object(); - } - - // notify server the edge identity, - // @see https://github.com/simple-rtmp-server/srs/issues/147 - SrsAmf0Object* data = req->args; - data->set("srs_sig", SrsAmf0Any::str(RTMP_SIG_SRS_KEY)); - data->set("srs_server", SrsAmf0Any::str(RTMP_SIG_SRS_KEY" "RTMP_SIG_SRS_VERSION" ("RTMP_SIG_SRS_URL_SHORT")")); - data->set("srs_license", SrsAmf0Any::str(RTMP_SIG_SRS_LICENSE)); - data->set("srs_role", SrsAmf0Any::str(RTMP_SIG_SRS_ROLE)); - data->set("srs_url", SrsAmf0Any::str(RTMP_SIG_SRS_URL)); - data->set("srs_version", SrsAmf0Any::str(RTMP_SIG_SRS_VERSION)); - data->set("srs_site", SrsAmf0Any::str(RTMP_SIG_SRS_WEB)); - data->set("srs_email", SrsAmf0Any::str(RTMP_SIG_SRS_EMAIL)); - data->set("srs_copyright", SrsAmf0Any::str(RTMP_SIG_SRS_COPYRIGHT)); - data->set("srs_primary", SrsAmf0Any::str(RTMP_SIG_SRS_PRIMARY)); - data->set("srs_authors", SrsAmf0Any::str(RTMP_SIG_SRS_AUTHROS)); - // for edge to directly get the id of client. - data->set("srs_pid", SrsAmf0Any::number(getpid())); - data->set("srs_id", SrsAmf0Any::number(_srs_context->get_id())); - - // local ip of edge - std::vector ips = srs_get_local_ipv4_ips(); - assert(_srs_config->get_stats_network() < (int)ips.size()); - std::string local_ip = ips[_srs_config->get_stats_network()]; - data->set("srs_server_ip", SrsAmf0Any::str(local_ip.c_str())); - - // generate the tcUrl - std::string param = ""; - std::string tc_url = srs_generate_tc_url(ep_server, req->vhost, req->app, ep_port, param); - - // upnode server identity will show in the connect_app of client. - // @see https://github.com/simple-rtmp-server/srs/issues/160 - // the debug_srs_upnode is config in vhost and default to true. - bool debug_srs_upnode = _srs_config->get_debug_srs_upnode(req->vhost); - if ((ret = client->connect_app(req->app, tc_url, req, debug_srs_upnode)) != ERROR_SUCCESS) { - srs_error("rtsp: connect with server failed, tcUrl=%s, dsu=%d. ret=%d", - tc_url.c_str(), debug_srs_upnode, ret); - return ret; - } - - return ret; -} - SrsRtspCaster::SrsRtspCaster(SrsConfDirective* c) { // TODO: FIXME: support reload. diff --git a/trunk/src/app/srs_app_rtsp.hpp b/trunk/src/app/srs_app_rtsp.hpp index 40c9c599a..353b1b351 100644 --- a/trunk/src/app/srs_app_rtsp.hpp +++ b/trunk/src/app/srs_app_rtsp.hpp @@ -56,6 +56,7 @@ class SrsSharedPtrMessage; class SrsCodecSample; class SrsSimpleStream; class SrsPithyPrint; +class SrsSimpleRtmpClient; /** * a rtp connection which transport a stream. @@ -139,11 +140,9 @@ private: SrsOneCycleThread* trd; private: SrsRequest* req; - SrsStSocket* io; - SrsRtmpClient* client; + SrsSimpleRtmpClient* sdk; SrsRtspJitter* vjitter; SrsRtspJitter* ajitter; - int stream_id; private: SrsRawH264Stream* avc; std::string h264_sps; @@ -181,7 +180,6 @@ private: // connect to rtmp output url. // @remark ignore when not connected, reconnect when disconnected. virtual int connect(); - virtual int connect_app(std::string ep_server, std::string ep_port); }; /** diff --git a/trunk/src/app/srs_app_server.cpp b/trunk/src/app/srs_app_server.cpp index 89d568c20..896f8622c 100755 --- a/trunk/src/app/srs_app_server.cpp +++ b/trunk/src/app/srs_app_server.cpp @@ -1267,7 +1267,7 @@ int SrsServer::accept_client(SrsListenerType type, st_netfd_t client_stfd) SrsConnection* conn = NULL; if (type == SrsListenerRtmpStream) { - conn = new SrsRtmpConn(this, client_stfd); + conn = new SrsRtmpConn(this, kafka, client_stfd); } else if (type == SrsListenerHttpApi) { #ifdef SRS_AUTO_HTTP_API conn = new SrsHttpApi(this, client_stfd, http_api_mux); diff --git a/trunk/src/app/srs_app_st.cpp b/trunk/src/app/srs_app_st.cpp index 810eccdb5..f240d6798 100644 --- a/trunk/src/app/srs_app_st.cpp +++ b/trunk/src/app/srs_app_st.cpp @@ -23,8 +23,12 @@ CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. #include +#include +using namespace std; + #include #include +#include namespace internal { @@ -406,6 +410,108 @@ int SrsStSocket::writev(const iovec *iov, int iov_size, ssize_t* nwrite) return ret; } +SrsTcpClient::SrsTcpClient() +{ + io = NULL; + stfd = NULL; +} + +SrsTcpClient::~SrsTcpClient() +{ + close(); +} + +bool SrsTcpClient::connected() +{ + return io; +} + +int SrsTcpClient::connect(string host, int port, int64_t timeout) +{ + int ret = ERROR_SUCCESS; + + // when connected, ignore. + if (io) { + return ret; + } + + // connect host. + if ((ret = srs_socket_connect(host, port, timeout, &stfd)) != ERROR_SUCCESS) { + srs_error("mpegts: connect server %s:%d failed. ret=%d", host.c_str(), port, ret); + return ret; + } + + io = new SrsStSocket(stfd); + + return ret; +} + +void SrsTcpClient::close() +{ + // when closed, ignore. + if (!io) { + return; + } + + srs_freep(io); + srs_close_stfd(stfd); +} + +bool SrsTcpClient::is_never_timeout(int64_t timeout_us) +{ + return io->is_never_timeout(timeout_us); +} + +void SrsTcpClient::set_recv_timeout(int64_t timeout_us) +{ + io->set_recv_timeout(timeout_us); +} + +int64_t SrsTcpClient::get_recv_timeout() +{ + return io->get_recv_timeout(); +} + +void SrsTcpClient::set_send_timeout(int64_t timeout_us) +{ + io->set_send_timeout(timeout_us); +} + +int64_t SrsTcpClient::get_send_timeout() +{ + return io->get_send_timeout(); +} + +int64_t SrsTcpClient::get_recv_bytes() +{ + return io->get_recv_bytes(); +} + +int64_t SrsTcpClient::get_send_bytes() +{ + return io->get_send_bytes(); +} + +int SrsTcpClient::read(void* buf, size_t size, ssize_t* nread) +{ + return io->read(buf, size, nread); +} + +int SrsTcpClient::read_fully(void* buf, size_t size, ssize_t* nread) +{ + return io->read_fully(buf, size, nread); +} + +int SrsTcpClient::write(void* buf, size_t size, ssize_t* nwrite) +{ + return io->write(buf, size, nwrite); +} + +int SrsTcpClient::writev(const iovec *iov, int iov_size, ssize_t* nwrite) +{ + return io->writev(iov, iov_size, nwrite); +} + #ifdef __linux__ #include diff --git a/trunk/src/app/srs_app_st.hpp b/trunk/src/app/srs_app_st.hpp index ff28c149f..faa30f6d5 100644 --- a/trunk/src/app/srs_app_st.hpp +++ b/trunk/src/app/srs_app_st.hpp @@ -30,6 +30,8 @@ CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. #include +#include + #include #include @@ -202,6 +204,52 @@ public: virtual int writev(const iovec *iov, int iov_size, ssize_t* nwrite); }; +/** + * the common tcp client, to connect to specified TCP server, + * reconnect and close the connection. + */ +class SrsTcpClient : public ISrsProtocolReaderWriter +{ +private: + st_netfd_t stfd; + SrsStSocket* io; +public: + SrsTcpClient(); + virtual ~SrsTcpClient(); +public: + /** + * whether connected to server. + */ + virtual bool connected(); +public: + /** + * connect to server over TCP. + * @param host the ip or hostname of server. + * @param port the port to connect to. + * @param timeout the timeout in us. + * @remark ignore when connected. + */ + virtual int connect(std::string host, int port, int64_t timeout); + /** + * close the connection. + * @remark ignore when closed. + */ + virtual void close(); +// interface ISrsProtocolReaderWriter +public: + virtual bool is_never_timeout(int64_t timeout_us); + virtual void set_recv_timeout(int64_t timeout_us); + virtual int64_t get_recv_timeout(); + virtual void set_send_timeout(int64_t timeout_us); + virtual int64_t get_send_timeout(); + virtual int64_t get_recv_bytes(); + virtual int64_t get_send_bytes(); + virtual int read(void* buf, size_t size, ssize_t* nread); + virtual int read_fully(void* buf, size_t size, ssize_t* nread); + virtual int write(void* buf, size_t size, ssize_t* nwrite); + virtual int writev(const iovec *iov, int iov_size, ssize_t* nwrite); +}; + // initialize st, requires epoll. extern int srs_st_init(); diff --git a/trunk/src/app/srs_app_utility.cpp b/trunk/src/app/srs_app_utility.cpp index 490cff7ee..f978909dc 100644 --- a/trunk/src/app/srs_app_utility.cpp +++ b/trunk/src/app/srs_app_utility.cpp @@ -1467,17 +1467,3 @@ void srs_api_dump_summaries(SrsJsonObject* obj) sys->set("conn_srs", SrsJsonAny::integer(nrs->nb_conn_srs)); } -string srs_join_vector_string(vector& vs, string separator) -{ - string str = ""; - - for (int i = 0; i < (int)vs.size(); i++) { - str += vs.at(i); - if (i != (int)vs.size() - 1) { - str += separator; - } - } - - return str; -} - diff --git a/trunk/src/app/srs_app_utility.hpp b/trunk/src/app/srs_app_utility.hpp index 2ef07d330..971bdbe99 100644 --- a/trunk/src/app/srs_app_utility.hpp +++ b/trunk/src/app/srs_app_utility.hpp @@ -677,8 +677,5 @@ extern bool srs_is_boolean(const std::string& str); // dump summaries for /api/v1/summaries. extern void srs_api_dump_summaries(SrsJsonObject* obj); -// join string in vector with indicated separator -extern std::string srs_join_vector_string(std::vector& vs, std::string separator); - #endif diff --git a/trunk/src/core/srs_core.hpp b/trunk/src/core/srs_core.hpp index 56e43471b..aa933d018 100644 --- a/trunk/src/core/srs_core.hpp +++ b/trunk/src/core/srs_core.hpp @@ -31,7 +31,7 @@ CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. // current release version #define VERSION_MAJOR 3 #define VERSION_MINOR 0 -#define VERSION_REVISION 5 +#define VERSION_REVISION 6 // server info. #define RTMP_SIG_SRS_KEY "SRS" diff --git a/trunk/src/kernel/srs_kernel_buffer.cpp b/trunk/src/kernel/srs_kernel_buffer.cpp index 2c998b198..96a04e8b0 100644 --- a/trunk/src/kernel/srs_kernel_buffer.cpp +++ b/trunk/src/kernel/srs_kernel_buffer.cpp @@ -29,19 +29,37 @@ using namespace std; #include #include +ISrsCodec::ISrsCodec() +{ +} + +ISrsCodec::~ISrsCodec() +{ +} + SrsBuffer::SrsBuffer() { - p = bytes = NULL; - nb_bytes = 0; - - // TODO: support both little and big endian. - srs_assert(srs_is_little_endian()); + set_value(NULL, 0); +} + +SrsBuffer::SrsBuffer(char* b, int nb_b) +{ + set_value(b, nb_b); } SrsBuffer::~SrsBuffer() { } +void SrsBuffer::set_value(char* b, int nb_b) +{ + p = bytes = b; + nb_bytes = nb_b; + + // TODO: support both little and big endian. + srs_assert(srs_is_little_endian()); +} + int SrsBuffer::initialize(char* b, int nb) { int ret = ERROR_SUCCESS; diff --git a/trunk/src/kernel/srs_kernel_buffer.hpp b/trunk/src/kernel/srs_kernel_buffer.hpp index f20d57933..46ce3f012 100644 --- a/trunk/src/kernel/srs_kernel_buffer.hpp +++ b/trunk/src/kernel/srs_kernel_buffer.hpp @@ -33,11 +33,60 @@ CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. #include #include +class SrsBuffer; + /** -* bytes utility, used to: -* convert basic types to bytes, -* build basic types from bytes. -*/ + * the srs codec, to code and decode object with bytes: + * code: to encode/serialize object to bytes in buffer, + * decode: to decode/deserialize object from bytes in buffer. + * we use SrsBuffer as bytes helper utility, + * for example, to code: + * ISrsCodec* obj = ... + * char* bytes = new char[obj->size()]; + * + * SrsBuffer* buf = new SrsBuffer(); + * buf->initialize(bytes, obj->size()) + * + * obj->encode(buf); + * for example, to decode: + * int nb_bytes = ... + * char* bytes = ... + * + * SrsBuffer* buf = new Srsbuffer(); + * buf->initialize(bytes, nb_bytes); + * + * ISrsCodec* obj = ... + * obj->decode(buf); + * @remark protocol or amf0 or json should implements this interface. + */ +// TODO: FIXME: protocol, amf0, json should implements it. +class ISrsCodec +{ +public: + ISrsCodec(); + virtual ~ISrsCodec(); +public: + /** + * get the number of bytes to code to. + */ + virtual int nb_bytes() = 0; + /** + * encode object to bytes in SrsBuffer. + */ + virtual int encode(SrsBuffer* buf) = 0; +public: + /** + * decode object from bytes in SrsBuffer. + */ + virtual int decode(SrsBuffer* buf) = 0; +}; + +/** + * bytes utility, used to: + * convert basic types to bytes, + * build basic types from bytes. + * @remark the buffer never mange the bytes, user must manage it. + */ class SrsBuffer { private: @@ -49,7 +98,10 @@ private: int nb_bytes; public: SrsBuffer(); + SrsBuffer(char* b, int nb_b); virtual ~SrsBuffer(); +private: + virtual void set_value(char* b, int nb_b); public: /** * initialize the stream from bytes. @@ -157,7 +209,8 @@ public: }; /** - * the bit stream. + * the bit stream, base on SrsBuffer, + * for exmaple, the h.264 avc stream is bit stream. */ class SrsBitBuffer { diff --git a/trunk/src/kernel/srs_kernel_consts.hpp b/trunk/src/kernel/srs_kernel_consts.hpp index bc05023b6..34b4a0fa4 100644 --- a/trunk/src/kernel/srs_kernel_consts.hpp +++ b/trunk/src/kernel/srs_kernel_consts.hpp @@ -68,13 +68,8 @@ CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. // the following is the timeout for rtmp protocol, // to avoid death connection. -// the timeout to send data to client, -// if timeout, close the connection. -#define SRS_CONSTS_RTMP_SEND_TIMEOUT_US (int64_t)(30*1000*1000LL) - -// the timeout to wait client data, -// if timeout, close the connection. -#define SRS_CONSTS_RTMP_RECV_TIMEOUT_US (int64_t)(30*1000*1000LL) +// the common io timeout, for both recv and send. +#define SRS_CONSTS_RTMP_TIMEOUT_US (int64_t)(30*1000*1000LL) // the timeout to wait for client control message, // if timeout, we generally ignore and send the data to client, @@ -403,5 +398,13 @@ CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. #define SRS_CONSTS_RTSP_RTSPVersionNotSupported_str "RTSP Version Not Supported" #define SRS_CONSTS_RTSP_OptionNotSupported_str "Option not support" +/////////////////////////////////////////////////////////// +// KAFKA consts values +/////////////////////////////////////////////////////////// +#define SRS_CONSTS_KAFKA_DEFAULT_PORT 9092 + +// the common io timeout, for both recv and send. +#define SRS_CONSTS_KAFKA_TIMEOUT_US (int64_t)(30*1000*1000LL) + #endif diff --git a/trunk/src/kernel/srs_kernel_error.hpp b/trunk/src/kernel/srs_kernel_error.hpp index 5e1fed433..4435450f9 100755 --- a/trunk/src/kernel/srs_kernel_error.hpp +++ b/trunk/src/kernel/srs_kernel_error.hpp @@ -236,7 +236,7 @@ CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. #define ERROR_REQUEST_DATA 3066 /////////////////////////////////////////////////////// -// HTTP/StreamCaster protocol error. +// HTTP/StreamCaster/KAFKA protocol error. /////////////////////////////////////////////////////// #define ERROR_HTTP_PATTERN_EMPTY 4000 #define ERROR_HTTP_PATTERN_DUPLICATED 4001 @@ -268,6 +268,14 @@ CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. #define ERROR_AVC_NALU_UEV 4027 #define ERROR_AAC_BYTES_INVALID 4028 #define ERROR_HTTP_REQUEST_EOF 4029 +#define ERROR_KAFKA_CODEC_STRING 4030 +#define ERROR_KAFKA_CODEC_BYTES 4031 +#define ERROR_KAFKA_CODEC_REQUEST 4032 +#define ERROR_KAFKA_CODEC_RESPONSE 4033 +#define ERROR_KAFKA_CODEC_ARRAY 4034 +#define ERROR_KAFKA_CODEC_METADATA 4035 +#define ERROR_KAFKA_CODEC_MESSAGE 4036 +#define ERROR_KAFKA_CODEC_PRODUCER 4037 /////////////////////////////////////////////////////// // HTTP API error. diff --git a/trunk/src/kernel/srs_kernel_ts.cpp b/trunk/src/kernel/srs_kernel_ts.cpp index 1b3f144a1..a532ae821 100644 --- a/trunk/src/kernel/srs_kernel_ts.cpp +++ b/trunk/src/kernel/srs_kernel_ts.cpp @@ -2141,7 +2141,7 @@ int SrsTsPayloadPSI::decode(SrsBuffer* stream, SrsTsMessage** /*ppmsg*/) CRC_32 = stream->read_4bytes(); // verify crc32. - int32_t crc32 = srs_crc32(ppat, stream->pos() - pat_pos - 4); + int32_t crc32 = srs_crc32_mpegts(ppat, stream->pos() - pat_pos - 4); if (crc32 != CRC_32) { ret = ERROR_STREAM_CASTER_TS_CRC32; srs_error("ts: verify PSI crc32 failed. ret=%d", ret); @@ -2238,7 +2238,7 @@ int SrsTsPayloadPSI::encode(SrsBuffer* stream) srs_error("ts: mux PSI crc32 failed. ret=%d", ret); return ret; } - CRC_32 = srs_crc32(ppat, stream->pos() - pat_pos); + CRC_32 = srs_crc32_mpegts(ppat, stream->pos() - pat_pos); stream->write_4bytes(CRC_32); return ret; diff --git a/trunk/src/kernel/srs_kernel_utility.cpp b/trunk/src/kernel/srs_kernel_utility.cpp index 550c8d2cd..76cb1ff63 100644 --- a/trunk/src/kernel/srs_kernel_utility.cpp +++ b/trunk/src/kernel/srs_kernel_utility.cpp @@ -464,6 +464,29 @@ string srs_path_basename(string path) return dirname; } + +string srs_path_filename(string path) +{ + std::string filename = path; + size_t pos = string::npos; + + if ((pos = filename.rfind(".")) != string::npos) { + return filename.substr(0, pos); + } + + return filename; +} + +string srs_path_filext(string path) +{ + size_t pos = string::npos; + + if ((pos = path.rfind(".")) != string::npos) { + return path.substr(pos); + } + + return ""; +} bool srs_avc_startswith_annexb(SrsBuffer* stream, int* pnb_start_code) { @@ -471,7 +494,7 @@ bool srs_avc_startswith_annexb(SrsBuffer* stream, int* pnb_start_code) char* p = bytes; for (;;) { - if (!stream->require(p - bytes + 3)) { + if (!stream->require((int)(p - bytes + 3))) { return false; } @@ -512,85 +535,163 @@ bool srs_aac_startswith_adts(SrsBuffer* stream) return true; } -/* - * MPEG2 transport stream (aka DVB) mux - * Copyright (c) 2003 Fabrice Bellard. - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 2 of the License, or (at your option) any later version. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with this library; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA - */ -static const u_int32_t crc_table[256] = { - 0x00000000, 0x04c11db7, 0x09823b6e, 0x0d4326d9, 0x130476dc, 0x17c56b6b, - 0x1a864db2, 0x1e475005, 0x2608edb8, 0x22c9f00f, 0x2f8ad6d6, 0x2b4bcb61, - 0x350c9b64, 0x31cd86d3, 0x3c8ea00a, 0x384fbdbd, 0x4c11db70, 0x48d0c6c7, - 0x4593e01e, 0x4152fda9, 0x5f15adac, 0x5bd4b01b, 0x569796c2, 0x52568b75, - 0x6a1936c8, 0x6ed82b7f, 0x639b0da6, 0x675a1011, 0x791d4014, 0x7ddc5da3, - 0x709f7b7a, 0x745e66cd, 0x9823b6e0, 0x9ce2ab57, 0x91a18d8e, 0x95609039, - 0x8b27c03c, 0x8fe6dd8b, 0x82a5fb52, 0x8664e6e5, 0xbe2b5b58, 0xbaea46ef, - 0xb7a96036, 0xb3687d81, 0xad2f2d84, 0xa9ee3033, 0xa4ad16ea, 0xa06c0b5d, - 0xd4326d90, 0xd0f37027, 0xddb056fe, 0xd9714b49, 0xc7361b4c, 0xc3f706fb, - 0xceb42022, 0xca753d95, 0xf23a8028, 0xf6fb9d9f, 0xfbb8bb46, 0xff79a6f1, - 0xe13ef6f4, 0xe5ffeb43, 0xe8bccd9a, 0xec7dd02d, 0x34867077, 0x30476dc0, - 0x3d044b19, 0x39c556ae, 0x278206ab, 0x23431b1c, 0x2e003dc5, 0x2ac12072, - 0x128e9dcf, 0x164f8078, 0x1b0ca6a1, 0x1fcdbb16, 0x018aeb13, 0x054bf6a4, - 0x0808d07d, 0x0cc9cdca, 0x7897ab07, 0x7c56b6b0, 0x71159069, 0x75d48dde, - 0x6b93dddb, 0x6f52c06c, 0x6211e6b5, 0x66d0fb02, 0x5e9f46bf, 0x5a5e5b08, - 0x571d7dd1, 0x53dc6066, 0x4d9b3063, 0x495a2dd4, 0x44190b0d, 0x40d816ba, - 0xaca5c697, 0xa864db20, 0xa527fdf9, 0xa1e6e04e, 0xbfa1b04b, 0xbb60adfc, - 0xb6238b25, 0xb2e29692, 0x8aad2b2f, 0x8e6c3698, 0x832f1041, 0x87ee0df6, - 0x99a95df3, 0x9d684044, 0x902b669d, 0x94ea7b2a, 0xe0b41de7, 0xe4750050, - 0xe9362689, 0xedf73b3e, 0xf3b06b3b, 0xf771768c, 0xfa325055, 0xfef34de2, - 0xc6bcf05f, 0xc27dede8, 0xcf3ecb31, 0xcbffd686, 0xd5b88683, 0xd1799b34, - 0xdc3abded, 0xd8fba05a, 0x690ce0ee, 0x6dcdfd59, 0x608edb80, 0x644fc637, - 0x7a089632, 0x7ec98b85, 0x738aad5c, 0x774bb0eb, 0x4f040d56, 0x4bc510e1, - 0x46863638, 0x42472b8f, 0x5c007b8a, 0x58c1663d, 0x558240e4, 0x51435d53, - 0x251d3b9e, 0x21dc2629, 0x2c9f00f0, 0x285e1d47, 0x36194d42, 0x32d850f5, - 0x3f9b762c, 0x3b5a6b9b, 0x0315d626, 0x07d4cb91, 0x0a97ed48, 0x0e56f0ff, - 0x1011a0fa, 0x14d0bd4d, 0x19939b94, 0x1d528623, 0xf12f560e, 0xf5ee4bb9, - 0xf8ad6d60, 0xfc6c70d7, 0xe22b20d2, 0xe6ea3d65, 0xeba91bbc, 0xef68060b, - 0xd727bbb6, 0xd3e6a601, 0xdea580d8, 0xda649d6f, 0xc423cd6a, 0xc0e2d0dd, - 0xcda1f604, 0xc960ebb3, 0xbd3e8d7e, 0xb9ff90c9, 0xb4bcb610, 0xb07daba7, - 0xae3afba2, 0xaafbe615, 0xa7b8c0cc, 0xa379dd7b, 0x9b3660c6, 0x9ff77d71, - 0x92b45ba8, 0x9675461f, 0x8832161a, 0x8cf30bad, 0x81b02d74, 0x857130c3, - 0x5d8a9099, 0x594b8d2e, 0x5408abf7, 0x50c9b640, 0x4e8ee645, 0x4a4ffbf2, - 0x470cdd2b, 0x43cdc09c, 0x7b827d21, 0x7f436096, 0x7200464f, 0x76c15bf8, - 0x68860bfd, 0x6c47164a, 0x61043093, 0x65c52d24, 0x119b4be9, 0x155a565e, - 0x18197087, 0x1cd86d30, 0x029f3d35, 0x065e2082, 0x0b1d065b, 0x0fdc1bec, - 0x3793a651, 0x3352bbe6, 0x3e119d3f, 0x3ad08088, 0x2497d08d, 0x2056cd3a, - 0x2d15ebe3, 0x29d4f654, 0xc5a92679, 0xc1683bce, 0xcc2b1d17, 0xc8ea00a0, - 0xd6ad50a5, 0xd26c4d12, 0xdf2f6bcb, 0xdbee767c, 0xe3a1cbc1, 0xe760d676, - 0xea23f0af, 0xeee2ed18, 0xf0a5bd1d, 0xf464a0aa, 0xf9278673, 0xfde69bc4, - 0x89b8fd09, 0x8d79e0be, 0x803ac667, 0x84fbdbd0, 0x9abc8bd5, 0x9e7d9662, - 0x933eb0bb, 0x97ffad0c, 0xafb010b1, 0xab710d06, 0xa6322bdf, 0xa2f33668, - 0xbcb4666d, 0xb8757bda, 0xb5365d03, 0xb1f740b4 -}; - // @see http://www.stmc.edu.hk/~vincent/ffmpeg_0.4.9-pre1/libavformat/mpegtsenc.c -unsigned int mpegts_crc32(const u_int8_t *data, int len) +unsigned int __mpegts_crc32(const u_int8_t *data, int len) { - register int i; - unsigned int crc = 0xffffffff; + /* + * MPEG2 transport stream (aka DVB) mux + * Copyright (c) 2003 Fabrice Bellard. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ + static const u_int32_t table[256] = { + 0x00000000, 0x04c11db7, 0x09823b6e, 0x0d4326d9, 0x130476dc, 0x17c56b6b, + 0x1a864db2, 0x1e475005, 0x2608edb8, 0x22c9f00f, 0x2f8ad6d6, 0x2b4bcb61, + 0x350c9b64, 0x31cd86d3, 0x3c8ea00a, 0x384fbdbd, 0x4c11db70, 0x48d0c6c7, + 0x4593e01e, 0x4152fda9, 0x5f15adac, 0x5bd4b01b, 0x569796c2, 0x52568b75, + 0x6a1936c8, 0x6ed82b7f, 0x639b0da6, 0x675a1011, 0x791d4014, 0x7ddc5da3, + 0x709f7b7a, 0x745e66cd, 0x9823b6e0, 0x9ce2ab57, 0x91a18d8e, 0x95609039, + 0x8b27c03c, 0x8fe6dd8b, 0x82a5fb52, 0x8664e6e5, 0xbe2b5b58, 0xbaea46ef, + 0xb7a96036, 0xb3687d81, 0xad2f2d84, 0xa9ee3033, 0xa4ad16ea, 0xa06c0b5d, + 0xd4326d90, 0xd0f37027, 0xddb056fe, 0xd9714b49, 0xc7361b4c, 0xc3f706fb, + 0xceb42022, 0xca753d95, 0xf23a8028, 0xf6fb9d9f, 0xfbb8bb46, 0xff79a6f1, + 0xe13ef6f4, 0xe5ffeb43, 0xe8bccd9a, 0xec7dd02d, 0x34867077, 0x30476dc0, + 0x3d044b19, 0x39c556ae, 0x278206ab, 0x23431b1c, 0x2e003dc5, 0x2ac12072, + 0x128e9dcf, 0x164f8078, 0x1b0ca6a1, 0x1fcdbb16, 0x018aeb13, 0x054bf6a4, + 0x0808d07d, 0x0cc9cdca, 0x7897ab07, 0x7c56b6b0, 0x71159069, 0x75d48dde, + 0x6b93dddb, 0x6f52c06c, 0x6211e6b5, 0x66d0fb02, 0x5e9f46bf, 0x5a5e5b08, + 0x571d7dd1, 0x53dc6066, 0x4d9b3063, 0x495a2dd4, 0x44190b0d, 0x40d816ba, + 0xaca5c697, 0xa864db20, 0xa527fdf9, 0xa1e6e04e, 0xbfa1b04b, 0xbb60adfc, + 0xb6238b25, 0xb2e29692, 0x8aad2b2f, 0x8e6c3698, 0x832f1041, 0x87ee0df6, + 0x99a95df3, 0x9d684044, 0x902b669d, 0x94ea7b2a, 0xe0b41de7, 0xe4750050, + 0xe9362689, 0xedf73b3e, 0xf3b06b3b, 0xf771768c, 0xfa325055, 0xfef34de2, + 0xc6bcf05f, 0xc27dede8, 0xcf3ecb31, 0xcbffd686, 0xd5b88683, 0xd1799b34, + 0xdc3abded, 0xd8fba05a, 0x690ce0ee, 0x6dcdfd59, 0x608edb80, 0x644fc637, + 0x7a089632, 0x7ec98b85, 0x738aad5c, 0x774bb0eb, 0x4f040d56, 0x4bc510e1, + 0x46863638, 0x42472b8f, 0x5c007b8a, 0x58c1663d, 0x558240e4, 0x51435d53, + 0x251d3b9e, 0x21dc2629, 0x2c9f00f0, 0x285e1d47, 0x36194d42, 0x32d850f5, + 0x3f9b762c, 0x3b5a6b9b, 0x0315d626, 0x07d4cb91, 0x0a97ed48, 0x0e56f0ff, + 0x1011a0fa, 0x14d0bd4d, 0x19939b94, 0x1d528623, 0xf12f560e, 0xf5ee4bb9, + 0xf8ad6d60, 0xfc6c70d7, 0xe22b20d2, 0xe6ea3d65, 0xeba91bbc, 0xef68060b, + 0xd727bbb6, 0xd3e6a601, 0xdea580d8, 0xda649d6f, 0xc423cd6a, 0xc0e2d0dd, + 0xcda1f604, 0xc960ebb3, 0xbd3e8d7e, 0xb9ff90c9, 0xb4bcb610, 0xb07daba7, + 0xae3afba2, 0xaafbe615, 0xa7b8c0cc, 0xa379dd7b, 0x9b3660c6, 0x9ff77d71, + 0x92b45ba8, 0x9675461f, 0x8832161a, 0x8cf30bad, 0x81b02d74, 0x857130c3, + 0x5d8a9099, 0x594b8d2e, 0x5408abf7, 0x50c9b640, 0x4e8ee645, 0x4a4ffbf2, + 0x470cdd2b, 0x43cdc09c, 0x7b827d21, 0x7f436096, 0x7200464f, 0x76c15bf8, + 0x68860bfd, 0x6c47164a, 0x61043093, 0x65c52d24, 0x119b4be9, 0x155a565e, + 0x18197087, 0x1cd86d30, 0x029f3d35, 0x065e2082, 0x0b1d065b, 0x0fdc1bec, + 0x3793a651, 0x3352bbe6, 0x3e119d3f, 0x3ad08088, 0x2497d08d, 0x2056cd3a, + 0x2d15ebe3, 0x29d4f654, 0xc5a92679, 0xc1683bce, 0xcc2b1d17, 0xc8ea00a0, + 0xd6ad50a5, 0xd26c4d12, 0xdf2f6bcb, 0xdbee767c, 0xe3a1cbc1, 0xe760d676, + 0xea23f0af, 0xeee2ed18, 0xf0a5bd1d, 0xf464a0aa, 0xf9278673, 0xfde69bc4, + 0x89b8fd09, 0x8d79e0be, 0x803ac667, 0x84fbdbd0, 0x9abc8bd5, 0x9e7d9662, + 0x933eb0bb, 0x97ffad0c, 0xafb010b1, 0xab710d06, 0xa6322bdf, 0xa2f33668, + 0xbcb4666d, 0xb8757bda, 0xb5365d03, 0xb1f740b4 + }; - for (i=0; i> 24) ^ *data++) & 0xff]; + u_int32_t crc = 0xffffffff; + + for (int i=0; i> 24) ^ *data++) & 0xff]; + } return crc; } -u_int32_t srs_crc32(const void* buf, int size) +// @see https://github.com/ETrun/crc32/blob/master/crc32.c +u_int32_t __crc32_ieee(u_int32_t init, const u_int8_t* buf, size_t nb_buf) { - return mpegts_crc32((const u_int8_t*)buf, size); + /*----------------------------------------------------------------------------*\ + * CRC-32 version 2.0.0 by Craig Bruce, 2006-04-29. + * + * This program generates the CRC-32 values for the files named in the + * command-line arguments. These are the same CRC-32 values used by GZIP, + * PKZIP, and ZMODEM. The Crc32_ComputeBuf() can also be detached and + * used independently. + * + * THIS PROGRAM IS PUBLIC-DOMAIN SOFTWARE. + * + * Based on the byte-oriented implementation "File Verification Using CRC" + * by Mark R. Nelson in Dr. Dobb's Journal, May 1992, pp. 64-67. + * + * v1.0.0: original release. + * v1.0.1: fixed printf formats. + * v1.0.2: fixed something else. + * v1.0.3: replaced CRC constant table by generator function. + * v1.0.4: reformatted code, made ANSI C. 1994-12-05. + * v2.0.0: rewrote to use memory buffer & static table, 2006-04-29. + * v2.1.0: modified by Nico, 2013-04-20 + \*----------------------------------------------------------------------------*/ + static const u_int32_t table[256] = { + 0x00000000,0x77073096,0xEE0E612C,0x990951BA,0x076DC419,0x706AF48F,0xE963A535, + 0x9E6495A3,0x0EDB8832,0x79DCB8A4,0xE0D5E91E,0x97D2D988,0x09B64C2B,0x7EB17CBD, + 0xE7B82D07,0x90BF1D91,0x1DB71064,0x6AB020F2,0xF3B97148,0x84BE41DE,0x1ADAD47D, + 0x6DDDE4EB,0xF4D4B551,0x83D385C7,0x136C9856,0x646BA8C0,0xFD62F97A,0x8A65C9EC, + 0x14015C4F,0x63066CD9,0xFA0F3D63,0x8D080DF5,0x3B6E20C8,0x4C69105E,0xD56041E4, + 0xA2677172,0x3C03E4D1,0x4B04D447,0xD20D85FD,0xA50AB56B,0x35B5A8FA,0x42B2986C, + 0xDBBBC9D6,0xACBCF940,0x32D86CE3,0x45DF5C75,0xDCD60DCF,0xABD13D59,0x26D930AC, + 0x51DE003A,0xC8D75180,0xBFD06116,0x21B4F4B5,0x56B3C423,0xCFBA9599,0xB8BDA50F, + 0x2802B89E,0x5F058808,0xC60CD9B2,0xB10BE924,0x2F6F7C87,0x58684C11,0xC1611DAB, + 0xB6662D3D,0x76DC4190,0x01DB7106,0x98D220BC,0xEFD5102A,0x71B18589,0x06B6B51F, + 0x9FBFE4A5,0xE8B8D433,0x7807C9A2,0x0F00F934,0x9609A88E,0xE10E9818,0x7F6A0DBB, + 0x086D3D2D,0x91646C97,0xE6635C01,0x6B6B51F4,0x1C6C6162,0x856530D8,0xF262004E, + 0x6C0695ED,0x1B01A57B,0x8208F4C1,0xF50FC457,0x65B0D9C6,0x12B7E950,0x8BBEB8EA, + 0xFCB9887C,0x62DD1DDF,0x15DA2D49,0x8CD37CF3,0xFBD44C65,0x4DB26158,0x3AB551CE, + 0xA3BC0074,0xD4BB30E2,0x4ADFA541,0x3DD895D7,0xA4D1C46D,0xD3D6F4FB,0x4369E96A, + 0x346ED9FC,0xAD678846,0xDA60B8D0,0x44042D73,0x33031DE5,0xAA0A4C5F,0xDD0D7CC9, + 0x5005713C,0x270241AA,0xBE0B1010,0xC90C2086,0x5768B525,0x206F85B3,0xB966D409, + 0xCE61E49F,0x5EDEF90E,0x29D9C998,0xB0D09822,0xC7D7A8B4,0x59B33D17,0x2EB40D81, + 0xB7BD5C3B,0xC0BA6CAD,0xEDB88320,0x9ABFB3B6,0x03B6E20C,0x74B1D29A,0xEAD54739, + 0x9DD277AF,0x04DB2615,0x73DC1683,0xE3630B12,0x94643B84,0x0D6D6A3E,0x7A6A5AA8, + 0xE40ECF0B,0x9309FF9D,0x0A00AE27,0x7D079EB1,0xF00F9344,0x8708A3D2,0x1E01F268, + 0x6906C2FE,0xF762575D,0x806567CB,0x196C3671,0x6E6B06E7,0xFED41B76,0x89D32BE0, + 0x10DA7A5A,0x67DD4ACC,0xF9B9DF6F,0x8EBEEFF9,0x17B7BE43,0x60B08ED5,0xD6D6A3E8, + 0xA1D1937E,0x38D8C2C4,0x4FDFF252,0xD1BB67F1,0xA6BC5767,0x3FB506DD,0x48B2364B, + 0xD80D2BDA,0xAF0A1B4C,0x36034AF6,0x41047A60,0xDF60EFC3,0xA867DF55,0x316E8EEF, + 0x4669BE79,0xCB61B38C,0xBC66831A,0x256FD2A0,0x5268E236,0xCC0C7795,0xBB0B4703, + 0x220216B9,0x5505262F,0xC5BA3BBE,0xB2BD0B28,0x2BB45A92,0x5CB36A04,0xC2D7FFA7, + 0xB5D0CF31,0x2CD99E8B,0x5BDEAE1D,0x9B64C2B0,0xEC63F226,0x756AA39C,0x026D930A, + 0x9C0906A9,0xEB0E363F,0x72076785,0x05005713,0x95BF4A82,0xE2B87A14,0x7BB12BAE, + 0x0CB61B38,0x92D28E9B,0xE5D5BE0D,0x7CDCEFB7,0x0BDBDF21,0x86D3D2D4,0xF1D4E242, + 0x68DDB3F8,0x1FDA836E,0x81BE16CD,0xF6B9265B,0x6FB077E1,0x18B74777,0x88085AE6, + 0xFF0F6A70,0x66063BCA,0x11010B5C,0x8F659EFF,0xF862AE69,0x616BFFD3,0x166CCF45, + 0xA00AE278,0xD70DD2EE,0x4E048354,0x3903B3C2,0xA7672661,0xD06016F7,0x4969474D, + 0x3E6E77DB,0xAED16A4A,0xD9D65ADC,0x40DF0B66,0x37D83BF0,0xA9BCAE53,0xDEBB9EC5, + 0x47B2CF7F,0x30B5FFE9,0xBDBDF21C,0xCABAC28A,0x53B39330,0x24B4A3A6,0xBAD03605, + 0xCDD70693,0x54DE5729,0x23D967BF,0xB3667A2E,0xC4614AB8,0x5D681B02,0x2A6F2B94, + 0xB40BBE37,0xC30C8EA1,0x5A05DF1B,0x2D02EF8D + }; + + u_int32_t crc = init ^ 0xFFFFFFFF; + + for (int i = 0; i < nb_buf; i++) { + crc = table[(crc ^ buf[i]) & 0xff] ^ (crc >> 8); + } + + return crc^0xFFFFFFFF; +} + +u_int32_t srs_crc32_mpegts(const void* buf, int size) +{ + return __mpegts_crc32((const u_int8_t*)buf, size); +} + +u_int32_t srs_crc32_ieee(const void* buf, int size, u_int32_t previous) +{ + return __crc32_ieee(previous, (const u_int8_t*)buf, size); } /* diff --git a/trunk/src/kernel/srs_kernel_utility.hpp b/trunk/src/kernel/srs_kernel_utility.hpp index d2bc344ec..efaf5e876 100644 --- a/trunk/src/kernel/srs_kernel_utility.hpp +++ b/trunk/src/kernel/srs_kernel_utility.hpp @@ -96,10 +96,14 @@ extern int srs_create_dir_recursively(std::string dir); // whether path exists. extern bool srs_path_exists(std::string path); -// get the dirname of path +// get the dirname of path, for instance, filename("/live/livestream")="/live" extern std::string srs_path_dirname(std::string path); -// get the basename of path +// get the basename of path, for instance, filename("/live/livestream")="livestream" extern std::string srs_path_basename(std::string path); +// get the filename of path, for instance, filename("livestream.flv")="livestream" +extern std::string srs_path_filename(std::string path); +// get the file extension of path, for instance, filext("live.flv")=".flv" +extern std::string srs_path_filext(std::string path); /** * whether stream starts with the avc NALU in "AnnexB" @@ -118,9 +122,14 @@ extern bool srs_avc_startswith_annexb(SrsBuffer* stream, int* pnb_start_code = N extern bool srs_aac_startswith_adts(SrsBuffer* stream); /** -* cacl the crc32 of bytes in buf. -*/ -extern u_int32_t srs_crc32(const void* buf, int size); +* cacl the crc32 of bytes in buf, for ffmpeg. + */ +extern u_int32_t srs_crc32_mpegts(const void* buf, int size); + +/** + * calc the crc32 of bytes in buf by IEEE, for zip. + */ +extern u_int32_t srs_crc32_ieee(const void* buf, int size, u_int32_t previous = 0); /** * Decode a base64-encoded string. diff --git a/trunk/src/libs/srs_librtmp.cpp b/trunk/src/libs/srs_librtmp.cpp index d8592d482..e2fa7b9dd 100644 --- a/trunk/src/libs/srs_librtmp.cpp +++ b/trunk/src/libs/srs_librtmp.cpp @@ -464,16 +464,9 @@ int srs_librtmp_context_parse_uri(Context* context) { int ret = ERROR_SUCCESS; - // parse uri - size_t pos = string::npos; - string uri = context->url; - // tcUrl, stream - if ((pos = uri.rfind("/")) != string::npos) { - context->stream = uri.substr(pos + 1); - context->tcUrl = uri = uri.substr(0, pos); - } - std::string schema; + + srs_parse_rtmp_url(context->url, context->tcUrl, context->stream); srs_discovery_tc_url(context->tcUrl, schema, context->host, context->vhost, context->app, context->port, context->param); diff --git a/trunk/src/main/srs_main_ingest_hls.cpp b/trunk/src/main/srs_main_ingest_hls.cpp index 8d84e0faa..f32c20f2d 100644 --- a/trunk/src/main/srs_main_ingest_hls.cpp +++ b/trunk/src/main/srs_main_ingest_hls.cpp @@ -47,6 +47,7 @@ using namespace std; #include #include #include +#include // pre-declare int proxy_hls2rtmp(std::string hls, std::string rtmp); @@ -378,7 +379,7 @@ int SrsIngestSrsInput::parseM3u8(SrsHttpUri* url, double& td, double& duration) int ret = ERROR_SUCCESS; SrsHttpClient client; - srs_trace("parse input hls %s", url->get_url()); + srs_trace("parse input hls %s", url->get_url().c_str()); if ((ret = client.initialize(url->get_host(), url->get_port())) != ERROR_SUCCESS) { srs_error("connect to server failed. ret=%d", ret); @@ -387,7 +388,7 @@ int SrsIngestSrsInput::parseM3u8(SrsHttpUri* url, double& td, double& duration) ISrsHttpMessage* msg = NULL; if ((ret = client.get(url->get_path(), "", &msg)) != ERROR_SUCCESS) { - srs_error("HTTP GET %s failed. ret=%d", url->get_url(), ret); + srs_error("HTTP GET %s failed. ret=%d", url->get_url().c_str(), ret); return ret; } @@ -609,7 +610,7 @@ int SrsIngestSrsInput::SrsTsPiece::fetch(string m3u8) ISrsHttpMessage* msg = NULL; if ((ret = client.get(uri.get_path(), "", &msg)) != ERROR_SUCCESS) { - srs_error("HTTP GET %s failed. ret=%d", uri.get_url(), ret); + srs_error("HTTP GET %s failed. ret=%d", uri.get_url().c_str(), ret); return ret; } @@ -637,10 +638,7 @@ private: int64_t raw_aac_dts; private: SrsRequest* req; - st_netfd_t stfd; - SrsStSocket* io; - SrsRtmpClient* client; - int stream_id; + SrsSimpleRtmpClient* sdk; private: SrsRawH264Stream* avc; std::string h264_sps; @@ -658,10 +656,7 @@ public: raw_aac_dts = srs_update_system_time_ms(); req = NULL; - io = NULL; - client = NULL; - stfd = NULL; - stream_id = 0; + sdk = new SrsSimpleRtmpClient(); avc = new SrsRawH264Stream(); aac = new SrsRawAacStream(); @@ -672,6 +667,7 @@ public: virtual ~SrsIngestSrsOutput() { close(); + srs_freep(sdk); srs_freep(avc); srs_freep(aac); @@ -708,7 +704,6 @@ public: */ virtual int flush_message_queue(); private: - virtual int connect_app(std::string ep_server, int ep_port); // close the connected io and rtmp to ready to be re-connect. virtual void close(); }; @@ -1188,7 +1183,7 @@ int SrsIngestSrsOutput::rtmp_write_packet(char type, u_int32_t timestamp, char* SrsSharedPtrMessage* msg = NULL; - if ((ret = srs_rtmp_create_msg(type, timestamp, data, size, stream_id, &msg)) != ERROR_SUCCESS) { + if ((ret = sdk->rtmp_create_msg(type, timestamp, data, size, &msg)) != ERROR_SUCCESS) { srs_error("mpegts: create shared ptr msg failed. ret=%d", ret); return ret; } @@ -1197,7 +1192,7 @@ int SrsIngestSrsOutput::rtmp_write_packet(char type, u_int32_t timestamp, char* srs_info("RTMP type=%d, dts=%d, size=%d", type, timestamp, size); // send out encoded msg. - if ((ret = client->send_and_free_message(msg, stream_id)) != ERROR_SUCCESS) { + if ((ret = sdk->send_and_free_message(msg)) != ERROR_SUCCESS) { srs_error("send RTMP type=%d, dts=%d, size=%d failed. ret=%d", type, timestamp, size, ret); return ret; } @@ -1211,109 +1206,24 @@ int SrsIngestSrsOutput::connect() // when ok, ignore. // TODO: FIXME: should reconnect when disconnected. - if (io || client) { + if (sdk->connected()) { return ret; } - srs_trace("connect output=%s", out_rtmp->get_url()); - - // parse uri - if (!req) { - req = new SrsRequest(); - - string uri = req->tcUrl = out_rtmp->get_url(); - - // tcUrl, stream - if (srs_string_contains(uri, "/")) { - req->stream = srs_path_basename(uri); - req->tcUrl = uri = srs_path_dirname(uri); - } - - srs_discovery_tc_url(req->tcUrl, - req->schema, req->host, req->vhost, req->app, req->port, - req->param); - } + std::string url = out_rtmp->get_url(); + srs_trace("connect output=%s", url.c_str()); // connect host. - if ((ret = srs_socket_connect(req->host, req->port, ST_UTIME_NO_TIMEOUT, &stfd)) != ERROR_SUCCESS) { - srs_error("mpegts: connect server %s:%d failed. ret=%d", req->host.c_str(), req->port, ret); - return ret; - } - io = new SrsStSocket(stfd); - client = new SrsRtmpClient(io); - - client->set_recv_timeout(SRS_CONSTS_RTMP_RECV_TIMEOUT_US); - client->set_send_timeout(SRS_CONSTS_RTMP_SEND_TIMEOUT_US); - - // connect to vhost/app - if ((ret = client->handshake()) != ERROR_SUCCESS) { - srs_error("mpegts: handshake with server failed. ret=%d", ret); - return ret; - } - if ((ret = connect_app(req->host, req->port)) != ERROR_SUCCESS) { - srs_error("mpegts: connect with server failed. ret=%d", ret); - return ret; - } - if ((ret = client->create_stream(stream_id)) != ERROR_SUCCESS) { - srs_error("mpegts: connect with server failed, stream_id=%d. ret=%d", stream_id, ret); + int64_t cto = SRS_CONSTS_RTMP_TIMEOUT_US; + int64_t sto = SRS_CONSTS_RTMP_PULSE_TIMEOUT_US; + if ((ret = sdk->connect(url, cto, sto)) != ERROR_SUCCESS) { + srs_error("mpegts: connect %s failed, cto=%"PRId64", sto=%"PRId64". ret=%d", url.c_str(), cto, sto, ret); return ret; } // publish. - if ((ret = client->publish(req->stream, stream_id)) != ERROR_SUCCESS) { - srs_error("mpegts: publish failed, stream=%s, stream_id=%d. ret=%d", - req->stream.c_str(), stream_id, ret); - return ret; - } - - return ret; -} - -// TODO: FIXME: refine the connect_app. -int SrsIngestSrsOutput::connect_app(string ep_server, int ep_port) -{ - int ret = ERROR_SUCCESS; - - // args of request takes the srs info. - if (req->args == NULL) { - req->args = SrsAmf0Any::object(); - } - - // notify server the edge identity, - // @see https://github.com/simple-rtmp-server/srs/issues/147 - SrsAmf0Object* data = req->args; - data->set("srs_sig", SrsAmf0Any::str(RTMP_SIG_SRS_KEY)); - data->set("srs_server", SrsAmf0Any::str(RTMP_SIG_SRS_KEY" "RTMP_SIG_SRS_VERSION" ("RTMP_SIG_SRS_URL_SHORT")")); - data->set("srs_license", SrsAmf0Any::str(RTMP_SIG_SRS_LICENSE)); - data->set("srs_role", SrsAmf0Any::str(RTMP_SIG_SRS_ROLE)); - data->set("srs_url", SrsAmf0Any::str(RTMP_SIG_SRS_URL)); - data->set("srs_version", SrsAmf0Any::str(RTMP_SIG_SRS_VERSION)); - data->set("srs_site", SrsAmf0Any::str(RTMP_SIG_SRS_WEB)); - data->set("srs_email", SrsAmf0Any::str(RTMP_SIG_SRS_EMAIL)); - data->set("srs_copyright", SrsAmf0Any::str(RTMP_SIG_SRS_COPYRIGHT)); - data->set("srs_primary", SrsAmf0Any::str(RTMP_SIG_SRS_PRIMARY)); - data->set("srs_authors", SrsAmf0Any::str(RTMP_SIG_SRS_AUTHROS)); - // for edge to directly get the id of client. - data->set("srs_pid", SrsAmf0Any::number(getpid())); - data->set("srs_id", SrsAmf0Any::number(_srs_context->get_id())); - - // local ip of edge - std::vector ips = srs_get_local_ipv4_ips(); - assert(0 < (int)ips.size()); - std::string local_ip = ips[0]; - data->set("srs_server_ip", SrsAmf0Any::str(local_ip.c_str())); - - // generate the tcUrl - std::string param = ""; - std::string tc_url = srs_generate_tc_url(ep_server, req->vhost, req->app, ep_port, param); - - // upnode server identity will show in the connect_app of client. - // @see https://github.com/simple-rtmp-server/srs/issues/160 - // the debug_srs_upnode is config in vhost and default to true. - bool debug_srs_upnode = true; - if ((ret = client->connect_app(req->app, tc_url, req, debug_srs_upnode)) != ERROR_SUCCESS) { - srs_error("mpegts: connect with server failed, tcUrl=%s, dsu=%d. ret=%d", - tc_url.c_str(), debug_srs_upnode, ret); + if ((ret = sdk->publish()) != ERROR_SUCCESS) { + srs_error("mpegts: publish %s failed. ret=%d", url.c_str(), ret); return ret; } @@ -1322,13 +1232,11 @@ int SrsIngestSrsOutput::connect_app(string ep_server, int ep_port) void SrsIngestSrsOutput::close() { - srs_trace("close output=%s", out_rtmp->get_url()); + srs_trace("close output=%s", out_rtmp->get_url().c_str()); h264_sps_pps_sent = false; - srs_freep(client); - srs_freep(io); srs_freep(req); - srs_close_stfd(stfd); + sdk->close(); } // the context for ingest hls stream. diff --git a/trunk/src/protocol/srs_http_stack.cpp b/trunk/src/protocol/srs_http_stack.cpp index b19c8519b..3c88fc44c 100755 --- a/trunk/src/protocol/srs_http_stack.cpp +++ b/trunk/src/protocol/srs_http_stack.cpp @@ -377,11 +377,7 @@ int SrsHttpFileServer::serve_file(ISrsHttpResponseWriter* w, ISrsHttpMessage* r, } if (true) { - size_t pos; - std::string ext = fullpath; - if ((pos = ext.rfind(".")) != string::npos) { - ext = ext.substr(pos); - } + std::string ext = srs_path_filext(fullpath); if (_mime.find(ext) == _mime.end()) { w->header()->set_content_type("application/octet-stream"); diff --git a/trunk/src/protocol/srs_kafka_stack.cpp b/trunk/src/protocol/srs_kafka_stack.cpp index f23d0d00a..be5dc365e 100644 --- a/trunk/src/protocol/srs_kafka_stack.cpp +++ b/trunk/src/protocol/srs_kafka_stack.cpp @@ -23,16 +23,36 @@ #include +#include using namespace std; +#include +#include +#include +#include +#include +#include +#include +#include + #ifdef SRS_AUTO_KAFKA +#define SRS_KAFKA_PRODUCER_MESSAGE_TIMEOUT_MS 300000 + SrsKafkaString::SrsKafkaString() { - size = -1; + _size = -1; data = NULL; } +SrsKafkaString::SrsKafkaString(string v) +{ + _size = -1; + data = NULL; + + set_value(v); +} + SrsKafkaString::~SrsKafkaString() { srs_freep(data); @@ -40,50 +60,238 @@ SrsKafkaString::~SrsKafkaString() bool SrsKafkaString::null() { - return size == -1; + return _size == -1; } bool SrsKafkaString::empty() { - return size <= 0; + return _size <= 0; } -int SrsKafkaString::total_size() +string SrsKafkaString::to_str() { - return 2 + (size == -1? 0 : size); + string ret; + if (_size > 0) { + ret.append(data, _size); + } + return ret; +} + +void SrsKafkaString::set_value(string v) +{ + // free previous data. + srs_freep(data); + + // copy new value to data. + _size = (int16_t)v.length(); + + srs_assert(_size > 0); + data = new char[_size]; + memcpy(data, v.data(), _size); +} + +int SrsKafkaString::nb_bytes() +{ + return _size == -1? 2 : 2 + _size; +} + +int SrsKafkaString::encode(SrsBuffer* buf) +{ + int ret = ERROR_SUCCESS; + + if (!buf->require(2)) { + ret = ERROR_KAFKA_CODEC_STRING; + srs_error("kafka encode string failed. ret=%d", ret); + return ret; + } + buf->write_2bytes(_size); + + if (_size <= 0) { + return ret; + } + + if (!buf->require(_size)) { + ret = ERROR_KAFKA_CODEC_STRING; + srs_error("kafka encode string data failed. ret=%d", ret); + return ret; + } + buf->write_bytes(data, _size); + + return ret; +} + +int SrsKafkaString::decode(SrsBuffer* buf) +{ + int ret = ERROR_SUCCESS; + + if (!buf->require(2)) { + ret = ERROR_KAFKA_CODEC_STRING; + srs_error("kafka decode string failed. ret=%d", ret); + return ret; + } + _size = buf->read_2bytes(); + + if (_size != -1 && _size < 0) { + ret = ERROR_KAFKA_CODEC_STRING; + srs_error("kafka string must be -1 or >=0, actual is %d. ret=%d", _size, ret); + return ret; + } + + if (_size <= 0) { + return ret; + } + + if (!buf->require(_size)) { + ret = ERROR_KAFKA_CODEC_STRING; + srs_error("kafka decode string data failed. ret=%d", ret); + return ret; + } + + srs_freep(data); + data = new char[_size]; + + buf->read_bytes(data, _size); + + return ret; } SrsKafkaBytes::SrsKafkaBytes() { - size = -1; - data = NULL; + _size = -1; + _data = NULL; +} + +SrsKafkaBytes::SrsKafkaBytes(const char* v, int nb_v) +{ + _size = -1; + _data = NULL; + + set_value(v, nb_v); } SrsKafkaBytes::~SrsKafkaBytes() { - srs_freep(data); + srs_freep(_data); +} + +char* SrsKafkaBytes::data() +{ + return _data; +} + +int SrsKafkaBytes::size() +{ + return _size; } bool SrsKafkaBytes::null() { - return size == -1; + return _size == -1; } bool SrsKafkaBytes::empty() { - return size <= 0; + return _size <= 0; } -int SrsKafkaBytes::total_size() +void SrsKafkaBytes::set_value(string v) { - return 4 + (size == -1? 0 : size); + set_value(v.data(), (int)v.length()); +} + +void SrsKafkaBytes::set_value(const char* v, int nb_v) +{ + // free previous data. + srs_freep(_data); + + // copy new value to data. + _size = (int16_t)nb_v; + + srs_assert(_size > 0); + _data = new char[_size]; + memcpy(_data, v, _size); +} + +u_int32_t SrsKafkaBytes::crc32(u_int32_t previous) +{ + char bsize[4]; + SrsBuffer(bsize, 4).write_4bytes(_size); + + if (_size <= 0) { + return srs_crc32_ieee(bsize, 4, previous); + } + + u_int32_t crc = srs_crc32_ieee(bsize, 4, previous); + crc = srs_crc32_ieee(_data, _size, crc); + + return crc; +} + +int SrsKafkaBytes::nb_bytes() +{ + return 4 + (_size == -1? 0 : _size); +} + +int SrsKafkaBytes::encode(SrsBuffer* buf) +{ + int ret = ERROR_SUCCESS; + + if (!buf->require(4)) { + ret = ERROR_KAFKA_CODEC_BYTES; + srs_error("kafka encode bytes failed. ret=%d", ret); + return ret; + } + buf->write_4bytes(_size); + + if (_size <= 0) { + return ret; + } + + if (!buf->require(_size)) { + ret = ERROR_KAFKA_CODEC_BYTES; + srs_error("kafka encode bytes data failed. ret=%d", ret); + return ret; + } + buf->write_bytes(_data, _size); + + return ret; +} + +int SrsKafkaBytes::decode(SrsBuffer* buf) +{ + int ret = ERROR_SUCCESS; + + if (!buf->require(4)) { + ret = ERROR_KAFKA_CODEC_BYTES; + srs_error("kafka decode bytes failed. ret=%d", ret); + return ret; + } + _size = buf->read_4bytes(); + + if (_size != -1 && _size < 0) { + ret = ERROR_KAFKA_CODEC_BYTES; + srs_error("kafka bytes must be -1 or >=0, actual is %d. ret=%d", _size, ret); + return ret; + } + + if (!buf->require(_size)) { + ret = ERROR_KAFKA_CODEC_BYTES; + srs_error("kafka decode bytes data failed. ret=%d", ret); + return ret; + } + + srs_freep(_data); + _data = new char[_size]; + buf->read_bytes(_data, _size); + + return ret; } SrsKafkaRequestHeader::SrsKafkaRequestHeader() { - size = 0; - api_key = api_version = 0; - correlation_id = 0; + _size = 0; + _api_key = api_version = 0; + _correlation_id = 0; client_id = new SrsKafkaString(); } @@ -94,69 +302,223 @@ SrsKafkaRequestHeader::~SrsKafkaRequestHeader() int SrsKafkaRequestHeader::header_size() { - return 2 + 2 + 4 + client_id->total_size(); + return 2 + 2 + 4 + client_id->nb_bytes(); } int SrsKafkaRequestHeader::message_size() { - return size - header_size(); + return _size - header_size(); } int SrsKafkaRequestHeader::total_size() { - return 4 + size; + return 4 + _size; } -bool SrsKafkaRequestHeader::is_producer_request() +void SrsKafkaRequestHeader::set_total_size(int s) { - return api_key == SrsKafkaApiKeyProduceRequest; + _size = s - 4; } -bool SrsKafkaRequestHeader::is_fetch_request() +int32_t SrsKafkaRequestHeader::correlation_id() { - return api_key == SrsKafkaApiKeyFetchRequest; + return _correlation_id; } -bool SrsKafkaRequestHeader::is_offset_request() +void SrsKafkaRequestHeader::set_correlation_id(int32_t cid) { - return api_key == SrsKafkaApiKeyOffsetRequest; + _correlation_id = cid; } -bool SrsKafkaRequestHeader::is_metadata_request() +SrsKafkaApiKey SrsKafkaRequestHeader::api_key() { - return api_key == SrsKafkaApiKeyMetadataRequest; -} - -bool SrsKafkaRequestHeader::is_offset_commit_request() -{ - return api_key == SrsKafkaApiKeyOffsetCommitRequest; -} - -bool SrsKafkaRequestHeader::is_offset_fetch_request() -{ - return api_key == SrsKafkaApiKeyOffsetFetchRequest; -} - -bool SrsKafkaRequestHeader::is_consumer_metadata_request() -{ - return api_key == SrsKafkaApiKeyConsumerMetadataRequest; + return (SrsKafkaApiKey)_api_key; } void SrsKafkaRequestHeader::set_api_key(SrsKafkaApiKey key) { - api_key = (int16_t)key; + _api_key = (int16_t)key; } -SrsKafkaResponse::SrsKafkaResponse() +bool SrsKafkaRequestHeader::is_producer_request() { - correlation_id = 0; + return _api_key == SrsKafkaApiKeyProduceRequest; } -SrsKafkaResponse::~SrsKafkaResponse() +bool SrsKafkaRequestHeader::is_fetch_request() +{ + return _api_key == SrsKafkaApiKeyFetchRequest; +} + +bool SrsKafkaRequestHeader::is_offset_request() +{ + return _api_key == SrsKafkaApiKeyOffsetRequest; +} + +bool SrsKafkaRequestHeader::is_metadata_request() +{ + return _api_key == SrsKafkaApiKeyMetadataRequest; +} + +bool SrsKafkaRequestHeader::is_offset_commit_request() +{ + return _api_key == SrsKafkaApiKeyOffsetCommitRequest; +} + +bool SrsKafkaRequestHeader::is_offset_fetch_request() +{ + return _api_key == SrsKafkaApiKeyOffsetFetchRequest; +} + +bool SrsKafkaRequestHeader::is_consumer_metadata_request() +{ + return _api_key == SrsKafkaApiKeyConsumerMetadataRequest; +} + +int SrsKafkaRequestHeader::nb_bytes() +{ + return 4 + header_size(); +} + +int SrsKafkaRequestHeader::encode(SrsBuffer* buf) +{ + int ret = ERROR_SUCCESS; + + if (!buf->require(4 + _size)) { + ret = ERROR_KAFKA_CODEC_REQUEST; + srs_error("kafka encode request failed. ret=%d", ret); + return ret; + } + + buf->write_4bytes(_size); + buf->write_2bytes(_api_key); + buf->write_2bytes(api_version); + buf->write_4bytes(_correlation_id); + + if ((ret = client_id->encode(buf)) != ERROR_SUCCESS) { + srs_error("kafka encode request client_id failed. ret=%d", ret); + return ret; + } + + return ret; +} + +int SrsKafkaRequestHeader::decode(SrsBuffer* buf) +{ + int ret = ERROR_SUCCESS; + + if (!buf->require(4)) { + ret = ERROR_KAFKA_CODEC_REQUEST; + srs_error("kafka decode request size failed. ret=%d", ret); + return ret; + } + _size = buf->read_4bytes(); + + if (_size <= 0) { + srs_warn("kafka got empty request"); + return ret; + } + + if (!buf->require(_size)) { + ret = ERROR_KAFKA_CODEC_REQUEST; + srs_error("kafka decode request message failed. ret=%d", ret); + return ret; + } + _api_key = buf->read_2bytes(); + api_version = buf->read_2bytes(); + _correlation_id = buf->read_4bytes(); + + if ((ret = client_id->decode(buf)) != ERROR_SUCCESS) { + srs_error("kafka decode request client_id failed. ret=%d", ret); + return ret; + } + + return ret; +} + +SrsKafkaResponseHeader::SrsKafkaResponseHeader() +{ + _size = 0; + _correlation_id = 0; +} + +SrsKafkaResponseHeader::~SrsKafkaResponseHeader() { } -SrsKafkaMessage::SrsKafkaMessage() +int SrsKafkaResponseHeader::header_size() +{ + return 4; +} + +int SrsKafkaResponseHeader::message_size() +{ + return _size - header_size(); +} + +int SrsKafkaResponseHeader::total_size() +{ + return 4 + _size; +} + +void SrsKafkaResponseHeader::set_total_size(int s) +{ + _size = s - 4; +} + +int32_t SrsKafkaResponseHeader::correlation_id() +{ + return _correlation_id; +} + +int SrsKafkaResponseHeader::nb_bytes() +{ + return 4 + header_size(); +} + +int SrsKafkaResponseHeader::encode(SrsBuffer* buf) +{ + int ret = ERROR_SUCCESS; + + if (!buf->require(4 + _size)) { + ret = ERROR_KAFKA_CODEC_RESPONSE; + srs_error("kafka encode response failed. ret=%d", ret); + return ret; + } + + buf->write_4bytes(_size); + buf->write_4bytes(_correlation_id); + + return ret; +} + +int SrsKafkaResponseHeader::decode(SrsBuffer* buf) +{ + int ret = ERROR_SUCCESS; + + if (!buf->require(4)) { + ret = ERROR_KAFKA_CODEC_RESPONSE; + srs_error("kafka decode response size failed. ret=%d", ret); + return ret; + } + _size = buf->read_4bytes(); + + if (_size <= 0) { + srs_warn("kafka got empty response"); + return ret; + } + + if (!buf->require(_size)) { + ret = ERROR_KAFKA_CODEC_RESPONSE; + srs_error("kafka decode response message failed. ret=%d", ret); + return ret; + } + _correlation_id = buf->read_4bytes(); + + return ret; +} + +SrsKafkaRawMessage::SrsKafkaRawMessage() { offset = 0; message_size = 0; @@ -167,26 +529,238 @@ SrsKafkaMessage::SrsKafkaMessage() value = new SrsKafkaBytes(); } -SrsKafkaMessage::~SrsKafkaMessage() +SrsKafkaRawMessage::~SrsKafkaRawMessage() { srs_freep(key); srs_freep(value); } -SrsKafkaMessageSet::SrsKafkaMessageSet() +int SrsKafkaRawMessage::create(SrsJsonObject* obj) +{ + int ret = ERROR_SUCCESS; + + // current must be 0. + magic_byte = 0; + + // no compression codec. + attributes = 0; + + // dumps the json to string. + value->set_value(obj->dumps()); + + // crc32 message. + crc = srs_crc32_ieee(&magic_byte, 1); + crc = srs_crc32_ieee(&attributes, 1, crc); + crc = key->crc32(crc); + crc = value->crc32(crc); + + srs_info("crc32 message is %#x", crc); + + message_size = raw_message_size(); + + return ret; +} + +int SrsKafkaRawMessage::raw_message_size() +{ + return 4 + 1 + 1 + key->nb_bytes() + value->nb_bytes(); +} + +int SrsKafkaRawMessage::nb_bytes() +{ + return 8 + 4 + 4 + 1 + 1 + key->nb_bytes() + value->nb_bytes(); +} + +int SrsKafkaRawMessage::encode(SrsBuffer* buf) +{ + int ret = ERROR_SUCCESS; + + if (!buf->require(8 + 4 + 4 + 1 + 1)) { + ret = ERROR_KAFKA_CODEC_MESSAGE; + srs_error("kafka encode message failed. ret=%d", ret); + return ret; + } + buf->write_8bytes(offset); + buf->write_4bytes(message_size); + buf->write_4bytes(crc); + buf->write_1bytes(magic_byte); + buf->write_1bytes(attributes); + + if ((ret = key->encode(buf)) != ERROR_SUCCESS) { + srs_error("kafka encode message key failed. ret=%d", ret); + return ret; + } + + if ((ret = value->encode(buf)) != ERROR_SUCCESS) { + srs_error("kafka encode message value failed. ret=%d", ret); + return ret; + } + + return ret; +} + +int SrsKafkaRawMessage::decode(SrsBuffer* buf) +{ + int ret = ERROR_SUCCESS; + + if (!buf->require(8 + 4 + 4 + 1 + 1)) { + ret = ERROR_KAFKA_CODEC_MESSAGE; + srs_error("kafka decode message failed. ret=%d", ret); + return ret; + } + offset = buf->read_8bytes(); + message_size = buf->read_4bytes(); + crc = buf->read_4bytes(); + magic_byte = buf->read_1bytes(); + attributes = buf->read_1bytes(); + + if ((ret = key->decode(buf)) != ERROR_SUCCESS) { + srs_error("kafka decode message key failed. ret=%d", ret); + return ret; + } + + if ((ret = value->decode(buf)) != ERROR_SUCCESS) { + srs_error("kafka decode message value failed. ret=%d", ret); + return ret; + } + + return ret; +} + +SrsKafkaRawMessageSet::SrsKafkaRawMessageSet() { } -SrsKafkaMessageSet::~SrsKafkaMessageSet() +SrsKafkaRawMessageSet::~SrsKafkaRawMessageSet() { - vector::iterator it; + vector::iterator it; for (it = messages.begin(); it != messages.end(); ++it) { - SrsKafkaMessage* message = *it; + SrsKafkaRawMessage* message = *it; srs_freep(message); } messages.clear(); } +void SrsKafkaRawMessageSet::append(SrsKafkaRawMessage* msg) +{ + messages.push_back(msg); +} + +int SrsKafkaRawMessageSet::nb_bytes() +{ + int s = 0; + + vector::iterator it; + for (it = messages.begin(); it != messages.end(); ++it) { + SrsKafkaRawMessage* message = *it; + s += message->nb_bytes(); + } + + return s; +} + +int SrsKafkaRawMessageSet::encode(SrsBuffer* buf) +{ + int ret = ERROR_SUCCESS; + + vector::iterator it; + for (it = messages.begin(); it != messages.end(); ++it) { + SrsKafkaRawMessage* message = *it; + if ((ret = message->encode(buf)) != ERROR_SUCCESS) { + srs_error("kafka encode message set failed. ret=%d", ret); + return ret; + } + } + + return ret; +} + +int SrsKafkaRawMessageSet::decode(SrsBuffer* buf) +{ + int ret = ERROR_SUCCESS; + + while (!buf->empty()) { + SrsKafkaRawMessage* message = new SrsKafkaRawMessage(); + + if ((ret = message->decode(buf)) != ERROR_SUCCESS) { + srs_freep(message); + srs_error("kafka decode message set elem failed. ret=%d", ret); + return ret; + } + + messages.push_back(message); + } + + return ret; +} + +SrsKafkaRequest::SrsKafkaRequest() +{ + header.set_correlation_id(SrsKafkaCorrelationPool::instance()->generate_correlation_id()); +} + +SrsKafkaRequest::~SrsKafkaRequest() +{ +} + +void SrsKafkaRequest::update_header(int s) +{ + header.set_total_size(s); +} + +int32_t SrsKafkaRequest::correlation_id() +{ + return header.correlation_id(); +} + +SrsKafkaApiKey SrsKafkaRequest::api_key() +{ + return header.api_key(); +} + +int SrsKafkaRequest::nb_bytes() +{ + return header.nb_bytes(); +} + +int SrsKafkaRequest::encode(SrsBuffer* buf) +{ + return header.encode(buf); +} + +int SrsKafkaRequest::decode(SrsBuffer* buf) +{ + return header.decode(buf); +} + +SrsKafkaResponse::SrsKafkaResponse() +{ +} + +SrsKafkaResponse::~SrsKafkaResponse() +{ +} + +void SrsKafkaResponse::update_header(int s) +{ + header.set_total_size(s); +} + +int SrsKafkaResponse::nb_bytes() +{ + return header.nb_bytes(); +} + +int SrsKafkaResponse::encode(SrsBuffer* buf) +{ + return header.encode(buf); +} + +int SrsKafkaResponse::decode(SrsBuffer* buf) +{ + return header.decode(buf); +} + SrsKafkaTopicMetadataRequest::SrsKafkaTopicMetadataRequest() { header.set_api_key(SrsKafkaApiKeyMetadataRequest); @@ -196,5 +770,751 @@ SrsKafkaTopicMetadataRequest::~SrsKafkaTopicMetadataRequest() { } +void SrsKafkaTopicMetadataRequest::add_topic(string topic) +{ + topics.append(new SrsKafkaString(topic)); +} + +int SrsKafkaTopicMetadataRequest::nb_bytes() +{ + return SrsKafkaRequest::nb_bytes() + topics.nb_bytes(); +} + +int SrsKafkaTopicMetadataRequest::encode(SrsBuffer* buf) +{ + int ret = ERROR_SUCCESS; + + if ((ret = SrsKafkaRequest::encode(buf)) != ERROR_SUCCESS) { + srs_error("kafka encode metadata request failed. ret=%d", ret); + return ret; + } + + if ((ret = topics.encode(buf)) != ERROR_SUCCESS) { + srs_error("kafka encode metadata topics failed. ret=%d", ret); + return ret; + } + + return ret; +} + +int SrsKafkaTopicMetadataRequest::decode(SrsBuffer* buf) +{ + int ret = ERROR_SUCCESS; + + if ((ret = SrsKafkaRequest::decode(buf)) != ERROR_SUCCESS) { + srs_error("kafka decode metadata request failed. ret=%d", ret); + return ret; + } + + if ((ret = topics.decode(buf)) != ERROR_SUCCESS) { + srs_error("kafka decode metadata topics failed. ret=%d", ret); + return ret; + } + + return ret; +} + +SrsKafkaBroker::SrsKafkaBroker() +{ + node_id = port = 0; +} + +SrsKafkaBroker::~SrsKafkaBroker() +{ +} + +int SrsKafkaBroker::nb_bytes() +{ + return 4 + host.nb_bytes() + 4; +} + +int SrsKafkaBroker::encode(SrsBuffer* buf) +{ + int ret = ERROR_SUCCESS; + + if (!buf->require(4)) { + ret = ERROR_KAFKA_CODEC_METADATA; + srs_error("kafka encode broker node_id failed. ret=%d", ret); + return ret; + } + buf->write_4bytes(node_id); + + if ((ret = host.encode(buf)) != ERROR_SUCCESS) { + srs_error("kafka encode broker host failed. ret=%d", ret); + return ret; + } + + if (!buf->require(4)) { + ret = ERROR_KAFKA_CODEC_METADATA; + srs_error("kafka encode broker port failed. ret=%d", ret); + return ret; + } + buf->write_4bytes(port); + + return ret; +} + +int SrsKafkaBroker::decode(SrsBuffer* buf) +{ + int ret = ERROR_SUCCESS; + + if (!buf->require(4)) { + ret = ERROR_KAFKA_CODEC_METADATA; + srs_error("kafka decode broker node_id failed. ret=%d", ret); + return ret; + } + node_id = buf->read_4bytes(); + + if ((ret = host.decode(buf)) != ERROR_SUCCESS) { + srs_error("kafka decode broker host failed. ret=%d", ret); + return ret; + } + + if (!buf->require(4)) { + ret = ERROR_KAFKA_CODEC_METADATA; + srs_error("kafka decode broker port failed. ret=%d", ret); + return ret; + } + port = buf->read_4bytes(); + + return ret; +} + +SrsKafkaPartitionMetadata::SrsKafkaPartitionMetadata() +{ + error_code = 0; + partition_id = 0; + leader = 0; +} + +SrsKafkaPartitionMetadata::~SrsKafkaPartitionMetadata() +{ +} + +int SrsKafkaPartitionMetadata::nb_bytes() +{ + return 2 + 4 + 4 + replicas.nb_bytes() + isr.nb_bytes(); +} + +int SrsKafkaPartitionMetadata::encode(SrsBuffer* buf) +{ + int ret = ERROR_SUCCESS; + + if (!buf->require(2 + 4 + 4)) { + ret = ERROR_KAFKA_CODEC_METADATA; + srs_error("kafka encode partition metadata failed. ret=%d", ret); + return ret; + } + buf->write_2bytes(error_code); + buf->write_4bytes(partition_id); + buf->write_4bytes(leader); + + if ((ret = replicas.encode(buf)) != ERROR_SUCCESS) { + srs_error("kafka encode partition metadata replicas failed. ret=%d", ret); + return ret; + } + if ((ret = isr.encode(buf)) != ERROR_SUCCESS) { + srs_error("kafka encode partition metadata isr failed. ret=%d", ret); + return ret; + } + + return ret; +} + +int SrsKafkaPartitionMetadata::decode(SrsBuffer* buf) +{ + int ret = ERROR_SUCCESS; + + if (!buf->require(2 + 4 + 4)) { + ret = ERROR_KAFKA_CODEC_METADATA; + srs_error("kafka decode partition metadata failed. ret=%d", ret); + return ret; + } + error_code = buf->read_2bytes(); + partition_id = buf->read_4bytes(); + leader = buf->read_4bytes(); + + if ((ret = replicas.decode(buf)) != ERROR_SUCCESS) { + srs_error("kafka decode partition metadata replicas failed. ret=%d", ret); + return ret; + } + if ((ret = isr.decode(buf)) != ERROR_SUCCESS) { + srs_error("kafka decode partition metadata isr failed. ret=%d", ret); + return ret; + } + + return ret; +} + +SrsKafkaTopicMetadata::SrsKafkaTopicMetadata() +{ + error_code = 0; +} + +SrsKafkaTopicMetadata::~SrsKafkaTopicMetadata() +{ +} + +int SrsKafkaTopicMetadata::nb_bytes() +{ + return 2 + name.nb_bytes() + metadatas.nb_bytes(); +} + +int SrsKafkaTopicMetadata::encode(SrsBuffer* buf) +{ + int ret = ERROR_SUCCESS; + + if (!buf->require(2)) { + ret = ERROR_KAFKA_CODEC_METADATA; + srs_error("kafka encode topic metadata failed. ret=%d", ret); + return ret; + } + buf->write_2bytes(error_code); + + if ((ret = name.encode(buf)) != ERROR_SUCCESS) { + srs_error("kafka encode topic name failed. ret=%d", ret); + return ret; + } + + if ((ret = metadatas.encode(buf)) != ERROR_SUCCESS) { + srs_error("kafka encode topic metadatas failed. ret=%d", ret); + return ret; + } + + return ret; +} + +int SrsKafkaTopicMetadata::decode(SrsBuffer* buf) +{ + int ret = ERROR_SUCCESS; + + if (!buf->require(2)) { + ret = ERROR_KAFKA_CODEC_METADATA; + srs_error("kafka decode topic metadata failed. ret=%d", ret); + return ret; + } + error_code = buf->read_2bytes(); + + if ((ret = name.decode(buf)) != ERROR_SUCCESS) { + srs_error("kafka decode topic name failed. ret=%d", ret); + return ret; + } + + if ((ret = metadatas.decode(buf)) != ERROR_SUCCESS) { + srs_error("kafka decode topic metadatas failed. ret=%d", ret); + return ret; + } + + return ret; +} + +SrsKafkaTopicMetadataResponse::SrsKafkaTopicMetadataResponse() +{ +} + +SrsKafkaTopicMetadataResponse::~SrsKafkaTopicMetadataResponse() +{ +} + +int SrsKafkaTopicMetadataResponse::nb_bytes() +{ + return SrsKafkaResponse::nb_bytes() + brokers.nb_bytes() + metadatas.nb_bytes(); +} + +int SrsKafkaTopicMetadataResponse::encode(SrsBuffer* buf) +{ + int ret = ERROR_SUCCESS; + + if ((ret = SrsKafkaResponse::encode(buf)) != ERROR_SUCCESS) { + srs_error("kafka encode metadata response failed. ret=%d", ret); + return ret; + } + + if ((ret = brokers.encode(buf)) != ERROR_SUCCESS) { + srs_error("kafka encode metadata brokers failed. ret=%d", ret); + return ret; + } + + if ((ret = metadatas.encode(buf)) != ERROR_SUCCESS) { + srs_error("kafka encode metadatas failed. ret=%d", ret); + return ret; + } + + return ret; +} + +int SrsKafkaTopicMetadataResponse::decode(SrsBuffer* buf) +{ + int ret = ERROR_SUCCESS; + + if ((ret = SrsKafkaResponse::decode(buf)) != ERROR_SUCCESS) { + srs_error("kafka decode metadata response failed. ret=%d", ret); + return ret; + } + + if ((ret = brokers.decode(buf)) != ERROR_SUCCESS) { + srs_error("kafka decode metadata brokers failed. ret=%d", ret); + return ret; + } + + if ((ret = metadatas.decode(buf)) != ERROR_SUCCESS) { + srs_error("kafka decode metadatas failed. ret=%d", ret); + return ret; + } + + return ret; +} + +int SrsKafkaProducerPartitionMessages::nb_bytes() +{ + return 4 + 4 + messages.nb_bytes(); +} + +int SrsKafkaProducerPartitionMessages::encode(SrsBuffer* buf) +{ + int ret = ERROR_SUCCESS; + + if (!buf->require(4 + 4)) { + ret = ERROR_KAFKA_CODEC_PRODUCER; + srs_error("kafka encode producer failed. ret=%d", ret); + return ret; + } + buf->write_4bytes(partition); + buf->write_4bytes(message_set_size); + + if ((ret = messages.encode(buf)) != ERROR_SUCCESS) { + srs_error("kafka encode producer messages failed. ret=%d", ret); + return ret; + } + + return ret; +} + +int SrsKafkaProducerPartitionMessages::decode(SrsBuffer* buf) +{ + int ret = ERROR_SUCCESS; + + if (!buf->require(4 + 4)) { + ret = ERROR_KAFKA_CODEC_PRODUCER; + srs_error("kafka decode producer failed. ret=%d", ret); + return ret; + } + partition = buf->read_4bytes(); + message_set_size = buf->read_4bytes(); + + // for the message set decode util empty, we must create a new buffer when + // there exists other objects after message set. + if (buf->size() - buf->pos() != message_set_size) { + SrsBuffer* tbuf = new SrsBuffer(); + SrsAutoFree(SrsBuffer, tbuf); + + if ((ret = tbuf->initialize(buf->data() + buf->pos(), message_set_size)) != ERROR_SUCCESS) { + return ret; + } + + if ((ret = messages.decode(tbuf)) != ERROR_SUCCESS) { + srs_error("kafka decode procuder messages failed. ret=%d", ret); + return ret; + } + } else { + if ((ret = messages.decode(buf)) != ERROR_SUCCESS) { + srs_error("kafka decode procuder messages failed. ret=%d", ret); + return ret; + } + } + + return ret; +} + +int SrsKafkaProducerTopicMessages::nb_bytes() +{ + return topic_name.nb_bytes() + partitions.nb_bytes(); +} + +int SrsKafkaProducerTopicMessages::encode(SrsBuffer* buf) +{ + int ret = ERROR_SUCCESS; + + if ((ret = topic_name.encode(buf)) != ERROR_SUCCESS) { + return ret; + } + + if ((ret = partitions.encode(buf)) != ERROR_SUCCESS) { + return ret; + } + + return ret; +} + +int SrsKafkaProducerTopicMessages::decode(SrsBuffer* buf) +{ + int ret = ERROR_SUCCESS; + + if ((ret = topic_name.decode(buf)) != ERROR_SUCCESS) { + return ret; + } + + if ((ret = partitions.decode(buf)) != ERROR_SUCCESS) { + return ret; + } + + return ret; +} + +SrsKafkaProducerRequest::SrsKafkaProducerRequest() +{ + required_acks = 0; + timeout = 0; +} + +SrsKafkaProducerRequest::~SrsKafkaProducerRequest() +{ +} + +int SrsKafkaProducerRequest::nb_bytes() +{ + return SrsKafkaRequest::nb_bytes() + 2 + 4 + topics.nb_bytes(); +} + +int SrsKafkaProducerRequest::encode(SrsBuffer* buf) +{ + int ret = ERROR_SUCCESS; + + if ((ret = SrsKafkaRequest::encode(buf)) != ERROR_SUCCESS) { + return ret; + } + + if (!buf->require(2 + 4)) { + ret = ERROR_KAFKA_CODEC_PRODUCER; + srs_error("kafka encode producer failed. ret=%d", ret); + return ret; + } + buf->write_2bytes(required_acks); + buf->write_4bytes(timeout); + + if ((ret = topics.encode(buf)) != ERROR_SUCCESS) { + srs_error("kafka encode producer topics failed. ret=%d", ret); + } + + return ret; +} + +int SrsKafkaProducerRequest::decode(SrsBuffer* buf) +{ + int ret = ERROR_SUCCESS; + + if ((ret = SrsKafkaRequest::decode(buf)) != ERROR_SUCCESS) { + return ret; + } + + if (!buf->require(2 + 4)) { + ret = ERROR_KAFKA_CODEC_PRODUCER; + srs_error("kafka decode producer failed. ret=%d", ret); + return ret; + } + required_acks = buf->read_2bytes(); + timeout = buf->read_4bytes(); + + if ((ret = topics.decode(buf)) != ERROR_SUCCESS) { + srs_error("kafka decode producer topics failed. ret=%d", ret); + } + + return ret; +} + +SrsKafkaCorrelationPool* SrsKafkaCorrelationPool::_instance = new SrsKafkaCorrelationPool(); + +SrsKafkaCorrelationPool* SrsKafkaCorrelationPool::instance() +{ + return _instance; +} + +SrsKafkaCorrelationPool::SrsKafkaCorrelationPool() +{ +} + +SrsKafkaCorrelationPool::~SrsKafkaCorrelationPool() +{ + correlation_ids.clear(); +} + +int32_t SrsKafkaCorrelationPool::generate_correlation_id() +{ + static int32_t cid = 1; + return cid++; +} + +SrsKafkaApiKey SrsKafkaCorrelationPool::set(int32_t correlation_id, SrsKafkaApiKey request) +{ + SrsKafkaApiKey previous = SrsKafkaApiKeyUnknown; + + std::map::iterator it = correlation_ids.find(correlation_id); + if (it != correlation_ids.end()) { + previous = it->second; + } + + correlation_ids[correlation_id] = request; + + return previous; +} + +SrsKafkaApiKey SrsKafkaCorrelationPool::unset(int32_t correlation_id) +{ + std::map::iterator it = correlation_ids.find(correlation_id); + + if (it != correlation_ids.end()) { + SrsKafkaApiKey key = it->second; + correlation_ids.erase(it); + return key; + } + + return SrsKafkaApiKeyUnknown; +} + +SrsKafkaApiKey SrsKafkaCorrelationPool::get(int32_t correlation_id) +{ + if (correlation_ids.find(correlation_id) == correlation_ids.end()) { + return SrsKafkaApiKeyUnknown; + } + + return correlation_ids[correlation_id]; +} + +SrsKafkaProtocol::SrsKafkaProtocol(ISrsProtocolReaderWriter* io) +{ + skt = io; + reader = new SrsFastStream(); +} + +SrsKafkaProtocol::~SrsKafkaProtocol() +{ + srs_freep(reader); +} + +int SrsKafkaProtocol::send_and_free_message(SrsKafkaRequest* msg) +{ + int ret = ERROR_SUCCESS; + + // TODO: FIXME: refine for performance issue. + SrsAutoFree(SrsKafkaRequest, msg); + + int size = msg->nb_bytes(); + if (size <= 0) { + return ret; + } + + // update the header of message. + msg->update_header(size); + + // cache the request correlation id to discovery response message. + SrsKafkaCorrelationPool* pool = SrsKafkaCorrelationPool::instance(); + pool->set(msg->correlation_id(), msg->api_key()); + + // TODO: FIXME: refine for performance issue. + char* bytes = new char[size]; + SrsAutoFree(char, bytes); + + // TODO: FIXME: refine for performance issue. + SrsBuffer* buf = new SrsBuffer(); + SrsAutoFree(SrsBuffer, buf); + + if ((ret = buf->initialize(bytes, size)) != ERROR_SUCCESS) { + srs_error("kafka create buffer failed. ret=%d", ret); + return ret; + } + + if ((ret = msg->encode(buf)) != ERROR_SUCCESS) { + srs_error("kafka encode message failed. ret=%d", ret); + return ret; + } + + if ((ret = skt->write(bytes, size, NULL)) != ERROR_SUCCESS) { + srs_error("kafka send message failed. ret=%d", ret); + return ret; + } + + return ret; +} + +int SrsKafkaProtocol::recv_message(SrsKafkaResponse** pmsg) +{ + *pmsg = NULL; + + int ret = ERROR_SUCCESS; + + while (true) { + SrsKafkaResponseHeader header; + + // ensure enough bytes for response header. + if ((ret = reader->grow(skt, header.nb_bytes())) != ERROR_SUCCESS) { + srs_error("kafka recv message failed. ret=%d", ret); + return ret; + } + + // decode response header. + SrsBuffer buffer; + if ((ret = buffer.initialize(reader->bytes(), reader->size())) != ERROR_SUCCESS) { + return ret; + } + + SrsBuffer* buf = &buffer; + if ((ret = header.decode(buf)) != ERROR_SUCCESS) { + srs_error("kafka decode response header failed. ret=%d", ret); + return ret; + } + + // skip the used buffer for header. + buf->skip(-1 * buf->pos()); + + // fetch cached api key. + SrsKafkaCorrelationPool* pool = SrsKafkaCorrelationPool::instance(); + SrsKafkaApiKey key = pool->unset(header.correlation_id()); + srs_info("kafka got %d bytes response, key=%d", header.total_size(), header.correlation_id()); + + // create message by cached api key. + SrsKafkaResponse* res = NULL; + switch (key) { + case SrsKafkaApiKeyMetadataRequest: + srs_info("kafka got metadata response"); + res = new SrsKafkaTopicMetadataResponse(); + break; + case SrsKafkaApiKeyUnknown: + default: + break; + } + + // ensure enough bytes to decode message. + if ((ret = reader->grow(skt, header.total_size())) != ERROR_SUCCESS) { + srs_freep(res); + srs_error("kafka recv message body failed. ret=%d", ret); + return ret; + } + + // dropped message, fetch next. + if (!res) { + reader->skip(header.total_size()); + srs_warn("kafka ignore unknown message, size=%d.", header.total_size()); + continue; + } + + // parse the whole message. + if ((ret = res->decode(buf)) != ERROR_SUCCESS) { + srs_freep(res); + srs_error("kafka decode message failed. ret=%d", ret); + return ret; + } + + *pmsg = res; + break; + } + + return ret; +} + +SrsKafkaClient::SrsKafkaClient(ISrsProtocolReaderWriter* io) +{ + protocol = new SrsKafkaProtocol(io); +} + +SrsKafkaClient::~SrsKafkaClient() +{ + srs_freep(protocol); +} + +int SrsKafkaClient::fetch_metadata(string topic, SrsKafkaTopicMetadataResponse** pmsg) +{ + *pmsg = NULL; + + int ret = ERROR_SUCCESS; + + SrsKafkaTopicMetadataRequest* req = new SrsKafkaTopicMetadataRequest(); + + req->add_topic(topic); + + if ((ret = protocol->send_and_free_message(req)) != ERROR_SUCCESS) { + srs_error("kafka send message failed. ret=%d", ret); + return ret; + } + + if ((ret = protocol->expect_message(pmsg)) != ERROR_SUCCESS) { + srs_error("kafka recv response failed. ret=%d", ret); + return ret; + } + + return ret; +} + +int SrsKafkaClient::write_messages(std::string topic, int32_t partition, vector& msgs) +{ + int ret = ERROR_SUCCESS; + + SrsKafkaProducerRequest* req = new SrsKafkaProducerRequest(); + + // 0 the server will not send any response. + req->required_acks = 0; + // timeout of producer message. + req->timeout = SRS_KAFKA_PRODUCER_MESSAGE_TIMEOUT_MS; + + // create the topic and partition to write message to. + SrsKafkaProducerTopicMessages* topics = new SrsKafkaProducerTopicMessages(); + SrsKafkaProducerPartitionMessages* partitions = new SrsKafkaProducerPartitionMessages(); + + topics->partitions.append(partitions); + req->topics.append(topics); + + topics->topic_name.set_value(topic); + partitions->partition = partition; + + // convert json objects to kafka raw messages. + vector::iterator it; + for (it = msgs.begin(); it != msgs.end(); ++it) { + SrsJsonObject* obj = *it; + SrsKafkaRawMessage* msg = new SrsKafkaRawMessage(); + + if ((ret = msg->create(obj)) != ERROR_SUCCESS) { + srs_freep(msg); + srs_freep(req); + srs_error("kafka write messages failed. ret=%d", ret); + return ret; + } + + partitions->messages.append(msg); + } + + partitions->message_set_size = partitions->messages.nb_bytes(); + + // write to kafka cluster. + if ((ret = protocol->send_and_free_message(req)) != ERROR_SUCCESS) { + srs_error("kafka write producer message failed. ret=%d", ret); + return ret; + } + + return ret; +} + +vector srs_kafka_array2vector(SrsKafkaArray* arr) +{ + vector strs; + + for (int i = 0; i < arr->size(); i++) { + SrsKafkaString* elem = arr->at(i); + strs.push_back(elem->to_str()); + } + + return strs; +} + +vector srs_kafka_array2vector(SrsKafkaArray* arr) +{ + vector strs; + + for (int i = 0; i < arr->size(); i++) { + int32_t elem = arr->at(i); + strs.push_back(srs_int2str(elem)); + } + + return strs; +} + #endif diff --git a/trunk/src/protocol/srs_kafka_stack.hpp b/trunk/src/protocol/srs_kafka_stack.hpp index e5df62b68..773ac0de0 100644 --- a/trunk/src/protocol/srs_kafka_stack.hpp +++ b/trunk/src/protocol/srs_kafka_stack.hpp @@ -30,12 +30,27 @@ #include #include +#include +#include + +#include +#include +#include + +class SrsFastStream; +class ISrsProtocolReaderWriter; +class SrsJsonObject; #ifdef SRS_AUTO_KAFKA -// https://cwiki.apache.org/confluence/display/KAFKA/A+Guide+To+The+Kafka+Protocol#AGuideToTheKafkaProtocol-ApiKeys +/** + * the api key used to identify the request type. + * @see https://cwiki.apache.org/confluence/display/KAFKA/A+Guide+To+The+Kafka+Protocol#AGuideToTheKafkaProtocol-ApiKeys + */ enum SrsKafkaApiKey { + SrsKafkaApiKeyUnknown = -1, + SrsKafkaApiKeyProduceRequest = 0, SrsKafkaApiKeyFetchRequest = 1, SrsKafkaApiKeyOffsetRequest = 2, @@ -51,18 +66,25 @@ enum SrsKafkaApiKey * A length of -1 indicates null. string uses an int16 for its size, and bytes uses an int32. * @see https://cwiki.apache.org/confluence/display/KAFKA/A+Guide+To+The+Kafka+Protocol#AGuideToTheKafkaProtocol-ProtocolPrimitiveTypes */ -class SrsKafkaString +class SrsKafkaString : public ISrsCodec { private: - int16_t size; + int16_t _size; char* data; public: SrsKafkaString(); + SrsKafkaString(std::string v); virtual ~SrsKafkaString(); public: virtual bool null(); virtual bool empty(); - virtual int total_size(); + virtual std::string to_str(); + virtual void set_value(std::string v); +// interface ISrsCodec +public: + virtual int nb_bytes(); + virtual int encode(SrsBuffer* buf); + virtual int decode(SrsBuffer* buf); }; /** @@ -70,18 +92,28 @@ public: * A length of -1 indicates null. string uses an int16 for its size, and bytes uses an int32. * @see https://cwiki.apache.org/confluence/display/KAFKA/A+Guide+To+The+Kafka+Protocol#AGuideToTheKafkaProtocol-ProtocolPrimitiveTypes */ -class SrsKafkaBytes +class SrsKafkaBytes : public ISrsCodec { private: - int32_t size; - char* data; + int32_t _size; + char* _data; public: SrsKafkaBytes(); + SrsKafkaBytes(const char* v, int nb_v); virtual ~SrsKafkaBytes(); public: + virtual char* data(); + virtual int size(); virtual bool null(); virtual bool empty(); - virtual int total_size(); + virtual void set_value(std::string v); + virtual void set_value(const char* v, int nb_v); + virtual u_int32_t crc32(u_int32_t previous); +// interface ISrsCodec +public: + virtual int nb_bytes(); + virtual int encode(SrsBuffer* buf); + virtual int decode(SrsBuffer* buf); }; /** @@ -93,14 +125,15 @@ public: * Usage: * SrsKafkaArray body; * body.append(new SrsKafkaBytes()); + * @remark array elem is the T*, which must be ISrsCodec* * * @see https://cwiki.apache.org/confluence/display/KAFKA/A+Guide+To+The+Kafka+Protocol#AGuideToTheKafkaProtocol-Requests */ template -class SrsKafkaArray +class SrsKafkaArray : public ISrsCodec { private: - int length; + int32_t length; std::vector elems; typedef typename std::vector::iterator SrsIterator; public: @@ -122,13 +155,167 @@ public: length++; elems.push_back(elem); } + virtual int size() + { + return length; + } + virtual bool empty() + { + return elems.empty(); + } + virtual T* at(int index) + { + return elems.at(index); + } +// interface ISrsCodec +public: + virtual int nb_bytes() + { + int s = 4; + + for (SrsIterator it = elems.begin(); it != elems.end(); ++it) { + T* elem = *it; + s += elem->nb_bytes(); + } + + return s; + } + virtual int encode(SrsBuffer* buf) + { + int ret = ERROR_SUCCESS; + + if (!buf->require(4)) { + ret = ERROR_KAFKA_CODEC_ARRAY; + srs_error("kafka encode array failed. ret=%d", ret); + return ret; + } + buf->write_4bytes(length); + + for (SrsIterator it = elems.begin(); it != elems.end(); ++it) { + T* elem = *it; + if ((ret = elem->encode(buf)) != ERROR_SUCCESS) { + srs_error("kafka encode array elem failed. ret=%d", ret); + return ret; + } + } + + return ret; + } + virtual int decode(SrsBuffer* buf) + { + int ret = ERROR_SUCCESS; + + if (!buf->require(4)) { + ret = ERROR_KAFKA_CODEC_ARRAY; + srs_error("kafka decode array failed. ret=%d", ret); + return ret; + } + length = buf->read_4bytes(); + + for (int i = 0; i < length; i++) { + T* elem = new T(); + if ((ret = elem->decode(buf)) != ERROR_SUCCESS) { + srs_error("kafka decode array elem failed. ret=%d", ret); + srs_freep(elem); + return ret; + } + + elems.push_back(elem); + } + + return ret; + } +}; +template<> +class SrsKafkaArray : public ISrsCodec +{ +private: + int32_t length; + std::vector elems; + typedef std::vector::iterator SrsIterator; +public: + SrsKafkaArray() + { + length = 0; + } + virtual ~SrsKafkaArray() + { + elems.clear(); + } +public: + virtual void append(int32_t elem) + { + length++; + elems.push_back(elem); + } + virtual int size() + { + return length; + } + virtual bool empty() + { + return elems.empty(); + } + virtual int32_t at(int index) + { + return elems.at(index); + } + // interface ISrsCodec +public: + virtual int nb_bytes() + { + return 4 + 4 * (int)elems.size(); + } + virtual int encode(SrsBuffer* buf) + { + int ret = ERROR_SUCCESS; + + if (!buf->require(4 + sizeof(int32_t) * (int)elems.size())) { + ret = ERROR_KAFKA_CODEC_ARRAY; + srs_error("kafka encode array failed. ret=%d", ret); + return ret; + } + buf->write_4bytes(length); + + for (SrsIterator it = elems.begin(); it != elems.end(); ++it) { + int32_t elem = *it; + buf->write_4bytes(elem); + } + + return ret; + } + virtual int decode(SrsBuffer* buf) + { + int ret = ERROR_SUCCESS; + + if (!buf->require(4)) { + ret = ERROR_KAFKA_CODEC_ARRAY; + srs_error("kafka decode array failed. ret=%d", ret); + return ret; + } + length = buf->read_4bytes(); + + for (int i = 0; i < length; i++) { + if (!buf->require(sizeof(int32_t))) { + ret = ERROR_KAFKA_CODEC_ARRAY; + srs_error("kafka decode array elem failed. ret=%d", ret); + return ret; + + } + + int32_t elem = buf->read_4bytes(); + elems.push_back(elem); + } + + return ret; + } }; /** * the header of request, includes the size of request. * @see https://cwiki.apache.org/confluence/display/KAFKA/A+Guide+To+The+Kafka+Protocol#AGuideToTheKafkaProtocol-Requests */ -class SrsKafkaRequestHeader +class SrsKafkaRequestHeader : public ISrsCodec { private: /** @@ -137,14 +324,14 @@ private: * size as an integer N, and then reading and parsing the subsequent N bytes * of the request. */ - int32_t size; + int32_t _size; private: /** * This is a numeric id for the API being invoked (i.e. is it * a metadata request, a produce request, a fetch request, etc). * @remark MetadataRequest | ProduceRequest | FetchRequest | OffsetRequest | OffsetCommitRequest | OffsetFetchRequest */ - int16_t api_key; + int16_t _api_key; /** * This is a numeric version number for this api. We version each API and * this version number allows the server to properly interpret the request @@ -157,7 +344,7 @@ private: * the response by the server, unmodified. It is useful for matching * request and response between the client and server. */ - int32_t correlation_id; + int32_t _correlation_id; /** * This is a user supplied identifier for the client application. * The user can use any identifier they like and it will be used @@ -171,22 +358,50 @@ private: public: SrsKafkaRequestHeader(); virtual ~SrsKafkaRequestHeader(); -public: +private: /** - * the size of header, exclude the 4bytes size. - * @remark total_size = 4 + header_size + message_size. + * the layout of request: + * +-----------+----------------------------------+ + * | 4B _size | [_size] bytes | + * +-----------+------------+---------------------+ + * | 4B _size | header | message | + * +-----------+------------+---------------------+ + * | total size = 4 + header + message | + * +----------------------------------------------+ + * where the header is specifies this request header without the start 4B size. + * @remark size = 4 + header + message. */ virtual int header_size(); /** - * the size of message, the left bytes left after the header. - * @remark total_size = 4 + header_size + message_size. + * the size of message, the bytes left after the header. */ virtual int message_size(); /** - * the total size of the request, 4bytes + size of header and message. - * @remark total_size = 4 + header_size + message_size. + * the total size of the request, includes the 4B size. */ virtual int total_size(); +public: + /** + * when got the whole message size, update the header. + * @param s the whole message, including the 4 bytes size size. + */ + virtual void set_total_size(int s); + /** + * get the correlation id for message. + */ + virtual int32_t correlation_id(); + /** + * set the correlation id for message. + */ + virtual void set_correlation_id(int32_t cid); + /** + * get the api key of header for message. + */ + virtual SrsKafkaApiKey api_key(); + /** + * set the api key of header for message. + */ + virtual void set_api_key(SrsKafkaApiKey key); public: /** * the api key enumeration. @@ -199,33 +414,84 @@ public: virtual bool is_offset_commit_request(); virtual bool is_offset_fetch_request(); virtual bool is_consumer_metadata_request(); - // set the api key. - virtual void set_api_key(SrsKafkaApiKey key); +// interface ISrsCodec +public: + virtual int nb_bytes(); + virtual int encode(SrsBuffer* buf); + virtual int decode(SrsBuffer* buf); }; /** - * the common kafka response. - * The response will always match the paired request (e.g. we will + * the header of response, include the size of response. + * The response will always match the paired request (e.g. we will * send a MetadataResponse in return to a MetadataRequest). * @see https://cwiki.apache.org/confluence/display/KAFKA/A+Guide+To+The+Kafka+Protocol#AGuideToTheKafkaProtocol-Responses */ -class SrsKafkaResponse +class SrsKafkaResponseHeader : public ISrsCodec { -protected: +private: /** - * The server passes back whatever integer the client supplied as the correlation in the request. + * The MessageSize field gives the size of the subsequent request or response + * message in bytes. The client can read requests by first reading this 4 byte + * size as an integer N, and then reading and parsing the subsequent N bytes + * of the request. */ - int32_t correlation_id; + int32_t _size; +private: + /** + * This is a user-supplied integer. It will be passed back in + * the response by the server, unmodified. It is useful for matching + * request and response between the client and server. + */ + int32_t _correlation_id; public: - SrsKafkaResponse(); - virtual ~SrsKafkaResponse(); + SrsKafkaResponseHeader(); + virtual ~SrsKafkaResponseHeader(); +private: + /** + * the layout of response: + * +-----------+----------------------------------+ + * | 4B _size | [_size] bytes | + * +-----------+------------+---------------------+ + * | 4B _size | 4B header | message | + * +-----------+------------+---------------------+ + * | total size = 4 + 4 + message | + * +----------------------------------------------+ + * where the header is specifies this request header without the start 4B size. + * @remark size = 4 + 4 + message. + */ + virtual int header_size(); + /** + * the size of message, the bytes left after the header. + */ + virtual int message_size(); +public: + /** + * the total size of the request, includes the 4B size and message body. + */ + virtual int total_size(); +public: + /** + * when got the whole message size, update the header. + * @param s the whole message, including the 4 bytes size size. + */ + virtual void set_total_size(int s); + /** + * get the correlation id of response message. + */ + virtual int32_t correlation_id(); +// interface ISrsCodec +public: + virtual int nb_bytes(); + virtual int encode(SrsBuffer* buf); + virtual int decode(SrsBuffer* buf); }; /** * the kafka message in message set. * @see https://cwiki.apache.org/confluence/display/KAFKA/A+Guide+To+The+Kafka+Protocol#AGuideToTheKafkaProtocol-Messagesets */ -struct SrsKafkaMessage +struct SrsKafkaRawMessage : public ISrsCodec { // metadata. public: @@ -269,21 +535,98 @@ public: */ SrsKafkaBytes* value; public: - SrsKafkaMessage(); - virtual ~SrsKafkaMessage(); + SrsKafkaRawMessage(); + virtual ~SrsKafkaRawMessage(); +public: + /** + * create message from json object. + */ + virtual int create(SrsJsonObject* obj); +private: + /** + * get the raw message, bytes after the message_size. + */ + virtual int raw_message_size(); +// interface ISrsCodec +public: + virtual int nb_bytes(); + virtual int encode(SrsBuffer* buf); + virtual int decode(SrsBuffer* buf); }; /** * a set of kafka message. * @see https://cwiki.apache.org/confluence/display/KAFKA/A+Guide+To+The+Kafka+Protocol#AGuideToTheKafkaProtocol-Messagesets + * @remark because the message set are not preceded by int32, so we decode the buffer util empty. */ -class SrsKafkaMessageSet +class SrsKafkaRawMessageSet : public ISrsCodec { private: - std::vector messages; + std::vector messages; public: - SrsKafkaMessageSet(); - virtual ~SrsKafkaMessageSet(); + SrsKafkaRawMessageSet(); + virtual ~SrsKafkaRawMessageSet(); +public: + virtual void append(SrsKafkaRawMessage* msg); +// interface ISrsCodec +public: + virtual int nb_bytes(); + virtual int encode(SrsBuffer* buf); + virtual int decode(SrsBuffer* buf); +}; + +/** + * the kafka request message, for protocol to send. + */ +class SrsKafkaRequest : public ISrsCodec +{ +protected: + SrsKafkaRequestHeader header; +public: + SrsKafkaRequest(); + virtual ~SrsKafkaRequest(); +public: + /** + * update the size in header. + * @param s an int value specifies the size of message in header. + */ + virtual void update_header(int s); + /** + * get the correlation id of header for message. + */ + virtual int32_t correlation_id(); + /** + * get the api key of request. + */ + virtual SrsKafkaApiKey api_key(); +// interface ISrsCodec +public: + virtual int nb_bytes(); + virtual int encode(SrsBuffer* buf); + virtual int decode(SrsBuffer* buf); +}; + +/** + * the kafka response message, for protocol to recv. + */ +class SrsKafkaResponse : public ISrsCodec +{ +protected: + SrsKafkaResponseHeader header; +public: + SrsKafkaResponse(); + virtual ~SrsKafkaResponse(); +public: + /** + * update the size in header. + * @param s an int value specifies the size of message in header. + */ + virtual void update_header(int s); +// interface ISrsCodec +public: + virtual int nb_bytes(); + virtual int encode(SrsBuffer* buf); + virtual int decode(SrsBuffer* buf); }; /** @@ -300,16 +643,304 @@ public: * * @see https://cwiki.apache.org/confluence/display/KAFKA/A+Guide+To+The+Kafka+Protocol#AGuideToTheKafkaProtocol-MetadataAPI */ -class SrsKafkaTopicMetadataRequest +class SrsKafkaTopicMetadataRequest : public SrsKafkaRequest { private: - SrsKafkaRequestHeader header; - SrsKafkaArray request; + SrsKafkaArray topics; public: SrsKafkaTopicMetadataRequest(); virtual ~SrsKafkaTopicMetadataRequest(); +public: + virtual void add_topic(std::string topic); +// interface ISrsCodec +public: + virtual int nb_bytes(); + virtual int encode(SrsBuffer* buf); + virtual int decode(SrsBuffer* buf); }; +/** + * the metadata response data. + * @see https://cwiki.apache.org/confluence/display/KAFKA/A+Guide+To+The+Kafka+Protocol#AGuideToTheKafkaProtocol-MetadataResponse + */ +struct SrsKafkaBroker : public ISrsCodec +{ +public: + int32_t node_id; + SrsKafkaString host; + int32_t port; +public: + SrsKafkaBroker(); + virtual ~SrsKafkaBroker(); +// interface ISrsCodec +public: + virtual int nb_bytes(); + virtual int encode(SrsBuffer* buf); + virtual int decode(SrsBuffer* buf); +}; +struct SrsKafkaPartitionMetadata : public ISrsCodec +{ +public: + int16_t error_code; + int32_t partition_id; + int32_t leader; + SrsKafkaArray replicas; + SrsKafkaArray isr; +public: + SrsKafkaPartitionMetadata(); + virtual ~SrsKafkaPartitionMetadata(); +// interface ISrsCodec +public: + virtual int nb_bytes(); + virtual int encode(SrsBuffer* buf); + virtual int decode(SrsBuffer* buf); +}; +struct SrsKafkaTopicMetadata : public ISrsCodec +{ +public: + int16_t error_code; + SrsKafkaString name; + SrsKafkaArray metadatas; +public: + SrsKafkaTopicMetadata(); + virtual ~SrsKafkaTopicMetadata(); +// interface ISrsCodec +public: + virtual int nb_bytes(); + virtual int encode(SrsBuffer* buf); + virtual int decode(SrsBuffer* buf); +}; + +/** + * response for the metadata request from broker. + * The response contains metadata for each partition, + * with partitions grouped together by topic. This + * metadata refers to brokers by their broker id. + * The brokers each have a host and port. + * @see https://cwiki.apache.org/confluence/display/KAFKA/A+Guide+To+The+Kafka+Protocol#AGuideToTheKafkaProtocol-MetadataResponse + */ +class SrsKafkaTopicMetadataResponse : public SrsKafkaResponse +{ +public: + SrsKafkaArray brokers; + SrsKafkaArray metadatas; +public: + SrsKafkaTopicMetadataResponse(); + virtual ~SrsKafkaTopicMetadataResponse(); +// interface ISrsCodec +public: + virtual int nb_bytes(); + virtual int encode(SrsBuffer* buf); + virtual int decode(SrsBuffer* buf); +}; + + +/** + * the messages for producer to send. + * @see https://cwiki.apache.org/confluence/display/KAFKA/A+Guide+To+The+Kafka+Protocol#AGuideToTheKafkaProtocol-ProduceRequest + */ +struct SrsKafkaProducerPartitionMessages : public ISrsCodec +{ +public: + /** + * The partition that data is being published to. + */ + int32_t partition; + /** + * The size, in bytes, of the message set that follows. + */ + int32_t message_set_size; + /** + * messages in set. + */ + SrsKafkaRawMessageSet messages; +// interface ISrsCodec +public: + virtual int nb_bytes(); + virtual int encode(SrsBuffer* buf); + virtual int decode(SrsBuffer* buf); +}; +struct SrsKafkaProducerTopicMessages : public ISrsCodec +{ +public: + /** + * The topic that data is being published to. + */ + SrsKafkaString topic_name; + /** + * messages of partitions. + */ + SrsKafkaArray partitions; +// interface ISrsCodec +public: + virtual int nb_bytes(); + virtual int encode(SrsBuffer* buf); + virtual int decode(SrsBuffer* buf); +}; + +/** + * the request for producer to send message. + * @see https://cwiki.apache.org/confluence/display/KAFKA/A+Guide+To+The+Kafka+Protocol#AGuideToTheKafkaProtocol-ProduceRequest + */ +class SrsKafkaProducerRequest : public SrsKafkaRequest +{ +public: + /** + * This field indicates how many acknowledgements the servers should receive + * before responding to the request. If it is 0 the server will not send any + * response (this is the only case where the server will not reply to a request). + * If it is 1, the server will wait the data is written to the local log + * before sending a response. If it is -1 the server will block until the + * message is committed by all in sync replicas before sending a response. + * For any number > 1 the server will block waiting for this number of + * acknowledgements to occur (but the server will never wait for more + * acknowledgements than there are in-sync replicas). + */ + int16_t required_acks; + /** + * This provides a maximum time in milliseconds the server can await the receipt + * of the number of acknowledgements in RequiredAcks. The timeout is not an exact + * limit on the request time for a few reasons: (1) it does not include network + * latency, (2) the timer begins at the beginning of the processing of this request + * so if many requests are queued due to server overload that wait time will not + * be included, (3) we will not terminate a local write so if the local write + * time exceeds this timeout it will not be respected. To get a hard timeout of + * this type the client should use the socket timeout. + */ + int32_t timeout; + /** + * messages of topics. + */ + SrsKafkaArray topics; +public: + SrsKafkaProducerRequest(); + virtual ~SrsKafkaProducerRequest(); +// interface ISrsCodec +public: + virtual int nb_bytes(); + virtual int encode(SrsBuffer* buf); + virtual int decode(SrsBuffer* buf); +}; + +/** + * the poll to discovery reponse. + * @param CorrelationId This is a user-supplied integer. It will be passed back + * in the response by the server, unmodified. It is useful for matching + * request and response between the client and server. + * @see https://cwiki.apache.org/confluence/display/KAFKA/A+Guide+To+The+Kafka+Protocol#AGuideToTheKafkaProtocol-Requests + */ +class SrsKafkaCorrelationPool +{ +private: + static SrsKafkaCorrelationPool* _instance; +public: + static SrsKafkaCorrelationPool* instance(); +private: + std::map correlation_ids; +private: + SrsKafkaCorrelationPool(); +public: + virtual ~SrsKafkaCorrelationPool(); +public: + /** + * generate a global correlation id. + */ + virtual int32_t generate_correlation_id(); + /** + * set the correlation id to specified request key. + */ + virtual SrsKafkaApiKey set(int32_t correlation_id, SrsKafkaApiKey request); + /** + * unset the correlation id. + * @return the previous api key; unknown if not set. + */ + virtual SrsKafkaApiKey unset(int32_t correlation_id); + /** + * get the key by specified correlation id. + * @return the specified api key; unknown if no correlation id. + */ + virtual SrsKafkaApiKey get(int32_t correlation_id); +}; + +/** + * the kafka protocol stack, use to send and recv kakfa messages. + */ +class SrsKafkaProtocol +{ +private: + ISrsProtocolReaderWriter* skt; + SrsFastStream* reader; +public: + SrsKafkaProtocol(ISrsProtocolReaderWriter* io); + virtual ~SrsKafkaProtocol(); +public: + /** + * write the message to kafka server. + * @param msg the msg to send. user must not free it again. + */ + virtual int send_and_free_message(SrsKafkaRequest* msg); + /** + * read the message from kafka server. + * @param pmsg output the received message. user must free it. + */ + virtual int recv_message(SrsKafkaResponse** pmsg); +public: + /** + * expect specified message. + */ + template + int expect_message(T** pmsg) + { + int ret = ERROR_SUCCESS; + + while (true) { + SrsKafkaResponse* res = NULL; + if ((ret = recv_message(&res)) != ERROR_SUCCESS) { + srs_error("recv response failed. ret=%d", ret); + return ret; + } + + // drop not matched. + T* msg = dynamic_cast(res); + if (!msg) { + srs_info("kafka drop response."); + srs_freep(res); + continue; + } + + *pmsg = msg; + break; + } + + return ret; + } +}; + +/** + * the kafka client, for producer or consumer. + */ +class SrsKafkaClient +{ +private: + SrsKafkaProtocol* protocol; +public: + SrsKafkaClient(ISrsProtocolReaderWriter* io); + virtual ~SrsKafkaClient(); +public: + /** + * fetch the metadata from broker for topic. + */ + virtual int fetch_metadata(std::string topic, SrsKafkaTopicMetadataResponse** pmsg); + /** + * write the messages to partition of topic. + */ + virtual int write_messages(std::string topic, int32_t partition, std::vector& msgs); +}; + +// convert kafka array[string] to vector[string] +extern std::vector srs_kafka_array2vector(SrsKafkaArray* arr); +extern std::vector srs_kafka_array2vector(SrsKafkaArray* arr); + #endif #endif diff --git a/trunk/src/protocol/srs_protocol_utility.cpp b/trunk/src/protocol/srs_protocol_utility.cpp index 682f92602..be36509b2 100644 --- a/trunk/src/protocol/srs_protocol_utility.cpp +++ b/trunk/src/protocol/srs_protocol_utility.cpp @@ -44,7 +44,7 @@ using namespace std; void srs_discovery_tc_url( string tcUrl, string& schema, string& host, string& vhost, - string& app, int& port, std::string& param + string& app, int& port, string& param ) { size_t pos = std::string::npos; std::string url = tcUrl; @@ -229,7 +229,7 @@ int srs_rtmp_create_msg(char type, u_int32_t timestamp, char* data, int size, in return ret; } -std::string srs_generate_stream_url(std::string vhost, std::string app, std::string stream) +string srs_generate_stream_url(string vhost, string app, string stream) { std::string url = ""; @@ -244,6 +244,18 @@ std::string srs_generate_stream_url(std::string vhost, std::string app, std::str return url; } +void srs_parse_rtmp_url(string url, string& tcUrl, string& stream) +{ + size_t pos; + + if ((pos = url.rfind("/")) != string::npos) { + stream = url.substr(pos + 1); + tcUrl = url.substr(0, pos); + } else { + tcUrl = url; + } +} + string srs_generate_rtmp_url(string server, int port, string vhost, string app, string stream) { std::stringstream ss; @@ -300,3 +312,17 @@ int srs_write_large_iovs(ISrsProtocolReaderWriter* skt, iovec* iovs, int size, s return ret; } +string srs_join_vector_string(vector& vs, string separator) +{ + string str = ""; + + for (int i = 0; i < (int)vs.size(); i++) { + str += vs.at(i); + if (i != (int)vs.size() - 1) { + str += separator; + } + } + + return str; +} + diff --git a/trunk/src/protocol/srs_protocol_utility.hpp b/trunk/src/protocol/srs_protocol_utility.hpp index f3650f0a9..41893a34c 100644 --- a/trunk/src/protocol/srs_protocol_utility.hpp +++ b/trunk/src/protocol/srs_protocol_utility.hpp @@ -35,6 +35,7 @@ CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. #endif #include +#include #include @@ -101,16 +102,37 @@ extern bool srs_bytes_equals(void* pa, void* pb, int size); * @param data the packet bytes. user should never free it. * @param ppmsg output the shared ptr message. user should free it. */ -extern int srs_rtmp_create_msg(char type, u_int32_t timestamp, char* data, int size, int stream_id, SrsSharedPtrMessage** ppmsg); +extern int srs_rtmp_create_msg( + char type, u_int32_t timestamp, char* data, int size, int stream_id, + SrsSharedPtrMessage** ppmsg +); // get the stream identify, vhost/app/stream. -extern std::string srs_generate_stream_url(std::string vhost, std::string app, std::string stream); +extern std::string srs_generate_stream_url( + std::string vhost, std::string app, std::string stream +); + +// parse the rtmp url to tcUrl/stream, +// for example, rtmp://v.ossrs.net/live/livestream to +// tcUrl: rtmp://v.ossrs.net/live +// stream: livestream +extern void srs_parse_rtmp_url( + std::string url, std::string& tcUrl, std::string& stream +); // genereate the rtmp url, for instance, rtmp://server:port/app...vhost...vhost/stream -extern std::string srs_generate_rtmp_url(std::string server, int port, std::string vhost, std::string app, std::string stream); +extern std::string srs_generate_rtmp_url( + std::string server, int port, std::string vhost, std::string app, std::string stream +); // write large numbers of iovs. -extern int srs_write_large_iovs(ISrsProtocolReaderWriter* skt, iovec* iovs, int size, ssize_t* pnwrite = NULL); +extern int srs_write_large_iovs( + ISrsProtocolReaderWriter* skt, iovec* iovs, int size, + ssize_t* pnwrite = NULL +); + +// join string in vector with indicated separator +extern std::string srs_join_vector_string(std::vector& vs, std::string separator); #endif diff --git a/trunk/src/protocol/srs_rtmp_stack.cpp b/trunk/src/protocol/srs_rtmp_stack.cpp index 0336f748e..8fc391d77 100644 --- a/trunk/src/protocol/srs_rtmp_stack.cpp +++ b/trunk/src/protocol/srs_rtmp_stack.cpp @@ -654,12 +654,12 @@ int SrsProtocol::do_simple_send(SrsMessageHeader* mh, char* payload, int size) int nbh = 0; if (p == payload) { nbh = srs_chunk_header_c0( - mh->perfer_cid, mh->timestamp, mh->payload_length, + mh->perfer_cid, (u_int32_t)mh->timestamp, mh->payload_length, mh->message_type, mh->stream_id, c0c3, sizeof(c0c3)); } else { nbh = srs_chunk_header_c3( - mh->perfer_cid, mh->timestamp, + mh->perfer_cid, (u_int32_t)mh->timestamp, c0c3, sizeof(c0c3)); } srs_assert(nbh > 0);; @@ -668,7 +668,7 @@ int SrsProtocol::do_simple_send(SrsMessageHeader* mh, char* payload, int size) iovs[0].iov_base = c0c3; iovs[0].iov_len = nbh; - int payload_size = srs_min(end - p, out_chunk_size); + int payload_size = srs_min((int)(end - p), out_chunk_size); iovs[1].iov_base = p; iovs[1].iov_len = payload_size; p += payload_size; diff --git a/trunk/src/protocol/srs_rtmp_stack.hpp b/trunk/src/protocol/srs_rtmp_stack.hpp index e895fc3f3..12dbb574d 100644 --- a/trunk/src/protocol/srs_rtmp_stack.hpp +++ b/trunk/src/protocol/srs_rtmp_stack.hpp @@ -997,7 +997,7 @@ public: * for example: * SrsCommonMessage* msg = NULL; * SrsConnectAppResPacket* pkt = NULL; - * if ((ret = server->expect_message(protocol, &msg, &pkt)) != ERROR_SUCCESS) { + * if ((ret = server->expect_message(&msg, &pkt)) != ERROR_SUCCESS) { * return ret; * } * // use then free msg and pkt diff --git a/trunk/src/protocol/srs_rtsp_stack.cpp b/trunk/src/protocol/srs_rtsp_stack.cpp index 772957fee..0987c7d9d 100644 --- a/trunk/src/protocol/srs_rtsp_stack.cpp +++ b/trunk/src/protocol/srs_rtsp_stack.cpp @@ -988,10 +988,7 @@ int SrsRtspStack::do_recv_message(SrsRtspRequest* req) // for setup, parse the stream id from uri. if (req->is_setup()) { size_t pos = string::npos; - std::string stream_id; - if ((pos = req->uri.rfind("/")) != string::npos) { - stream_id = req->uri.substr(pos + 1); - } + std::string stream_id = srs_path_basename(req->uri); if ((pos = stream_id.find("=")) != string::npos) { stream_id = stream_id.substr(pos + 1); }