mptcp/wanproxy
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

version 3.01

Browse source
This commit is contained in:
Bramfeld Team 2016-02-19 14:27:00 +01:00
parent 5ab09e78b2
commit 7adaaf99be
9 changed files with 214 additions and 112 deletions
Download patch file Download diff file Expand all files Collapse all files

4
xcodec/xcodec_decoder.cc
View file

@ -72,8 +72,8 @@ XCodecDecoder::~XCodecDecoder()
* instance UUID in the HELLO message and can tell which streams
* share an originator.
*/
bool
XCodecDecoder::decode (Buffer& output, Buffer& input, std::set<uint64_t>& unknown_hashes)
bool XCodecDecoder::decode (Buffer& output, Buffer& input, std::set<uint64_t>& unknown_hashes)
{
uint8_t data[XCODEC_SEGMENT_LENGTH];
Buffer old;

143
xcodec/xcodec_encoder.cc
View file

@ -29,7 +29,6 @@
#include <xcodec/xcodec.h>
#include <xcodec/xcodec_cache.h>
#include <xcodec/xcodec_encoder.h>
#include <xcodec/xcodec_hash.h>
////////////////////////////////////////////////////////////////////////////////
// //
@ -41,17 +40,13 @@
// //
////////////////////////////////////////////////////////////////////////////////
struct candidate_symbol
{
bool set_;
unsigned offset_;
uint64_t symbol_;
};
XCodecEncoder::XCodecEncoder(XCodecCache *cache)
: log_("/xcodec/encoder"),
cache_(cache)
{ }
{
candidate_start_ = -1;
candidate_symbol_ = 0;
}
XCodecEncoder::~XCodecEncoder()
{ }
@ -61,14 +56,13 @@ XCodecEncoder::~XCodecEncoder()
* to other data, declarations of data to be referenced, and data that needs
* escaped.
*/
void
XCodecEncoder::encode (Buffer& output, Buffer& input)
void XCodecEncoder::encode (Buffer& output, Buffer& input, bool wait)
{
XCodecHash xcodec_hash;
candidate_symbol candidate = {0, 0, 0};
unsigned offset = 0;
unsigned o = 0;
int off = source_.length ();
Buffer old;
source_.append (input);
for (Buffer::SegmentIterator it = input.segments (); ! it.end (); it.next ())
{
@ -80,14 +74,14 @@ XCodecEncoder::encode (Buffer& output, Buffer& input)
/*
* Add bytes to the hash until we have a complete hash.
*/
if (++o < XCODEC_SEGMENT_LENGTH)
xcodec_hash.add (*p);
if (++off < XCODEC_SEGMENT_LENGTH)
xcodec_hash_.add (*p);
else
{
if (o == XCODEC_SEGMENT_LENGTH)
xcodec_hash.add (*p);
if (off == XCODEC_SEGMENT_LENGTH)
xcodec_hash_.add (*p);
else
xcodec_hash.roll (*p);
xcodec_hash_.roll (*p);
/*
* And then mix the hash's internal state into a
@ -95,19 +89,18 @@ XCodecEncoder::encode (Buffer& output, Buffer& input)
* and to look up possible past occurances of that
* data in the XCodecCache.
*/
uint64_t hash = xcodec_hash.mix ();
uint64_t hash = xcodec_hash_.mix ();
/*
* If there is a pending candidate hash that wouldn't
* overlap with the data that the rolling hash presently
* covers, declare it now.
*/
if (candidate.set_ && candidate.offset_ + (XCODEC_SEGMENT_LENGTH * 2) <= offset + o)
if (candidate_start_ >= 0 && candidate_start_ + (XCODEC_SEGMENT_LENGTH * 2) <= off)
{
encode_declaration (output, input, offset, candidate.offset_, candidate.symbol_);
o -= (candidate.offset_ + XCODEC_SEGMENT_LENGTH - offset);
offset = (candidate.offset_ + XCODEC_SEGMENT_LENGTH);
candidate.set_ = false;
encode_declaration (output, source_, candidate_start_, candidate_symbol_);
off -= (candidate_start_ + XCODEC_SEGMENT_LENGTH);
candidate_start_ = -1;
}
/*
@ -122,17 +115,16 @@ XCodecEncoder::encode (Buffer& output, Buffer& input)
* identical to this chunk of data, then that's
* positively fantastic.
*/
if (encode_reference (output, input, offset, offset + o - XCODEC_SEGMENT_LENGTH, hash, old))
if (encode_reference (output, source_, off - XCODEC_SEGMENT_LENGTH, hash, old))
{
/*
* We have output any data before this hash
* in escaped form, so any candidate hash
* before it is invalid now.
*/
offset += o;
o = 0;
xcodec_hash.reset();
candidate.set_ = false;
off = 0;
xcodec_hash_.reset();
candidate_start_ = -1;
}
else
{
@ -152,7 +144,7 @@ XCodecEncoder::encode (Buffer& output, Buffer& input)
* Not defined before, it's a candidate for declaration
* if we don't already have one.
*/
if (candidate.set_)
if (candidate_start_ >= 0)
{
/*
* We already have a hash that occurs earlier,
@ -160,7 +152,7 @@ XCodecEncoder::encode (Buffer& output, Buffer& input)
* covered by this hash, so don't remember it
* and keep going.
*/
ASSERT(log_, candidate.offset_ + (XCODEC_SEGMENT_LENGTH * 2) > offset + o);
ASSERT(log_, candidate_start_ + (XCODEC_SEGMENT_LENGTH * 2) > off);
}
else
{
@ -171,80 +163,99 @@ XCodecEncoder::encode (Buffer& output, Buffer& input)
* find something to reference we can declare this one
* for future use.
*/
candidate.offset_ = offset + o - XCODEC_SEGMENT_LENGTH;
candidate.symbol_ = hash;
candidate.set_ = true;
candidate_start_ = off - XCODEC_SEGMENT_LENGTH;
candidate_symbol_ = hash;
}
}
}
}
}
if (! wait)
flush (output);
}
bool XCodecEncoder::flush (Buffer& output)
{
bool vld = false;
/*
* There's a hash we can declare, do it.
*/
if (candidate.set_)
if (candidate_start_ >= 0)
{
encode_declaration (output, input, offset, candidate.offset_, candidate.symbol_);
o -= (candidate.offset_ + XCODEC_SEGMENT_LENGTH - offset);
offset = (candidate.offset_ + XCODEC_SEGMENT_LENGTH);
candidate.set_ = false;
encode_declaration (output, source_, candidate_start_, candidate_symbol_);
candidate_start_ = -1;
vld = true;
}
/*
* There's data after that hash or no candidate hash, so
* just escape it.
* There's data after that hash or no candidate hash, so just escape it.
*/
if (offset < input.length ())
encode_escape (output, input, offset, input.length ());
if (source_.length () > 0)
{
encode_escape (output, source_, source_.length ());
vld = true;
}
xcodec_hash_.reset();
return vld;
}
void
XCodecEncoder::encode_declaration (Buffer& output, Buffer& input, unsigned offset, unsigned start, uint64_t hash)
void XCodecEncoder::encode_declaration (Buffer& output, Buffer& input, unsigned start, uint64_t hash)
{
if (offset < start)
encode_escape (output, input, offset, start);
if (start > 0)
encode_escape (output, input, start);
cache_->enter (hash, input, start);
cache_->enter (hash, input, 0);
output.append (XCODEC_MAGIC);
output.append (XCODEC_OP_EXTRACT);
output.append (input, start, XCODEC_SEGMENT_LENGTH);
output.append (input, XCODEC_SEGMENT_LENGTH);
input.skip (XCODEC_SEGMENT_LENGTH);
}
void
XCodecEncoder::encode_escape (Buffer& output, Buffer& input, unsigned offset, unsigned limit)
void XCodecEncoder::encode_escape (Buffer& output, Buffer& input, unsigned length)
{
unsigned pos;
while (offset < limit && input.find (XCODEC_MAGIC, offset, limit - offset, &pos))
while (length > 0)
{
if (offset < pos)
output.append (input, offset, pos - offset);
output.append (XCODEC_MAGIC);
output.append (XCODEC_OP_ESCAPE);
offset = pos + 1;
if (input.find (XCODEC_MAGIC, 0, length, &pos))
{
if (pos > 0)
output.append (input, 0, pos);
output.append (XCODEC_MAGIC);
output.append (XCODEC_OP_ESCAPE);
input.skip (pos + 1);
length -= pos + 1;
}
else
{
output.append (input, length);
input.skip (length);
break;
}
}
if (offset < limit)
output.append (input, offset, limit - offset);
}
bool
XCodecEncoder::encode_reference (Buffer& output, Buffer& input, unsigned offset, unsigned start, uint64_t hash, Buffer& old)
bool XCodecEncoder::encode_reference (Buffer& output, Buffer& input, unsigned start, uint64_t hash, Buffer& old)
{
uint8_t data[XCODEC_SEGMENT_LENGTH];
input.copyout (data, start, XCODEC_SEGMENT_LENGTH);
if (old.equal (data, sizeof data))
{
if (offset < start)
encode_escape (output, input, offset, start);
if (start > 0)
encode_escape (output, input, start);
output.append (XCODEC_MAGIC);
output.append (XCODEC_OP_REF);
uint64_t behash = BigEndian::encode (hash);
output.append (&behash);
input.skip (XCODEC_SEGMENT_LENGTH);
return true;
}

16
xcodec/xcodec_encoder.h
View file

@ -26,6 +26,8 @@
#ifndef XCODEC_XCODEC_ENCODER_H
#define XCODEC_XCODEC_ENCODER_H
#include <xcodec/xcodec_hash.h>
////////////////////////////////////////////////////////////////////////////////
// //
// File: xcodec_encoder.h //
@ -42,16 +44,22 @@ class XCodecEncoder
{
LogHandle log_;
XCodecCache* cache_;
Buffer source_;
XCodecHash xcodec_hash_;
int candidate_start_;
uint64_t candidate_symbol_;
public:
XCodecEncoder(XCodecCache*);
~XCodecEncoder();
void encode (Buffer&, Buffer&);
void encode (Buffer&, Buffer&, bool);
bool flush (Buffer&);
private:
void encode_declaration (Buffer&, Buffer&, unsigned, unsigned, uint64_t);
void encode_escape (Buffer&, Buffer&, unsigned, unsigned);
bool encode_reference (Buffer&, Buffer&, unsigned, unsigned, uint64_t, Buffer&);
void encode_declaration (Buffer&, Buffer&, unsigned, uint64_t);
void encode_escape (Buffer&, Buffer&, unsigned);
bool encode_reference (Buffer&, Buffer&, unsigned, uint64_t, Buffer&);
};
#endif /* !XCODEC_XCODEC_ENCODER_H */

67
xcodec/xcodec_filter.cc
View file

@ -25,6 +25,7 @@
#include <common/buffer.h>
#include <common/endian.h>
#include <event/event_system.h>
#include <programs/wanproxy/wanproxy.h>
#include "xcodec_filter.h"
@ -120,8 +121,7 @@
bool EncodeFilter::consume (Buffer& buf)
{
Buffer output;
Buffer enc;
Buffer enc, output;
ASSERT(log_, ! flushing_);
@ -143,26 +143,18 @@ bool EncodeFilter::consume (Buffer& buf)
return false;
}
encoder_->encode (enc, buf);
while (! enc.empty ())
{
int n = enc.length ();
if (n > XCODEC_PIPE_MAX_FRAME)
n = XCODEC_PIPE_MAX_FRAME;
Buffer frame;
enc.moveout (&frame, n);
uint16_t len = n;
len = BigEndian::encode (len);
output.append (XCODEC_PIPE_OP_FRAME);
output.append (&len);
output.append (frame);
}
encoder_->encode (enc, buf, waiting_);
while (! enc.empty ())
encode_frame (enc, output);
return produce (output);
if (waiting_)
{
if (wait_action_)
wait_action_->cancel ();
wait_action_ = event_system.track (150, StreamModeWait, callback (this, &EncodeFilter::on_read_timeout));
}
return (! output.empty () ? produce (output) : true);
}
void EncodeFilter::flush (int flg)
@ -175,7 +167,9 @@ void EncodeFilter::flush (int flg)
flush_flags_ |= flg;
if (! sent_eos_)
{
Buffer output;
Buffer enc, output;
if (waiting_ && encoder_ && encoder_->flush (enc))
encode_frame (enc, output);
output.append (XCODEC_PIPE_OP_EOS);
sent_eos_ = produce (output);
}
@ -184,6 +178,35 @@ void EncodeFilter::flush (int flg)
Filter::flush (flush_flags_);
}
void EncodeFilter::encode_frame (Buffer& src, Buffer& trg)
{
int n = src.length ();
if (n > XCODEC_PIPE_MAX_FRAME)
n = XCODEC_PIPE_MAX_FRAME;
uint16_t len = n;
len = BigEndian::encode (len);
trg.append (XCODEC_PIPE_OP_FRAME);
trg.append (&len);
trg.append (src, n);
src.skip (n);
}
void EncodeFilter::on_read_timeout (Event e)
{
if (wait_action_)
wait_action_->cancel (), wait_action_ = 0;
Buffer enc, output;
if (encoder_ && encoder_->flush (enc))
{
encode_frame (enc, output);
produce (output);
}
}
// Decoding
bool DecodeFilter::consume (Buffer& buf)

14
xcodec/xcodec_filter.h
View file

@ -13,6 +13,8 @@
#include <set>
#include <common/filter.h>
#include <event/event.h>
#include <event/action.h>
#include <xcodec/xcodec.h>
#include <xcodec/xcodec_cache.h>
#include <xcodec/xcodec_hash.h>
@ -24,22 +26,30 @@ class EncodeFilter : public BufferedFilter
private:
XCodecCache* cache_;
XCodecEncoder* encoder_;
Action* wait_action_;
bool waiting_;
bool sent_eos_;
bool eos_ack_;
public:
EncodeFilter (const LogHandle& log, XCodecCache* cc) : BufferedFilter (log)
EncodeFilter (const LogHandle& log, XCodecCache* cc, int flg = 0) : BufferedFilter (log)
{
cache_ = cc; encoder_ = 0; sent_eos_ = eos_ack_ = false;
cache_ = cc; encoder_ = 0; wait_action_ = 0; waiting_ = (flg & 1); sent_eos_ = eos_ack_ = false;
}
virtual ~EncodeFilter ()
{
if (wait_action_)
wait_action_->cancel ();
delete encoder_;
}
virtual bool consume (Buffer& buf);
virtual void flush (int flg);
private:
void encode_frame (Buffer& src, Buffer& trg);
void on_read_timeout (Event e);
};
class DecodeFilter : public LogisticFilter