version 3.0
This commit is contained in:
commit
d837490606
209 changed files with 19662 additions and 0 deletions
4
xcodec/cache/coss/Makefile
vendored
Normal file
4
xcodec/cache/coss/Makefile
vendored
Normal file
|
@ -0,0 +1,4 @@
|
|||
SUBDIR+=test
|
||||
|
||||
include ../../../common/subdir.mk
|
||||
|
5
xcodec/cache/coss/lib.mk
vendored
Normal file
5
xcodec/cache/coss/lib.mk
vendored
Normal file
|
@ -0,0 +1,5 @@
|
|||
|
||||
VPATH+= ${TOPDIR}/xcodec/cache/coss
|
||||
|
||||
SRCS+= xcodec_cache_coss.cc
|
||||
|
3
xcodec/cache/coss/test/Makefile
vendored
Normal file
3
xcodec/cache/coss/test/Makefile
vendored
Normal file
|
@ -0,0 +1,3 @@
|
|||
SUBDIR+=xcodec-coss1
|
||||
|
||||
include ../../../../common/subdir.mk
|
7
xcodec/cache/coss/test/xcodec-coss1/Makefile
vendored
Normal file
7
xcodec/cache/coss/test/xcodec-coss1/Makefile
vendored
Normal file
|
@ -0,0 +1,7 @@
|
|||
TEST=xcodec-coss1
|
||||
|
||||
TOPDIR=../../../../..
|
||||
USE_LIBS=common common/uuid xcodec xcodec/cache/coss
|
||||
include ${TOPDIR}/common/program.mk
|
||||
LDADD+=-lboost_filesystem -lboost_system
|
||||
|
96
xcodec/cache/coss/test/xcodec-coss1/xcodec-coss1.cc
vendored
Normal file
96
xcodec/cache/coss/test/xcodec-coss1/xcodec-coss1.cc
vendored
Normal file
|
@ -0,0 +1,96 @@
|
|||
#include <common/buffer.h>
|
||||
#include <common/test.h>
|
||||
#include <common/uuid/uuid.h>
|
||||
|
||||
#include <xcodec/xcodec.h>
|
||||
#include <xcodec/xcodec_cache.h>
|
||||
#include <xcodec/xcodec_decoder.h>
|
||||
#include <xcodec/xcodec_encoder.h>
|
||||
#include <xcodec/xcodec_hash.h>
|
||||
#include <xcodec/cache/coss/xcodec_cache_coss.h>
|
||||
|
||||
#include <boost/filesystem.hpp>
|
||||
|
||||
#include <fstream>
|
||||
|
||||
#include <stdlib.h>
|
||||
|
||||
using namespace boost::filesystem;
|
||||
|
||||
int
|
||||
main(void)
|
||||
{
|
||||
|
||||
char tmp_template[] = "/tmp/cache-coss-XXXXXX";
|
||||
path cache_path = mkdtemp(tmp_template);
|
||||
create_directory(cache_path);
|
||||
|
||||
typedef pair<uint64_t, const uint8_t*> segment_list_element_t;
|
||||
typedef deque<segment_list_element_t> segment_list_t;
|
||||
segment_list_t segment_list;
|
||||
|
||||
{
|
||||
TestGroup g("/test/xcodec/encode-decode-coss/2/char_kat",
|
||||
"XCodecEncoder::encode / XCodecDecoder::decode #2");
|
||||
|
||||
UUID uuid;
|
||||
std::string cache_path_str = cache_path.string();
|
||||
unsigned i, j;
|
||||
|
||||
uuid.generate();
|
||||
|
||||
for (j = 0; j < 4; j++) {
|
||||
XCodecCache *cache = new XCodecCacheCOSS(uuid, cache_path_str,
|
||||
10, 10, 10);
|
||||
|
||||
for (i = 0; i < 10000; i++) {
|
||||
uint8_t random[XCODEC_SEGMENT_LENGTH];
|
||||
|
||||
ifstream rand_fd("/dev/urandom");
|
||||
rand_fd.read(random, sizeof(random));
|
||||
ASSERT("xcodec-coss1", rand_fd.good());
|
||||
|
||||
uint64_t hash = XCodecHash::hash(random);
|
||||
const uint8_t* data = cache->lookup(hash);
|
||||
if (data)
|
||||
continue;
|
||||
segment_list.push_front(make_pair(hash, data));
|
||||
Buffer buf (data, XCODEC_SEGMENT_LENGTH);
|
||||
cache->enter(hash, buf, 0);
|
||||
}
|
||||
|
||||
delete cache;
|
||||
cache = new XCodecCacheCOSS(uuid, cache_path_str,
|
||||
10, 10, 10);
|
||||
|
||||
segment_list_element_t el;
|
||||
const uint8_t *seg1, *seg2;
|
||||
|
||||
uint64_t hash;
|
||||
while (!segment_list.empty()){
|
||||
el = segment_list.back();
|
||||
segment_list.pop_back();
|
||||
seg1 = el.second;
|
||||
seg2 = cache->lookup(el.first);
|
||||
hash = el.first;
|
||||
if (!seg2)
|
||||
cout << "Segment not found: " << hash << endl;;
|
||||
if (seg2) {
|
||||
if (memcmp (seg1, seg2, XCODEC_SEGMENT_LENGTH))
|
||||
cout << "Segments are not equal: " << hash <<
|
||||
endl;
|
||||
Test _(g, "Segment are not equal.",
|
||||
seg1->equal(seg2));
|
||||
}
|
||||
}
|
||||
|
||||
delete cache;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
remove_all(cache_path);
|
||||
|
||||
return (0);
|
||||
}
|
||||
|
371
xcodec/cache/coss/xcodec_cache_coss.cc
vendored
Normal file
371
xcodec/cache/coss/xcodec_cache_coss.cc
vendored
Normal file
|
@ -0,0 +1,371 @@
|
|||
/*
|
||||
*
|
||||
* XCodec COSS Cache
|
||||
*
|
||||
* COSS = Cyclic Object storage system
|
||||
*
|
||||
* Idea taken from Squid COSS cache.
|
||||
*
|
||||
* Diego Woitasen <diegows@xtech.com.ar>
|
||||
* XTECH
|
||||
*
|
||||
*/
|
||||
|
||||
#include <unistd.h>
|
||||
#include <sys/types.h>
|
||||
#include <sys/stat.h>
|
||||
|
||||
#include <xcodec/cache/coss/xcodec_cache_coss.h>
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// //
|
||||
// File: xcodec_cache_coss.cc //
|
||||
// Description: persistent cache on disk for xcodec protocol streams //
|
||||
// Project: WANProxy XTech //
|
||||
// Adapted by: Andreu Vidal Bramfeld-Software //
|
||||
// Last modified: 2015-08-31 //
|
||||
// //
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
|
||||
XCodecCacheCOSS::XCodecCacheCOSS (const UUID& uuid, const std::string& cache_dir, size_t cache_size)
|
||||
: XCodecCache(uuid, cache_size),
|
||||
log_("xcodec/cache/coss")
|
||||
{
|
||||
uint8_t str[UUID_STRING_SIZE + 1];
|
||||
uuid.to_string (str);
|
||||
file_path_ = cache_dir;
|
||||
if (file_path_.size() > 0 && file_path_[file_path_.size() - 1] != '/')
|
||||
file_path_.append ("/");
|
||||
file_path_.append ((const char*) str, UUID_STRING_SIZE);
|
||||
file_path_.append (".wpc");
|
||||
|
||||
struct stat st;
|
||||
if (::stat (file_path_.c_str(), &st) == 0 && (st.st_mode & S_IFREG))
|
||||
file_size_ = st.st_size;
|
||||
else
|
||||
{
|
||||
ofstream tmp (file_path_.c_str());
|
||||
file_size_ = 0;
|
||||
}
|
||||
|
||||
serial_number_ = 0;
|
||||
stripe_range_ = 0;
|
||||
if (! cache_size)
|
||||
cache_size = CACHE_BASIC_SIZE;
|
||||
uint64_t size = ROUND_UP((uint64_t) cache_size * 1048576, sizeof (COSSStripe));
|
||||
stripe_limit_ = size / sizeof (COSSStripe);
|
||||
freshness_level_ = 0;
|
||||
active_ = 0;
|
||||
|
||||
directory_ = new COSSMetadata[stripe_limit_];
|
||||
memset (directory_, 0, sizeof (COSSMetadata) * stripe_limit_);
|
||||
|
||||
stream_.open (file_path_.c_str(), fstream::in | fstream::out | fstream::binary);
|
||||
if (! read_file ())
|
||||
{
|
||||
stream_.close ();
|
||||
stream_.open (file_path_.c_str(), fstream::in | fstream::out | fstream::trunc | fstream::binary);
|
||||
file_size_ = 0;
|
||||
initialize_stripe (stripe_range_, active_);
|
||||
}
|
||||
|
||||
DEBUG(log_) << "Cache file: " << file_path_;
|
||||
DEBUG(log_) << "Max size: " << size;
|
||||
DEBUG(log_) << "Stripe size: " << sizeof (COSSStripe);
|
||||
DEBUG(log_) << "Stripe header size: " << sizeof (COSSStripeHeader);
|
||||
DEBUG(log_) << "Serial: " << serial_number_;
|
||||
DEBUG(log_) << "Stripe number: " << stripe_range_;
|
||||
}
|
||||
|
||||
XCodecCacheCOSS::~XCodecCacheCOSS()
|
||||
{
|
||||
for (int i = 0; i < LOADED_STRIPE_COUNT; ++i)
|
||||
if (stripe_[i].header.metadata.state == 1)
|
||||
store_stripe (i, (i == active_ ? sizeof (COSSStripe) : sizeof (COSSStripeHeader)));
|
||||
|
||||
stream_.close();
|
||||
|
||||
delete[] directory_;
|
||||
|
||||
/*
|
||||
INFO(log_) << "Stats: ";
|
||||
INFO(log_) << "\tLookups=" << stats_.lookups;
|
||||
INFO(log_) << "\tHits=" << (stats_.found_1 + stats_.found_2) << " (" << stats_.found_1 << " + " << stats_.found_2 << ")";
|
||||
if (stats_.lookups > 0)
|
||||
INFO(log_) << "\tHit ratio=" << ((stats_.found_1 + stats_.found_2) * 100) / stats_.lookups << "%";
|
||||
*/
|
||||
|
||||
DEBUG(log_) << "Closing coss file: " << file_path_;
|
||||
DEBUG(log_) << "Serial: " << serial_number_;
|
||||
DEBUG(log_) << "Stripe number: " << stripe_range_;
|
||||
DEBUG(log_) << "Index size: " << cache_index_.size();
|
||||
}
|
||||
|
||||
bool XCodecCacheCOSS::read_file ()
|
||||
{
|
||||
COSSStripeHeader header;
|
||||
COSSIndexEntry entry;
|
||||
uint64_t serial, range, limit, level;
|
||||
uint64_t hash;
|
||||
|
||||
serial = range = limit = level = 0;
|
||||
limit = file_size_ / sizeof (COSSStripe);
|
||||
if (limit * sizeof (COSSStripe) != file_size_)
|
||||
return false;
|
||||
if (limit > stripe_limit_)
|
||||
limit = stripe_limit_;
|
||||
stream_.seekg (0);
|
||||
|
||||
for (uint64_t n = 0; n < limit; ++n)
|
||||
{
|
||||
stream_.read ((char*) &header, sizeof header);
|
||||
if (! stream_.good() || stream_.gcount () != sizeof header)
|
||||
return false;
|
||||
if (header.metadata.signature != CACHE_SIGNATURE)
|
||||
return false;
|
||||
if (header.metadata.segment_count > STRIPE_SEGMENT_COUNT)
|
||||
return false;
|
||||
stream_.seekg (sizeof (COSSStripe) - sizeof header, ios::cur);
|
||||
|
||||
if (header.metadata.serial_number > serial)
|
||||
serial = header.metadata.serial_number, range = n;
|
||||
if (header.metadata.freshness > level)
|
||||
level = header.metadata.freshness;
|
||||
|
||||
directory_[n] = header.metadata;
|
||||
directory_[n].state = 0;
|
||||
|
||||
for (int i = 0; i < STRIPE_SEGMENT_COUNT; ++i)
|
||||
{
|
||||
if ((hash = header.hash_array[i]))
|
||||
{
|
||||
entry.stripe_range = n;
|
||||
entry.position = i;
|
||||
cache_index_.insert (hash, entry);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (serial > 0)
|
||||
{
|
||||
serial_number_ = serial;
|
||||
stripe_range_ = range;
|
||||
freshness_level_ = level;
|
||||
load_stripe (stripe_range_, active_);
|
||||
}
|
||||
else
|
||||
{
|
||||
initialize_stripe (stripe_range_, active_);
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
void XCodecCacheCOSS::enter (const uint64_t& hash, const Buffer& buf, unsigned off)
|
||||
{
|
||||
COSSIndexEntry entry;
|
||||
|
||||
while (stripe_[active_].header.metadata.segment_index >= STRIPE_SEGMENT_COUNT)
|
||||
new_active ();
|
||||
|
||||
COSSStripe& act = stripe_[active_];
|
||||
act.header.hash_array[act.header.metadata.segment_index] = hash;
|
||||
buf.copyout (act.segment_array[act.header.metadata.segment_index].bytes, off, XCODEC_SEGMENT_LENGTH);
|
||||
entry.stripe_range = act.header.metadata.stripe_range;
|
||||
entry.position = act.header.metadata.segment_index;
|
||||
|
||||
act.header.metadata.segment_index++;
|
||||
while (act.header.metadata.segment_index < STRIPE_SEGMENT_COUNT &&
|
||||
act.header.hash_array[act.header.metadata.segment_index])
|
||||
act.header.metadata.segment_index++;
|
||||
act.header.metadata.segment_count++;
|
||||
act.header.metadata.freshness = ++freshness_level_;
|
||||
|
||||
cache_index_.insert (hash, entry);
|
||||
}
|
||||
|
||||
bool XCodecCacheCOSS::lookup (const uint64_t& hash, Buffer& buf)
|
||||
{
|
||||
const COSSIndexEntry* entry;
|
||||
const uint8_t* data;
|
||||
int slot;
|
||||
|
||||
stats_.lookups++;
|
||||
|
||||
if ((data = find_recent (hash)))
|
||||
{
|
||||
buf.append (data, XCODEC_SEGMENT_LENGTH);
|
||||
stats_.found_1++;
|
||||
return true;
|
||||
}
|
||||
|
||||
if (! (entry = cache_index_.lookup (hash)))
|
||||
return false;
|
||||
|
||||
for (slot = 0; slot < LOADED_STRIPE_COUNT; ++slot)
|
||||
if (stripe_[slot].header.metadata.stripe_range == entry->stripe_range)
|
||||
break;
|
||||
|
||||
if (slot >= LOADED_STRIPE_COUNT)
|
||||
{
|
||||
slot = best_unloadable_slot ();
|
||||
detach_stripe (slot);
|
||||
load_stripe (entry->stripe_range, slot);
|
||||
}
|
||||
|
||||
if (stripe_[slot].header.hash_array[entry->position] != hash)
|
||||
return false;
|
||||
|
||||
stripe_[slot].header.metadata.freshness = ++freshness_level_;
|
||||
stripe_[slot].header.metadata.uses++;
|
||||
stripe_[slot].header.metadata.credits++;
|
||||
stripe_[slot].header.metadata.load_uses++;
|
||||
stripe_[slot].header.flags[entry->position] |= 3;
|
||||
|
||||
data = stripe_[slot].segment_array[entry->position].bytes;
|
||||
remember (hash, data);
|
||||
buf.append (data, XCODEC_SEGMENT_LENGTH);
|
||||
stats_.found_2++;
|
||||
return true;
|
||||
}
|
||||
|
||||
void XCodecCacheCOSS::initialize_stripe (uint64_t range, int slot)
|
||||
{
|
||||
memset (&stripe_[slot].header, 0, sizeof (COSSStripeHeader));
|
||||
stripe_[slot].header.metadata.signature = CACHE_SIGNATURE;
|
||||
stripe_[slot].header.metadata.version = CACHE_VERSION;
|
||||
stripe_[slot].header.metadata.serial_number = ++serial_number_;
|
||||
stripe_[slot].header.metadata.stripe_range = range;
|
||||
stripe_[slot].header.metadata.state = 1;
|
||||
directory_[range] = stripe_[slot].header.metadata;
|
||||
}
|
||||
|
||||
bool XCodecCacheCOSS::load_stripe (uint64_t range, int slot)
|
||||
{
|
||||
uint64_t pos = range * sizeof (COSSStripe);
|
||||
if (pos < file_size_)
|
||||
{
|
||||
stream_.seekg (pos);
|
||||
stream_.read ((char*) &stripe_[slot], sizeof (COSSStripe));
|
||||
if (stream_.gcount () == sizeof (COSSStripe))
|
||||
{
|
||||
stripe_[slot].header.metadata.stripe_range = range;
|
||||
stripe_[slot].header.metadata.load_uses = 0;
|
||||
stripe_[slot].header.metadata.state = 1;
|
||||
directory_[range].state = 1;
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
stream_.clear ();
|
||||
return false;
|
||||
}
|
||||
|
||||
void XCodecCacheCOSS::store_stripe (int slot, size_t size)
|
||||
{
|
||||
uint64_t pos = stripe_[slot].header.metadata.stripe_range * sizeof (COSSStripe);
|
||||
if (pos != (uint64_t) stream_.tellp ())
|
||||
stream_.seekp (pos);
|
||||
stream_.write ((char*) &stripe_[slot], size);
|
||||
if (stream_.good () && pos + sizeof (COSSStripe) > file_size_)
|
||||
file_size_ = pos + sizeof (COSSStripe);
|
||||
stream_.clear ();
|
||||
}
|
||||
|
||||
void XCodecCacheCOSS::new_active ()
|
||||
{
|
||||
store_stripe (active_, sizeof (COSSStripe));
|
||||
active_ = best_unloadable_slot ();
|
||||
detach_stripe (active_);
|
||||
stripe_range_ = best_erasable_stripe ();
|
||||
if (load_stripe (stripe_range_, active_))
|
||||
purge_stripe (active_);
|
||||
else
|
||||
initialize_stripe (stripe_range_, active_);
|
||||
}
|
||||
|
||||
int XCodecCacheCOSS::best_unloadable_slot ()
|
||||
{
|
||||
uint64_t v, n = 0xFFFFFFFFFFFFFFFFull;
|
||||
int j = 0;
|
||||
|
||||
for (int i = 0; i < LOADED_STRIPE_COUNT; ++i)
|
||||
{
|
||||
if (i == active_)
|
||||
continue;
|
||||
if (stripe_[i].header.metadata.signature == 0)
|
||||
return i;
|
||||
if ((v = stripe_[i].header.metadata.freshness + stripe_[i].header.metadata.load_uses) < n)
|
||||
j = i, n = v;
|
||||
}
|
||||
|
||||
return j;
|
||||
}
|
||||
|
||||
uint64_t XCodecCacheCOSS::best_erasable_stripe ()
|
||||
{
|
||||
COSSMetadata* m;
|
||||
uint64_t v, n = 0xFFFFFFFFFFFFFFFFull;
|
||||
uint64_t i, j = 0;
|
||||
|
||||
for (m = directory_, i = 0; i < stripe_limit_; ++i, ++m)
|
||||
{
|
||||
if (m->state == 1)
|
||||
continue;
|
||||
if (m->signature == 0)
|
||||
return i;
|
||||
if ((v = m->freshness + m->uses) < n)
|
||||
j = i, n = v;
|
||||
}
|
||||
|
||||
return j;
|
||||
}
|
||||
|
||||
void XCodecCacheCOSS::detach_stripe (int slot)
|
||||
{
|
||||
if (stripe_[slot].header.metadata.state == 1)
|
||||
{
|
||||
uint64_t range = stripe_[slot].header.metadata.stripe_range;
|
||||
directory_[range] = stripe_[slot].header.metadata;
|
||||
directory_[range].state = 2;
|
||||
|
||||
for (int i = 0; i < STRIPE_SEGMENT_COUNT; ++i)
|
||||
{
|
||||
if (stripe_[slot].header.flags[i] & 1)
|
||||
{
|
||||
forget (stripe_[slot].header.hash_array[i]);
|
||||
stripe_[slot].header.flags[i] &= ~1;
|
||||
}
|
||||
}
|
||||
|
||||
stripe_[slot].header.metadata.state = 0;
|
||||
store_stripe (slot, sizeof (COSSStripeHeader));
|
||||
}
|
||||
}
|
||||
|
||||
void XCodecCacheCOSS::purge_stripe (int slot)
|
||||
{
|
||||
for (int i = STRIPE_SEGMENT_COUNT - 1; i >= 0; --i)
|
||||
{
|
||||
uint64_t hash = stripe_[slot].header.hash_array[i];
|
||||
if (hash && ! (stripe_[slot].header.flags[i] & 2))
|
||||
{
|
||||
cache_index_.erase (hash);
|
||||
stripe_[slot].header.hash_array[i] = 0;
|
||||
stripe_[slot].header.flags[i] = 0;
|
||||
stripe_[slot].header.metadata.segment_count--;
|
||||
}
|
||||
|
||||
stripe_[slot].header.flags[i] &= ~2;
|
||||
if (! stripe_[slot].header.hash_array[i])
|
||||
stripe_[slot].header.metadata.segment_index = i;
|
||||
}
|
||||
|
||||
stripe_[slot].header.metadata.serial_number = ++serial_number_;
|
||||
stripe_[slot].header.metadata.uses = stripe_[slot].header.metadata.credits;
|
||||
stripe_[slot].header.metadata.credits = 0;
|
||||
|
||||
if (stripe_[slot].header.metadata.segment_count >= STRIPE_SEGMENT_COUNT)
|
||||
INFO(log_) << "No more space available in cache";
|
||||
}
|
223
xcodec/cache/coss/xcodec_cache_coss.h
vendored
Normal file
223
xcodec/cache/coss/xcodec_cache_coss.h
vendored
Normal file
|
@ -0,0 +1,223 @@
|
|||
/*
|
||||
*
|
||||
* XCodec COSS Cache
|
||||
*
|
||||
* COSS = Cyclic Object Storage System
|
||||
*
|
||||
* Idea taken from Squid COSS.
|
||||
*
|
||||
* Diego Woitasen <diegows@xtech.com.ar>
|
||||
* XTECH
|
||||
*
|
||||
*/
|
||||
|
||||
#ifndef XCODEC_XCODEC_CACHE_COSS_H
|
||||
#define XCODEC_XCODEC_CACHE_COSS_H
|
||||
|
||||
#include <string>
|
||||
#include <map>
|
||||
#include <fstream>
|
||||
|
||||
#include <common/buffer.h>
|
||||
#include <xcodec/xcodec.h>
|
||||
#include <xcodec/xcodec_cache.h>
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// //
|
||||
// File: xcodec_cache_coss.h //
|
||||
// Description: persistent cache on disk for xcodec protocol streams //
|
||||
// Project: WANProxy XTech //
|
||||
// Adapted by: Andreu Vidal Bramfeld-Software //
|
||||
// Last modified: 2015-08-31 //
|
||||
// //
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
using namespace std;
|
||||
|
||||
/*
|
||||
* - In COSS, we have one file per cache (UUID). The file is divided in
|
||||
* stripes.
|
||||
*
|
||||
* - Each stripe is composed by:
|
||||
* metadata + hash array + segment size array + segment array.
|
||||
*
|
||||
* - The arrays elements are the same order:
|
||||
* hash1, hash2, ..., hashN - size1, size2, ..., sizeN - seg1, seg2, ..., segN.
|
||||
*
|
||||
* - The segments are indexed in memory. This index is loaded when the cache is
|
||||
* openned, reading the hash array of each stripe. Takes a few millisecons in
|
||||
* a 10 GB cache.
|
||||
*
|
||||
* - We have one active stripe in memory at a time. New segments are written
|
||||
* in the current stripe in order of appearance.
|
||||
*
|
||||
* - When a cached segment is requested and it's out of the active stripe,
|
||||
* is copied to it.
|
||||
*
|
||||
* - When the current stripe is full, we move to the next one.
|
||||
*
|
||||
* - When we reach the EOF, first stripe is zeroed and becomes active.
|
||||
*
|
||||
*/
|
||||
|
||||
// Changes introduced in version 2:
|
||||
//
|
||||
// - segment size is made independent of BUFFER_SEGMENT_SIZE and not stored explicitly
|
||||
// since it is always XCODEC_SEGMENT_LENGTH
|
||||
// - several stripes can be help simultaneously in memory, and when a segment is requested
|
||||
// which lies outside the active stripe it is read into an alternate slot together
|
||||
// with the whole stripe to be ready to satisfy requests for neighbour segments
|
||||
// - an array of bits keeps track of the state of each segment within a stripe signaling
|
||||
// if it has been recently used
|
||||
// - when no more place is available, the LRU stripe is purged and any segments
|
||||
// no used during the last period are erased
|
||||
|
||||
/*
|
||||
* This values should be page aligned.
|
||||
*/
|
||||
|
||||
#define CACHE_SIGNATURE 0xF150E964
|
||||
#define CACHE_VERSION 2
|
||||
#define STRIPE_SEGMENT_COUNT 512 // segments of XCODEC_SEGMENT_LENGTH per stripe (must fit into 16 bits)
|
||||
#define LOADED_STRIPE_COUNT 4 // number of stripes held in memory (must be greater than 1)
|
||||
#define CACHE_BASIC_SIZE 1024 // MB
|
||||
|
||||
#define CACHE_ALIGNEMENT 4096
|
||||
#define HEADER_ARRAY_SIZE (STRIPE_SEGMENT_COUNT * (sizeof (uint64_t) + sizeof (uint32_t)))
|
||||
#define METADATA_SIZE (sizeof (COSSMetadata))
|
||||
#define ROUND_UP(N, S) ((((N) + (S) - 1) / (S)) * (S))
|
||||
#define HEADER_ALIGNED_SIZE ROUND_UP(HEADER_ARRAY_SIZE + METADATA_SIZE, CACHE_ALIGNEMENT)
|
||||
#define METADATA_PADDING (HEADER_ALIGNED_SIZE - HEADER_ARRAY_SIZE - METADATA_SIZE)
|
||||
|
||||
struct COSSIndexEntry
|
||||
{
|
||||
uint64_t stripe_range : 48;
|
||||
uint64_t position : 16;
|
||||
};
|
||||
|
||||
class COSSIndex
|
||||
{
|
||||
typedef __gnu_cxx::hash_map<Hash64, COSSIndexEntry> index_t;
|
||||
index_t index;
|
||||
|
||||
public:
|
||||
void insert (const uint64_t& hash, const COSSIndexEntry& entry)
|
||||
{
|
||||
index[hash] = entry;
|
||||
}
|
||||
|
||||
const COSSIndexEntry* lookup (const uint64_t& hash)
|
||||
{
|
||||
index_t::iterator it = index.find (hash);
|
||||
return (it != index.end () ? &it->second : 0);
|
||||
}
|
||||
|
||||
void erase (const uint64_t& hash)
|
||||
{
|
||||
index.erase (hash);
|
||||
}
|
||||
|
||||
size_t size()
|
||||
{
|
||||
return index.size();
|
||||
}
|
||||
};
|
||||
|
||||
struct COSSOnDiskSegment
|
||||
{
|
||||
uint8_t bytes[XCODEC_SEGMENT_LENGTH];
|
||||
string hexdump() {
|
||||
string dump;
|
||||
char buf[8];
|
||||
int i;
|
||||
for (i = 0; i < 70; i++) {
|
||||
snprintf(buf, 8, "%02x", bytes[i]);
|
||||
dump += buf;
|
||||
}
|
||||
return dump;
|
||||
}
|
||||
};
|
||||
|
||||
struct COSSMetadata
|
||||
{
|
||||
uint32_t signature;
|
||||
uint32_t version;
|
||||
uint64_t serial_number;
|
||||
uint64_t stripe_range;
|
||||
uint32_t segment_index;
|
||||
uint32_t segment_count;
|
||||
uint64_t freshness;
|
||||
uint64_t uses;
|
||||
uint64_t credits;
|
||||
uint32_t load_uses;
|
||||
uint32_t state;
|
||||
};
|
||||
|
||||
struct COSSStripeHeader
|
||||
{
|
||||
COSSMetadata metadata;
|
||||
char padding[METADATA_PADDING];
|
||||
uint32_t flags[STRIPE_SEGMENT_COUNT];
|
||||
uint64_t hash_array[STRIPE_SEGMENT_COUNT];
|
||||
};
|
||||
|
||||
struct COSSStripe
|
||||
{
|
||||
COSSStripeHeader header;
|
||||
COSSOnDiskSegment segment_array[STRIPE_SEGMENT_COUNT];
|
||||
|
||||
public:
|
||||
COSSStripe() { memset (&header, 0, sizeof header); }
|
||||
};
|
||||
|
||||
struct COSSStats
|
||||
{
|
||||
uint64_t lookups;
|
||||
uint64_t found_1;
|
||||
uint64_t found_2;
|
||||
|
||||
public:
|
||||
COSSStats() { lookups = found_1 = found_2 = 0; }
|
||||
};
|
||||
|
||||
|
||||
class XCodecCacheCOSS : public XCodecCache
|
||||
{
|
||||
std::string file_path_;
|
||||
uint64_t file_size_;
|
||||
fstream stream_;
|
||||
|
||||
uint64_t serial_number_;
|
||||
uint64_t stripe_range_;
|
||||
uint64_t stripe_limit_;
|
||||
uint64_t freshness_level_;
|
||||
|
||||
COSSStripe stripe_[LOADED_STRIPE_COUNT];
|
||||
int active_;
|
||||
|
||||
COSSMetadata* directory_;
|
||||
COSSIndex cache_index_;
|
||||
COSSStats stats_;
|
||||
LogHandle log_;
|
||||
|
||||
public:
|
||||
XCodecCacheCOSS (const UUID& uuid, const std::string& cache_dir, size_t cache_size);
|
||||
~XCodecCacheCOSS();
|
||||
|
||||
virtual void enter (const uint64_t& hash, const Buffer& buf, unsigned off);
|
||||
virtual bool lookup (const uint64_t& hash, Buffer& buf);
|
||||
|
||||
private:
|
||||
bool read_file ();
|
||||
void initialize_stripe (uint64_t range, int slot);
|
||||
bool load_stripe (uint64_t range, int slot);
|
||||
void store_stripe (int slot, size_t size);
|
||||
void new_active ();
|
||||
int best_unloadable_slot ();
|
||||
uint64_t best_erasable_stripe ();
|
||||
void detach_stripe (int slot);
|
||||
void purge_stripe (int slot);
|
||||
};
|
||||
|
||||
#endif /* !XCODEC_XCODEC_CACHE_COSS_H */
|
||||
|
Loading…
Add table
Add a link
Reference in a new issue