external/tgcloud
1
0
Fork 0
mirror of https://github.com/SlavikMIPT/tgcloud.git synced 2025-02-14 20:22:12 +00:00
Code Issues Releases Wiki Activity

Refactored by Sourcery

Browse source
This commit is contained in:
Sourcery AI 2020-06-30 20:54:14 +00:00
parent 15a44135e9
commit f2e38ff496
4 changed files with 72 additions and 71 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

102
dedupfs/dedupfs.py
View file

@ -1025,25 +1025,26 @@ class DedupFS(fuse.Fuse): # {{{1
self.__report_timings() self.__report_timings()
def __report_timings(self): # {{{3 def __report_timings(self): # {{{3
if self.logger.isEnabledFor(logging.DEBUG): if not self.logger.isEnabledFor(logging.DEBUG):
timings = [(self.time_spent_traversing_tree, 'Traversing the tree'), return
(self.time_spent_caching_nodes, 'Caching tree nodes'), timings = [(self.time_spent_traversing_tree, 'Traversing the tree'),
(self.time_spent_interning, 'Interning path components'), (self.time_spent_caching_nodes, 'Caching tree nodes'),
(self.time_spent_writing_blocks, 'Writing data blocks'), (self.time_spent_interning, 'Interning path components'),
(self.time_spent_hashing, 'Hashing data blocks'), (self.time_spent_writing_blocks, 'Writing data blocks'),
(self.time_spent_querying_tree, 'Querying the tree')] (self.time_spent_hashing, 'Hashing data blocks'),
maxdescwidth = max([len(l) for t, l in timings]) + 3 (self.time_spent_querying_tree, 'Querying the tree')]
timings.sort(reverse=True) maxdescwidth = max(len(l) for t, l in timings) + 3
uptime = time.time() - self.fs_mounted_at timings.sort(reverse=True)
printed_heading = False uptime = time.time() - self.fs_mounted_at
for timespan, description in timings: printed_heading = False
percentage = timespan / (uptime / 100) for timespan, description in timings:
if percentage >= 1: percentage = timespan / (uptime / 100)
if not printed_heading: if percentage >= 1:
self.logger.debug("Cumulative timings of slowest operations:") if not printed_heading:
printed_heading = True self.logger.debug("Cumulative timings of slowest operations:")
self.logger.debug( printed_heading = True
" - %-*s%s (%i%%)" % (maxdescwidth, description + ':', format_timespan(timespan), percentage)) self.logger.debug(
" - %-*s%s (%i%%)" % (maxdescwidth, description + ':', format_timespan(timespan), percentage))
def report_disk_usage(self): # {{{3 def report_disk_usage(self): # {{{3
disk_usage = self.__fetchval('PRAGMA page_size') * self.__fetchval('PRAGMA page_count') disk_usage = self.__fetchval('PRAGMA page_size') * self.__fetchval('PRAGMA page_count')
@ -1063,7 +1064,7 @@ class DedupFS(fuse.Fuse): # {{{1
self.memory_usage = memory_usage self.memory_usage = memory_usage
def __report_throughput(self, nbytes=None, nseconds=None, label=None): # {{{3 def __report_throughput(self, nbytes=None, nseconds=None, label=None): # {{{3
if nbytes == None: if nbytes is None:
self.bytes_read, self.time_spent_reading = \ self.bytes_read, self.time_spent_reading = \
self.__report_throughput(self.bytes_read, self.time_spent_reading, "read") self.__report_throughput(self.bytes_read, self.time_spent_reading, "read")
self.bytes_written, self.time_spent_writing = \ self.bytes_written, self.time_spent_writing = \
@ -1078,6 +1079,9 @@ class DedupFS(fuse.Fuse): # {{{1
return nbytes, nseconds return nbytes, nseconds
def __report_top_blocks(self): # {{{3 def __report_top_blocks(self): # {{{3
if not self.logger.isEnabledFor(logging.DEBUG):
return
printed_header = False
query = """ query = """
SELECT * FROM ( SELECT * FROM (
SELECT *, COUNT(*) AS "count" FROM "index" SELECT *, COUNT(*) AS "count" FROM "index"
@ -1086,21 +1090,19 @@ class DedupFS(fuse.Fuse): # {{{1
"count" > 1 AND "count" > 1 AND
hash_id = hashes.id hash_id = hashes.id
LIMIT 10 """ LIMIT 10 """
if self.logger.isEnabledFor(logging.DEBUG): for row in self.conn.execute(query):
printed_header = False if not printed_header:
for row in self.conn.execute(query): self.logger.debug("A listing of the most used blocks follows:")
if not printed_header: printed_header = True
self.logger.debug("A listing of the most used blocks follows:") msg = "Block #%s of %s has been used %i times: %r"
printed_header = True preview = row['value']
msg = "Block #%s of %s has been used %i times: %r" max_length = 60
preview = row['value'] if len(preview) < max_length:
max_length = 60 preview = str(preview)
if len(preview) < max_length: else:
preview = str(preview) preview = preview[0: max_length] + '...'
else: nbytes = format_size(len(row['value']))
preview = preview[0: max_length] + '...' self.logger.debug(msg, row['hash_id'], nbytes, row['count'], preview)
nbytes = format_size(len(row['value']))
self.logger.debug(msg, row['hash_id'], nbytes, row['count'], preview)
def __gc_hook(self, nested=False): # {{{3 def __gc_hook(self, nested=False): # {{{3
# Don't collect any garbage for nested calls. # Don't collect any garbage for nested calls.
@ -1116,19 +1118,21 @@ class DedupFS(fuse.Fuse): # {{{1
self.gc_hook_last_run = time.time() self.gc_hook_last_run = time.time()
def __collect_garbage(self): # {{{3 def __collect_garbage(self): # {{{3
if self.gc_enabled and not self.read_only: if not self.gc_enabled or self.read_only:
start_time = time.time() return
self.logger.info("Performing garbage collection (this might take a while) ..")
self.should_vacuum = False start_time = time.time()
for method in self.__collect_strings, self.__collect_inodes, \ self.logger.info("Performing garbage collection (this might take a while) ..")
self.__collect_indices, self.__collect_blocks, self.__vacuum_metastore: self.should_vacuum = False
sub_start_time = time.time() for method in self.__collect_strings, self.__collect_inodes, \
msg = method() self.__collect_indices, self.__collect_blocks, self.__vacuum_metastore:
if msg: sub_start_time = time.time()
elapsed_time = time.time() - sub_start_time msg = method()
self.logger.info(msg, format_timespan(elapsed_time)) if msg:
elapsed_time = time.time() - start_time elapsed_time = time.time() - sub_start_time
self.logger.info("Finished garbage collection in %s.", format_timespan(elapsed_time)) self.logger.info(msg, format_timespan(elapsed_time))
elapsed_time = time.time() - start_time
self.logger.info("Finished garbage collection in %s.", format_timespan(elapsed_time))
def __collect_strings(self): # {{{4 def __collect_strings(self): # {{{4
count = self.conn.execute('DELETE FROM strings WHERE id NOT IN (SELECT name FROM tree)').rowcount count = self.conn.execute('DELETE FROM strings WHERE id NOT IN (SELECT name FROM tree)').rowcount
@ -1150,7 +1154,7 @@ class DedupFS(fuse.Fuse): # {{{1
def __collect_blocks(self): # {{{4 def __collect_blocks(self): # {{{4
should_reorganize = False should_reorganize = False
for row in self.conn.execute('SELECT hash FROM hashes WHERE id NOT IN (SELECT hash_id FROM "index")'): for _ in self.conn.execute('SELECT hash FROM hashes WHERE id NOT IN (SELECT hash_id FROM "index")'):
# del self.blocks[str(row[0])] # del self.blocks[str(row[0])]
should_reorganize = True should_reorganize = True
if should_reorganize: if should_reorganize:

31
dedupfs/fuse.py
View file

@ -73,8 +73,12 @@ def get_compat_0_1():
# API version to be used # API version to be used
fuse_python_api = __getenv__('FUSE_PYTHON_API', '^[\d.]+$', fuse_python_api = __getenv__(
lambda x: tuple([int(i) for i in x.split('.')])) 'FUSE_PYTHON_API',
'^[\d.]+$',
lambda x: tuple(int(i) for i in x.split('.')),
)
# deprecated way of API specification # deprecated way of API specification
compat_0_1 = __getenv__('FUSE_PYTHON_COMPAT', '^(0.1|ALL)$', lambda x: True) compat_0_1 = __getenv__('FUSE_PYTHON_COMPAT', '^(0.1|ALL)$', lambda x: True)
@ -126,9 +130,7 @@ class FuseArgs(SubOptsHive):
def mount_expected(self): def mount_expected(self):
if self.getmod('showhelp'): if self.getmod('showhelp'):
return False return False
if self.getmod('showversion'): return not self.getmod('showversion')
return False
return True
def assemble(self): def assemble(self):
"""Mangle self into an argument array""" """Mangle self into an argument array"""
@ -141,9 +143,7 @@ class FuseArgs(SubOptsHive):
if v: if v:
args.append(self.fuse_modifiers[m]) args.append(self.fuse_modifiers[m])
opta = [] opta = [o + '=' + v for o, v in self.optdict.iteritems()]
for o, v in self.optdict.iteritems():
opta.append(o + '=' + v)
opta.extend(self.optlist) opta.extend(self.optlist)
if opta: if opta:
@ -167,7 +167,7 @@ class FuseArgs(SubOptsHive):
class FuseFormatter(SubbedOptIndentedFormatter): class FuseFormatter(SubbedOptIndentedFormatter):
def __init__(self, **kw): def __init__(self, **kw):
if not 'indent_increment' in kw: if 'indent_increment' not in kw:
kw['indent_increment'] = 4 kw['indent_increment'] = 4
SubbedOptIndentedFormatter.__init__(self, **kw) SubbedOptIndentedFormatter.__init__(self, **kw)
@ -338,7 +338,7 @@ class FuseOptParse(SubbedOptParse):
"having options or specifying the `subopt' attribute conflicts with `mountopt' attribute") "having options or specifying the `subopt' attribute conflicts with `mountopt' attribute")
opts = ('-o',) opts = ('-o',)
attrs['subopt'] = attrs.pop('mountopt') attrs['subopt'] = attrs.pop('mountopt')
if not 'dest' in attrs: if 'dest' not in attrs:
attrs['dest'] = attrs['subopt'] attrs['dest'] = attrs['subopt']
SubbedOptParse.add_option(self, *opts, **attrs) SubbedOptParse.add_option(self, *opts, **attrs)
@ -559,8 +559,7 @@ def feature_needs(*feas):
maxva[0] = max(maxva[0], fp) maxva[0] = max(maxva[0], fp)
continue continue
if isinstance(fp, list) or isinstance(fp, tuple): if isinstance(fp, list) or isinstance(fp, tuple):
for f in fp: yield from fp
yield f
continue continue
ma = isinstance(fp, str) and re.compile("(!\s*|)re:(.*)").match(fp) ma = isinstance(fp, str) and re.compile("(!\s*|)re:(.*)").match(fp)
if isinstance(fp, type(re.compile(''))) or ma: if isinstance(fp, type(re.compile(''))) or ma:
@ -574,7 +573,7 @@ def feature_needs(*feas):
yield f yield f
continue continue
ma = re.compile("has_(.*)").match(fp) ma = re.compile("has_(.*)").match(fp)
if ma and ma.groups()[0] in Fuse._attrs and not fp in fmap: if ma and ma.groups()[0] in Fuse._attrs and fp not in fmap:
yield 21 yield 21
continue continue
yield fmap[fp] yield fmap[fp]
@ -728,8 +727,10 @@ class Fuse(object):
if get_compat_0_1(): if get_compat_0_1():
args = self.main_0_1_preamble() args = self.main_0_1_preamble()
d = {'multithreaded': self.multithreaded and 1 or 0} d = {
d['fuse_args'] = args or self.fuse_args.assemble() 'multithreaded': self.multithreaded and 1 or 0,
'fuse_args': args or self.fuse_args.assemble(),
}
for t in 'file_class', 'dir_class': for t in 'file_class', 'dir_class':
if hasattr(self, t): if hasattr(self, t):

4
download_service.py
View file

@ -53,8 +53,8 @@ def download_block(hash_uid, filename):
entity = client.get_entity(client.get_me()) entity = client.get_entity(client.get_me())
messages = client.get_messages(entity, limit=1, search=hash_uid) messages = client.get_messages(entity, limit=1, search=hash_uid)
for i in range(len(messages)): for message in messages:
msg = messages[i] msg = message
if msg.message == hash_uid: if msg.message == hash_uid:
# FIFO = f"dpipe_{hash_uid}" # FIFO = f"dpipe_{hash_uid}"
# import errno # import errno

6
telegram_client_x.py
View file

@ -187,11 +187,7 @@ class TelegramClientX(TelegramClient):
# Set a default file name if None was specified # Set a default file name if None was specified
file_id = helpers.generate_random_long() file_id = helpers.generate_random_long()
if not file_name: if not file_name:
if isinstance(file, str): file_name = os.path.basename(file) if isinstance(file, str) else str(file_id)
file_name = os.path.basename(file)
else:
file_name = str(file_id)
# Determine whether the file is too big (over 10MB) or not # Determine whether the file is too big (over 10MB) or not
# Telegram does make a distinction between smaller or larger files # Telegram does make a distinction between smaller or larger files
is_large = file_size > 10 * 1024 * 1024 is_large = file_size > 10 * 1024 * 1024