"""This module gives the mkfs creation options for an existing filesystem.
tune2fs or xfs_growfs is called according to the filesystem. The results,
filesystem tunables, are parsed and mapped to corresponding mkfs options.
"""
import os
import re
import tempfile
try:
import autotest.common as common # pylint: disable=W0611
except ImportError:
import common # pylint: disable=W0611
from autotest.client.shared import error, utils
[docs]def opt_string2dict(opt_string):
"""Breaks the mkfs.ext* option string into dictionary."""
# Example string: '-j -q -i 8192 -b 4096'. There may be extra whitespaces.
opt_dict = {}
for item in opt_string.split('-'):
item = item.strip()
if ' ' in item:
(opt, value) = item.split(' ', 1)
opt_dict['-%s' % opt] = value
elif item != '':
opt_dict['-%s' % item] = None
# Convert all the digit strings to int.
for key, value in opt_dict.iteritems():
if value and value.isdigit():
opt_dict[key] = int(value)
return opt_dict
[docs]def parse_mke2fs_conf(fs_type, conf_file='/etc/mke2fs.conf'):
"""Parses mke2fs config file for default settings."""
# Please see /etc/mke2fs.conf for an example.
default_opt = {}
fs_opt = {}
current_fs_type = ''
current_section = ''
f = open(conf_file, 'r')
for line in f:
if '[defaults]' == line.strip():
current_section = '[defaults]'
elif '[fs_types]' == line.strip():
current_section = '[fs_types]'
elif current_section == '[defaults]':
components = line.split('=', 1)
if len(components) == 2:
default_opt[components[0].strip()] = components[1].strip()
elif current_section == '[fs_types]':
m = re.search('(\w+) = {', line)
if m:
current_fs_type = m.group(1)
else:
components = line.split('=', 1)
if len(components) == 2 and current_fs_type == fs_type:
default_opt[components[0].strip()] = components[1].strip()
f.close()
# fs_types options override the defaults options
for key, value in fs_opt.iteritems():
default_opt[key] = value
# Convert all the digit strings to int.
for key, value in default_opt.iteritems():
if value and value.isdigit():
default_opt[key] = int(value)
return default_opt
[docs]def convert_conf_opt(default_opt):
conf_opt_mapping = {'blocksize': '-b',
'inode_ratio': '-i',
'inode_size': '-I'}
mkfs_opt = {}
# Here we simply concatenate the feature string while we really need
# to do the better and/or operations.
if 'base_features' in default_opt:
mkfs_opt['-O'] = default_opt['base_features']
if 'default_features' in default_opt:
mkfs_opt['-O'] += ',%s' % default_opt['default_features']
if 'features' in default_opt:
mkfs_opt['-O'] += ',%s' % default_opt['features']
for key, value in conf_opt_mapping.iteritems():
if key in default_opt:
mkfs_opt[value] = default_opt[key]
if '-O' in mkfs_opt:
mkfs_opt['-O'] = mkfs_opt['-O'].split(',')
return mkfs_opt
[docs]def merge_ext_features(conf_feature, user_feature):
user_feature_list = user_feature.split(',')
merged_feature = []
# Removes duplicate entries in conf_list.
for item in conf_feature:
if item not in merged_feature:
merged_feature.append(item)
# User options override config options.
for item in user_feature_list:
if item[0] == '^':
if item[1:] in merged_feature:
merged_feature.remove(item[1:])
else:
merged_feature.append(item)
elif item not in merged_feature:
merged_feature.append(item)
return merged_feature
[docs]def ext_tunables(dev):
"""Call tune2fs -l and parse the result."""
cmd = 'tune2fs -l %s' % dev
try:
out = utils.system_output(cmd)
except error.CmdError:
tools_dir = os.path.join(os.environ['AUTODIR'], 'tools')
cmd = '%s/tune2fs.ext4dev -l %s' % (tools_dir, dev)
out = utils.system_output(cmd)
# Load option mappings
tune2fs_dict = {}
for line in out.splitlines():
components = line.split(':', 1)
if len(components) == 2:
value = components[1].strip()
option = components[0]
if value.isdigit():
tune2fs_dict[option] = int(value)
else:
tune2fs_dict[option] = value
return tune2fs_dict
[docs]def ext_mkfs_options(tune2fs_dict, mkfs_option):
"""Map the tune2fs options to mkfs options."""
def __inode_count(tune_dict, k):
return (tune_dict['Block count'] / tune_dict[k] + 1) * (
tune_dict['Block size'])
def __block_count(tune_dict, k):
return int(100 * tune_dict[k] / tune_dict['Block count'] + 1)
def __volume_name(tune_dict, k):
if tune_dict[k] != '<none>':
return tune_dict[k]
else:
return ''
# mappings between fs features and mkfs options
ext_mapping = {'Blocks per group': '-g',
'Block size': '-b',
'Filesystem features': '-O',
'Filesystem OS type': '-o',
'Filesystem revision #': '-r',
'Filesystem volume name': '-L',
'Flex block group size': '-G',
'Fragment size': '-f',
'Inode count': '-i',
'Inode size': '-I',
'Journal inode': '-j',
'Reserved block count': '-m'}
conversions = {
'Journal inode': lambda d, k: None,
'Filesystem volume name': __volume_name,
'Reserved block count': __block_count,
'Inode count': __inode_count,
'Filesystem features': lambda d, k: re.sub(' ', ',', d[k]),
'Filesystem revision #': lambda d, k: d[k][0]}
for key, value in ext_mapping.iteritems():
if key not in tune2fs_dict:
continue
if key in conversions:
mkfs_option[value] = conversions[key](tune2fs_dict, key)
else:
mkfs_option[value] = tune2fs_dict[key]
[docs]def xfs_tunables(dev):
"""Call xfs_grow -n to get filesystem tunables."""
# Have to mount the filesystem to call xfs_grow.
tmp_mount_dir = tempfile.mkdtemp()
cmd = 'mount %s %s' % (dev, tmp_mount_dir)
utils.system_output(cmd)
xfs_growfs = os.path.join(os.environ['AUTODIR'], 'tools', 'xfs_growfs')
cmd = '%s -n %s' % (xfs_growfs, dev)
try:
out = utils.system_output(cmd)
finally:
# Clean.
cmd = 'umount %s' % dev
utils.system_output(cmd, ignore_status=True)
os.rmdir(tmp_mount_dir)
# The output format is given in report_info (xfs_growfs.c)
# "meta-data=%-22s isize=%-6u agcount=%u, agsize=%u blks\n"
# " =%-22s sectsz=%-5u attr=%u\n"
# "data =%-22s bsize=%-6u blocks=%llu, imaxpct=%u\n"
# " =%-22s sunit=%-6u swidth=%u blks\n"
# "naming =version %-14u bsize=%-6u\n"
# "log =%-22s bsize=%-6u blocks=%u, version=%u\n"
# " =%-22s sectsz=%-5u sunit=%u blks, lazy-count=%u\n"
# "realtime =%-22s extsz=%-6u blocks=%llu, rtextents=%llu\n"
tune2fs_dict = {}
# Flag for extracting naming version number
keep_version = False
for line in out.splitlines():
m = re.search('^([-\w]+)', line)
if m:
main_tag = m.group(1)
pairs = line.split()
for pair in pairs:
# naming: version needs special treatment
if pair == '=version':
# 1 means the next pair is the version number we want
keep_version = True
continue
if keep_version:
tune2fs_dict['naming: version'] = pair
# Resets the flag since we have logged the version
keep_version = False
continue
# Ignores the strings without '=', such as 'blks'
if '=' not in pair:
continue
key, value = pair.split('=')
tagged_key = '%s: %s' % (main_tag, key)
if re.match('[0-9]+', value):
tune2fs_dict[tagged_key] = int(value.rstrip(','))
else:
tune2fs_dict[tagged_key] = value.rstrip(',')
return tune2fs_dict
[docs]def xfs_mkfs_options(tune2fs_dict, mkfs_option):
"""Maps filesystem tunables to their corresponding mkfs options."""
# Mappings
xfs_mapping = {'meta-data: isize': '-i size',
'meta-data: agcount': '-d agcount',
'meta-data: sectsz': '-s size',
'meta-data: attr': '-i attr',
'data: bsize': '-b size',
'data: imaxpct': '-i maxpct',
'data: sunit': '-d sunit',
'data: swidth': '-d swidth',
'data: unwritten': '-d unwritten',
'naming: version': '-n version',
'naming: bsize': '-n size',
'log: version': '-l version',
'log: sectsz': '-l sectsize',
'log: sunit': '-l sunit',
'log: lazy-count': '-l lazy-count',
'realtime: extsz': '-r extsize',
'realtime: blocks': '-r size',
'realtime: rtextents': '-r rtdev'}
mkfs_option['-l size'] = tune2fs_dict['log: bsize'] * (
tune2fs_dict['log: blocks'])
for key, value in xfs_mapping.iteritems():
mkfs_option[value] = tune2fs_dict[key]
[docs]def compare_features(needed_feature, current_feature):
"""Compare two ext* feature lists."""
if len(needed_feature) != len(current_feature):
return False
for feature in current_feature:
if feature not in needed_feature:
return False
return True
[docs]def match_ext_options(fs_type, dev, needed_options):
"""Compare the current ext* filesystem tunables with needed ones."""
# mkfs.ext* will load default options from /etc/mke2fs.conf
conf_opt = parse_mke2fs_conf(fs_type)
# We need to convert the conf options to mkfs options.
conf_mkfs_opt = convert_conf_opt(conf_opt)
# Breaks user mkfs option string to dictionary.
needed_opt_dict = opt_string2dict(needed_options)
# Removes ignored options.
ignored_option = ['-c', '-q', '-E', '-F']
for opt in ignored_option:
if opt in needed_opt_dict:
del needed_opt_dict[opt]
# User options override config options.
needed_opt = conf_mkfs_opt
for key, value in needed_opt_dict.iteritems():
if key == '-N' or key == '-T':
raise Exception('-N/T is not allowed.')
elif key == '-O':
needed_opt[key] = merge_ext_features(needed_opt[key], value)
else:
needed_opt[key] = value
# '-j' option will add 'has_journal' feature.
if '-j' in needed_opt and 'has_journal' not in needed_opt['-O']:
needed_opt['-O'].append('has_journal')
# 'extents' will be shown as 'extent' in the outcome of tune2fs
if 'extents' in needed_opt['-O']:
needed_opt['-O'].append('extent')
needed_opt['-O'].remove('extents')
# large_file is a byproduct of resize_inode.
if 'large_file' not in needed_opt['-O'] and (
'resize_inode' in needed_opt['-O']):
needed_opt['-O'].append('large_file')
current_opt = {}
tune2fs_dict = ext_tunables(dev)
ext_mkfs_options(tune2fs_dict, current_opt)
# Does the match
for key, value in needed_opt.iteritems():
if key == '-O':
if not compare_features(value, current_opt[key].split(',')):
return False
elif key not in current_opt or value != current_opt[key]:
return False
return True
[docs]def match_xfs_options(dev, needed_options):
"""Compare the current ext* filesystem tunables with needed ones."""
tmp_mount_dir = tempfile.mkdtemp()
cmd = 'mount %s %s' % (dev, tmp_mount_dir)
utils.system_output(cmd)
xfs_growfs = os.path.join(os.environ['AUTODIR'], 'tools', 'xfs_growfs')
cmd = '%s -n %s' % (xfs_growfs, dev)
try:
current_option = utils.system_output(cmd)
finally:
# Clean.
cmd = 'umount %s' % dev
utils.system_output(cmd, ignore_status=True)
os.rmdir(tmp_mount_dir)
# '-N' has the same effect as '-n' in mkfs.ext*. Man mkfs.xfs for details.
cmd = 'mkfs.xfs %s -N -f %s' % (needed_options, dev)
needed_out = utils.system_output(cmd)
# 'mkfs.xfs -N' produces slightly different result than 'xfs_growfs -n'
needed_out = re.sub('internal log', 'internal ', needed_out)
if current_option == needed_out:
return True
else:
return False
[docs]def match_mkfs_option(fs_type, dev, needed_options):
"""Compare the current filesystem tunables with needed ones."""
if fs_type.startswith('ext'):
ret = match_ext_options(fs_type, dev, needed_options)
elif fs_type == 'xfs':
ret = match_xfs_options(dev, needed_options)
else:
ret = False
return ret