Commit f03e1dba authored by Sriram Karra's avatar Sriram Karra

Auto migrate user custom json system to config.py based system

The existing config.json system is not very amenable to changes.
We realzed this when the carddav branch made some changes an
introduced v5 of the json. PUshing out major updates like this
can be intrusive to those users who have made customizations to
their copy of the config.json file. To address this issue, The
modified protocol will be as follows:

- Teh system will load the stock version of the latest config json
- the parsed config dictionary will be passed to the user in
  user_dir/config.py which can be any python code. It is expected
  that the user modifies the config idctionary and sends it back

This way we can keep pushing updates to teh config json while
seamlessly retaining the user customizations.

In this patch, the auto migrate will work if the user has not
made any changes to his json. If we identify any changes then
The auto migrate will stop and user will be asked to manually
migrate his changes.
parent 7bc0df8a
......@@ -19,7 +19,7 @@
## not, see <http://www.gnu.org/licenses/>.
import argparse, datetime, logging, os, platform
import netrc, re, shutil, string, sys, traceback
import netrc, re, string, sys, traceback
## First up we need to fix the sys.path before we can even import stuff we
## want... Just some weirdness specific to our code layout...
......@@ -67,34 +67,7 @@ def main (argv=sys.argv):
print 'Creating ASynK User directory at: ', uinps.user_dir
os.makedirs(uinps.user_dir)
# If there is no config file, then let's copy something that makes
# sense...
if not os.path.isfile(os.path.join(uinps.user_dir, 'state.json')):
# Let's first see if there is anything in the asynk source root
# directory - this would be the case with early users of ASynK when
# there was no support for a user-level config dir in ~/.asynk/
if os.path.isfile(os.path.join(ASYNK_BASE_DIR, 'state.json')):
shutil.copy2(os.path.join(ASYNK_BASE_DIR, 'state.json'),
os.path.join(uinps.user_dir, 'state.json'))
print 'We have copied your state.json to new user directory: ',
print uinps.user_dir
print 'We have not copied any of your logs and backup directories.'
else:
## Looks like this is a pretty "clean" run. So just copy the
## state.init file to get things rolling
shutil.copy2(os.path.join(ASYNK_BASE_DIR, 'state.init.json'),
os.path.join(uinps.user_dir, 'state.json'))
# Now copy the config.json file as well.
if not os.path.isfile(os.path.join(uinps.user_dir, 'config.json')):
shutil.copy2(os.path.join(ASYNK_BASE_DIR, 'config.json'),
os.path.join(uinps.user_dir, 'config.json'))
state_filen = os.path.join(uinps.user_dir, 'state.json')
conf_filen = os.path.join(uinps.user_dir, 'config.json')
config = Config(conf_filen, state_filen)
config.set_user_dir(uinps.user_dir)
config = Config(ASYNK_BASE_DIR, uinps.user_dir)
setup_logging(config)
logging.debug('Command line: "%s"', ' '.join(sys.argv))
......
......@@ -27,7 +27,7 @@
## and we are continuing to use the same handling framework...
import iso8601, demjson
import logging, os, re, time
import logging, os, re, shutil, sys, time
sync_dirs = ['SYNC1WAY', 'SYNC2WAY']
......@@ -35,42 +35,62 @@ class AsynkConfigError(Exception):
pass
class Config:
confi_curr_ver = 3
def __init__ (self, confn, staten, sync_through=True):
def __init__ (self, asynk_base_dir, user_dir, sync_through=True):
"""If sync_through is True, any change to the configuration is
immediately written back to the original disk file, otherwise
the user has to explicitly save to disk."""
self.state = { 'state' : {},
'config' : {} }
self.sync_through = sync_through
self.set_app_root(asynk_base_dir)
self.set_user_dir(user_dir)
confi = None
statei = None
self.sync_through = False
self.confi_curr_ver = self._get_latest_config_version()
self.confpy = os.path.abspath(os.path.join(user_dir, 'config.py'))
self.confn = 'config_v%d.json' % self.confi_curr_ver
self.confn = os.path.abspath(os.path.join(asynk_base_dir,
'config', self.confn))
self.staten = os.path.abspath(os.path.join(user_dir, 'state.json'))
self.confn = os.path.abspath(confn)
self.staten = os.path.abspath(staten)
self.sync_through = False
self._setup_state_json()
self._migrate_config_if_reqd(self.confi_curr_ver)
try:
confi = open(confn, "r")
confi = open(self.confn, "r")
except IOError, e:
logging.critical('Error! Could not Open file (%s): %s', confn, e)
logging.critical('Error! Could not Open file (%s): %s', self.confn, e)
raise
try:
statei = open(staten, "r")
statei = open(self.staten, "r")
except IOError, e:
logging.critical('Error! Could not Open file (%s): %s', staten, e)
logging.critical('Error! Could not Open file (%s): %s', self.staten, e)
raise
stc = confi.read()
sts = statei.read()
stc = demjson.decode(confi.read())
sts = demjson.decode(statei.read())
self._customize_config(self.confpy, stc)
# # A sample profile is given in the initial distribution, that should
# # not be written back to the file. Do the needful.
# if 'sample' in sts['profiles']:
# del sts['profiles']['sample']
self.state = { 'state' : demjson.decode(sts),
'config' : demjson.decode(stc), }
self.state = { 'state' : sts,
'config' : stc, }
## FIXME: A bit grotesque that we have to do this again. But let's go
## with this flow for now.
self.set_app_root(asynk_base_dir)
self.set_user_dir(user_dir)
confi.close()
statei.close()
......@@ -82,13 +102,95 @@ class Config:
'to configure. Note that this is optional and your '
'ASynK will continue to work as before.')
self.set_app_root(os.path.abspath(''))
self.sync_through = sync_through
##
## Helper routines
##
def _get_latest_config_version (self):
## FIXME: We need to read the root_dir/config/ directory to list all
## the configuration jsons and pick out the latest file version number
## from that list. For now hard coding this stuff as we need to
## implement the migration stuff at priority.
return 5
def _setup_state_json (self):
user_dir = self.get_user_dir()
base_dir = self.get_app_root()
# If there is no config file, then let's copy something that makes
# sense...
if not os.path.isfile(os.path.join(user_dir, 'state.json')):
# Let's first see if there is anything in the asynk source root
# directory - this would be the case with early users of ASynK when
# there was no support for a user-level config dir in ~/.asynk/
if os.path.isfile(os.path.join(base_dir, 'state.json')):
shutil.copy2(os.path.join(base_dir, 'state.json'),
os.path.join(user_dir, 'state.json'))
print 'We have copied your state.json to new user directory: ',
print user_dir
print 'We have not copied any of your logs and backup directories.'
else:
## Looks like this is a pretty "clean" run. So just copy the
## state.init file to get things rolling
shutil.copy2(os.path.join(base_dir, 'state.init.json'),
os.path.join(user_dir, 'state.json'))
def _migrate_config_if_reqd (self, curr_ver):
user_dir = self.get_user_dir()
base_dir = self.get_app_root()
confpy_init = os.path.join(base_dir, 'config', 'config.init.py')
confpy = os.path.join(user_dir, 'config.py')
confjs = os.path.join(user_dir, 'config.json')
confjs_curr = os.path.join(base_dir, 'config',
'config_v%d.json' % curr_ver)
if not os.path.isfile(confpy):
shutil.copy2(confpy_init, confpy)
if os.path.isfile(confjs):
user_config = open(confjs, 'r').read()
user_ver = demjson.decode(user_config)['file_version']
confjs_curr1 = os.path.join(base_dir, 'config',
'config_v%d.json' % user_ver)
std_config = open(confjs_curr1, 'r').read()
if user_config != std_config:
print
print '*** NOTE: Due to recent changes to the customization system'
print '*** your config needs to be'
print '*** migrated. However as you have modified your'
print '*** configuration, auto migration is not possible'
print '*** and we need your manual intervention.'
print
print '*** You need to do the following steps:'
print
print '*** 1) delete your customization json'
print '*** 2) port your changes to the new config.py file'
print '*** that has been copied to you asynk config dir.'
print
print '*** You can view the comments in config.py for ideas.'
print '*** ASynK will now exit without doing anything more.'
print
sys.exit(0)
else:
os.remove(confjs)
print '*** NOTE: Custom config auto migrated from v%d' % user_ver
def _customize_config (self, confpy, config):
user_dir = self.get_user_dir()
sys.path += [user_dir]
confpy_m = None
try:
confpy_m = __import__('config')
except Exception, e:
print 'Error importing config from %s: %s' % (user_dir, e)
return
confpy_m.customize_config(config)
## Not dependent on sync state between a pair of PIMDs
def _get_prop (self, group, key):
......
##
## This is where you would customize ASynK's configuration settings. You can
## have any python code in here. Ensure you do not break the basic structure
## of the 'config' object which should be a dictionary as defined in the base
## .json configuration files that are distributed with the sources. You can
## view all the json configurations in the directory asynk_app_root/config/
## directory.
##
def customize_config (config):
"""The input 'config' object is a dictionary containing the ASynK
Configuration. This method is invoked immediately after the main
configuration file is read and parsed. You can view the format and
contents of the configuration dictionary in the file ./config_v5.json
which is heavily commented and should be largely self-explanatory.
This method should return the config object after making any
customizations as required."""
## The following line to increase the backup retention duration from the
## 'default' 7 days to 10 days
config['backup_hold_period'] = 10
## Uncomment the following line to change the name of the log directory
## from the default 'logs' to 'logfiles'
# config['log_dir'] = 'logfiles'
This diff is collapsed.
This diff is collapsed.
......@@ -27,17 +27,39 @@ sys.path = EXTRA_PATHS + sys.path
from state import Config, AsynkConfigError
conf_fn = '../../config.json'
state_src = '../../state.init.json'
state_dest = './state.test.json'
user_dir = os.path.abspath('user_dir')
state_src = os.path.join('..', '..', 'state.init.json')
state_dest = os.path.join(user_dir, 'state.json')
shutil.copyfile(state_src, state_dest)
config = Config(confn=conf_fn, staten=state_dest)
confnv4_src = os.path.join('..', '..', 'config', 'config_v4.json')
confnv5_src = os.path.join('..', '..', 'config', 'config_v5.json')
confn_dest = os.path.join(user_dir, 'config.json')
confnv4_src_dirty = os.path.join('.', 'config_v4.dirty.json')
config = None
def run (conf_src):
if os.path.exists(user_dir):
logging.debug('Clearing user directory: %s', user_dir)
shutil.rmtree(user_dir)
else:
logging.debug('Creating user directory: %s', user_dir)
os.makedirs(user_dir)
shutil.copyfile(state_src, state_dest)
shutil.copyfile(conf_src, confn_dest)
global config
config = Config(asynk_base_dir='../../', user_dir=user_dir)
def main (argv=None):
suite = unittest.TestLoader().loadTestsFromTestCase(TestStateFunctions)
unittest.TextTestRunner(verbosity=2).run(suite)
def main ():
run(conf_src=confnv4_src)
run(conf_src=confnv4_src_dirty)
class TestStateFunctions(unittest.TestCase):
## This module is for quick testing of the Config read/write
......@@ -54,7 +76,7 @@ class TestStateFunctions(unittest.TestCase):
def test_get_conf_file_version (self):
val = self.config.get_conf_file_version()
self.assertTrue(val == 4)
self.assertEqual(val, 5)
def test_read_label_prefix (self):
val = self.config.get_label_prefix()
......@@ -97,6 +119,10 @@ class TestStateFunctions(unittest.TestCase):
val = self.config.get_ol_gid_base('gc')
self.assertTrue(val == 0x9001)
def test_read_backup_hold_period (self):
val = self.config.get_backup_hold_period()
self.assertEqual(val, 10)
##
## Now onto the state.json accessors
##
......
This diff is collapsed.
##
## This is where you would customize ASynK's configuration settings. You can
## have any python code in here. Ensure you do not break the basic structure
## of the 'config' object which should be a dictionary as defined in the base
## .json configuration files that are distributed with the sources. You can
## view all the json configurations in the directory asynk_app_root/config/
## directory.
##
def customize_config (config):
"""The input 'config' object is a dictionary containing the ASynK
Configuration. This method is invoked immediately after the main
configuration file is read and parsed. You can view the format and
contents of the configuration dictionary in the file ./config_v5.json
which is heavily commented and should be largely self-explanatory.
This method should return the config object after making any
customizations as required."""
## The following line to increase the backup retention duration from the
## 'default' 7 days to 10 days
config['backup_hold_period'] = 10
## Uncomment the following line to change the name of the log directory
## from the default 'logs' to 'logfiles'
# config['log_dir'] = 'logfiles'
......@@ -44,7 +44,7 @@ def main (argv=None):
conf = Config(os.path.join('..', 'config.json'), 'state.test.json')
user = raw_input('Enter Username:')
pw = raw_input('Password:')
cd = CDPIMDB(conf, 'localhost:8008', user, pw)
cd = CDPIMDB(conf, 'https://localhost:8443', user, pw)
root = '/addressbooks/__uids__/skarrag/addressbook/'
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment