"""These XML-RPC functions are called by the front method
:meth:`~.lib.build.front.FrontBuildManager.call`"""
from os import listdir, remove, walk, makedirs, rename
from os.path import exists, join, dirname, relpath, normpath
from shutil import rmtree
from cStringIO import StringIO
from random import randint
import xmlrpclib
from ..lib.i18n import _, localizer
from ..lib.log import log_activity
from ..lib.utils import camel_case
from ..lib.rsync import SigFile, PatchedFile, DeltaFile, get_block_size
# =============================================================================
[docs]def agent_id(request, context):
"""Return the unique ID of an agent.
:param request: (:class:`pyramid.request.Request` instance)
Current request.
:param context: (dictionary)
See :meth:`~.lib.build.front.FrontBuildManager.call`.
:return: (tuple)
``(error, uid)``
"""
if not _authorized(request, context):
return _not_authorized(context)
return '', request.registry['abuild'].agent_id()
# =============================================================================
[docs]def processor_list(request, context):
"""Return a list of available processors.
:param request: (:class:`pyramid.request.Request` instance)
Current request.
:param context: (dictionary)
See :meth:`~.lib.build.front.FrontBuildManager.call`.
:return: (tuple)
``(error, processor_list)``
"""
if not _authorized(request, context):
return _not_authorized(context)
return '', request.registry['abuild'].processor_list()
# =============================================================================
[docs]def processor_xml(request, context, processor_id):
"""Return XML of processor ``processor_id``.
:param request: (:class:`pyramid.request.Request` instance)
Current request.
:param context: (dictionary)
See :meth:`~.lib.build.front.FrontBuildManager.call`.
:param processor_id: (string)
Processor_id
:return: (tuple)
``(<error>, <processor_list>)``
"""
if not _authorized(request, context):
return _not_authorized(context)
return '', request.registry['abuild'].processor_xml(processor_id)
# =============================================================================
[docs]def activity(request, context):
"""Return the agent activity i.e. the number of active builds.
:param request: (:class:`pyramid.request.Request` instance)
Current request.
:param context: (dictionary)
See :meth:`~.lib.build.front.FrontBuildManager.call`.
:param password: (string)
Front password to use agent services.
:return: (tuple)
``(<error>, <activity>)``
"""
if not _authorized(request, context):
return _not_authorized(context)
return '', request.registry['abuild'].activity()
# =============================================================================
[docs]def synchronizing(request, context, build_id, lock=True):
"""The data synchronization for the build ``build_id`` is running.
:param request: (:class:`pyramid.request.Request` instance)
Current request.
:param context: (dictionary)
See :meth:`~.lib.build.front.FrontBuildManager.call`.
:param build_id: (string)
Build ID.
:param lock: (boolean, default=True)
If ``True`` try to add build ``build_id`` in the list of
synchronizations.
:return: (tuple)
``(<error>, '')``
"""
if not _authorized(request, context):
return _not_authorized(context)
build_id = _agent_build_id(request, context, build_id)
if not request.registry['abuild'].synchronizing(build_id, lock):
return _error(context, _('Too many activities!'))
return '', ''
# =============================================================================
[docs]def start(request, context, build_id, processing, pack, end_url):
"""Start the build.
:param request: (:class:`pyramid.request.Request` instance)
Current request.
:param context: (dictionary)
See :meth:`~.lib.build.front.FrontBuildManager.call`.
:param build_id: (string)
Build ID.
:param processing: (dictionary)
A processing dictionary.
:param pack: (dictionary)
A pack dictionary.
:param end_url: (string)
URL to call to complete the build.
:return: (tuple)
``(<error>, '')``
Arguments of this RPC function are: ``<front_id>, <password>, <user>,
<build_id>, <processing>, <pack>, <end_url>``.
"""
if not _authorized(request, context):
return _not_authorized(context)
# Adjust paths
if not context['local']:
processing['resources'] = _agent_paths(
request, context, *processing['resources'])
for item in processing['templates']:
item[0] = _agent_paths(request, context, item[0])[0]
del pack['project_id']
del pack['pack_id']
pack['files'] = _agent_paths(request, context, *pack['files'])
pack['resources'] = _agent_paths(request, context, *pack['resources'])
for item in pack['templates']:
item[0] = _agent_paths(request, context, item[0])[0]
# Start build
build_id = _agent_build_id(request, context, build_id)
if not request.registry['abuild'].start_build(
build_id, context, processing, pack, end_url):
return _error(context, _('Too many activities!'))
return '', ''
# =============================================================================
[docs]def progress(request, context, build_id):
"""Return the progress of a build.
:param request: (:class:`pyramid.request.Request` instance)
Current request.
:param context: (dictionary)
See :meth:`~.lib.build.front.FrontBuildManager.call`.
:param build_id: (string)
Build ID.
:return: (tuple)
``(<error>, <result>)``
"""
return _agent_method(request, context, build_id, 'progress')
# =============================================================================
[docs]def stop(request, context, build_id):
"""Stop a build.
:param request: (:class:`pyramid.request.Request` instance)
Current request.
:param context: (dictionary)
See :meth:`~.lib.build.front.FrontBuildManager.call`.
:param build_id: (string)
Build ID.
:return: (tuple)
``(<error>, <result>)``
"""
return _agent_method(request, context, build_id, 'stop')
# =============================================================================
[docs]def result(request, context, build_id):
"""Return the result of a build.
:param request: (:class:`pyramid.request.Request` instance)
Current request.
:param context: (dictionary)
See :meth:`~.lib.build.front.FrontBuildManager.call`.
:param build_id: (string)
Build ID.
"""
if not _authorized(request, context):
return _not_authorized(context)
build_id = _agent_build_id(request, context, build_id)
_result = request.registry['abuild'].result(build_id)
request.registry['context'] = context
status = u'a_end'
if _result['status'] != 'a_end':
status = u'{0} "{1}"'.format(
_result['status'], _result['message'][:64])
else:
errors = [k[3] for k in _result['log'] if k[1] == 'a_error']
if errors:
status = u'a_error "{0}"'.format(errors[0][:64])
log_activity(request, 'build_result', build_id, status)
return '', _result
# =============================================================================
[docs]def buildspace_cleanup(request, context, root, filenames):
"""Delete files in path ``path`` of buildspace directory which are not in
``files``.
:param request: (:class:`pyramid.request.Request` instance)
Current request.
:param context: (dictionary)
See :meth:`~.lib.build.front.FrontBuildManager.call`.
:param root: (string)
Path to clean up.
:param filenames: (list)
Names of file to keep.
:return: (tuple)
``(<error>, '')``
"""
if not _authorized(request, context):
return _not_authorized(context)
# Clean directory
root = join(
request.registry.settings['buildspace.root'],
_agent_paths(request, context, root)[0])
if not exists(root):
return '', ''
for path, dirs, files in walk(root, topdown=False):
for name in files:
if normpath(join(relpath(path, root), name).decode('utf8')) \
not in filenames and exists(join(path, name)):
remove(join(path, name))
for name in dirs:
if not listdir(join(path, name)):
rmtree(join(path, name))
return '', ''
# =============================================================================
[docs]def buildspace_send_signature(request, context, filename):
"""Return signature of a buildspace file.
:param request: (:class:`pyramid.request.Request` instance)
Current request.
:param context: (dictionary)
See :meth:`~.lib.build.front.FrontBuildManager.call`.
:param filename: (string)
Filename that should send the signature.
:return: (tuple)
``(<error>, <signature>)``
"""
if not _authorized(request, context):
return _not_authorized(context)
# Empty string or signature
filename = join(request.registry.settings['buildspace.root'],
_agent_paths(request, context, filename)[0])
if not exists(filename):
if not exists(dirname(filename)):
makedirs(dirname(filename))
with open(filename, 'wb') as hdl:
hdl.write('')
sig_file = SigFile(open(filename, 'rb'), get_block_size(filename))
sig_buf = sig_file.read()
sig_file.close()
return '', xmlrpclib.Binary(sig_buf)
# =============================================================================
[docs]def buildspace_receive_delta(request, context, filename, delta):
"""Receive a delta (rsync) for a file in buildspace directory.
:param request: (:class:`pyramid.request.Request` instance)
Current request.
:param context: (dictionary)
See :meth:`~.lib.build.front.FrontBuildManager.call`.
:param filename: (string)
Filename that should send the signature.
:param delta: (:class:`xmlrpclib.Binary` instance)
Patch to apply.
:return: (tuple)
``(<error>, '')``
"""
if not _authorized(request, context):
return _not_authorized(context)
# Patch file
filename = join(request.registry.settings['buildspace.root'],
_agent_paths(request, context, filename)[0])
if not exists(filename):
return _error(context, _('Unknown file "${n}"', {'n': filename}))
patch_file = PatchedFile(open(filename, 'rb'), StringIO(delta.data))
temp_name = '%s~%d~' % (filename, randint(1, 999999))
with open(temp_name, 'wb') as hdl:
hdl.write(patch_file.read())
patch_file.close()
if exists(temp_name):
rename(temp_name, filename)
return '', ''
# =============================================================================
[docs]def output_list(request, context, build_id):
"""List output files of a build.
:param request: (:class:`pyramid.request.Request` instance)
Current request.
:param context: (dictionary)
See :meth:`~.lib.build.front.FrontBuildManager.call`.
:param build_id: (string)
Build ID.
:return: (tuple)
``(<error>, <file_list>)``
"""
if not _authorized(request, context):
return _not_authorized(context)
# List files
build_id = _agent_build_id(request, context, build_id)
output = join(request.registry.settings['build.root'],
camel_case(build_id), 'Output')
file_list = []
for path, ignored_, files in walk(output):
for name in files:
file_list.append(relpath(join(path, name), output))
return '', file_list
# =============================================================================
[docs]def output_send_delta(request, context, build_id, filename, sig):
"""Send a delta list of a file of the ``Output`` directory..
:param request: (:class:`pyramid.request.Request` instance)
Current request.
:param context: (dictionary)
See :meth:`~.lib.build.front.FrontBuildManager.call`.
:param build_id: (string)
Build ID.
:param filename: (string)
Name of the file to transfer.
:param sig: (:class:`xmlrpclib.Binary` instance)
File signature.
:return: (tuple)
``(<error>, <delta>)``
"""
if not _authorized(request, context):
return _not_authorized(context)
# Send delta
build_id = _agent_build_id(request, context, build_id)
filename = join(request.registry.settings['build.root'],
camel_case(build_id), 'Output', filename)
with open(filename, 'rb') as hdl:
delta_file = DeltaFile(sig.data, hdl)
delta_buf = delta_file.read()
delta_file.close()
return '', xmlrpclib.Binary(delta_buf)
# =============================================================================
def _authorized(request, context):
"""Check if the request is authorized.
:param request: (:class:`pyramid.request.Request` instance)
Current request.
:param context: (dictionary)
See :meth:`~.lib.build.front.FrontBuildManager.call`.
:return: (boolean)
``True`` if authorized.
"""
return request.registry['abuild'].authorized_front(
context['front_id'], context['password'])
# =============================================================================
def _not_authorized(context):
"""Return ``('Not authozied!', '')`` where error message is translated.
:param context: (dictionary)
See :meth:`~.lib.build.front.FrontBuildManager.call`.
:return: (tuple)
"""
return _error(context, _('Not authorized!'))
# =============================================================================
def _error(context, text):
"""Return ``(text, '')`` where error message ``text`` is translated.
:param context: (dictionary)
See :meth:`~.lib.build.front.FrontBuildManager.call`.
:param text: (string)
Error message to translate.
:return: (tuple)
"""
return localizer(context['lang']).translate(text), ''
# =============================================================================
def _agent_build_id(request, context, build_id):
"""Prefix ``build_id`` with ``front_id`` if agent is not called locally.
:param request: (:class:`pyramid.request.Request` instance)
Current request.
:param context: (dictionary)
See :meth:`~.lib.build.front.FrontBuildManager.call`.
:param build_id: (string)
Build ID.
:return: (string)
"""
if request.registry.settings['uid'] != context['front_id']:
return '%s-%s' % (context['front_id'].lower(), build_id)
return build_id
# =============================================================================
def _agent_paths(request, context, *files):
"""Prefix files if agent is not called locally.
:param request: (:class:`pyramid.request.Request` instance)
Current request.
:param context: (dictionary)
See :meth:`~.lib.build.front.FrontBuildManager.call`.
:param files: (list)
List of files on front in storage.
:return: (list)
"""
if request.registry.settings['uid'] != context['front_id']:
prefix = camel_case(context['front_id'])
return ['%s-%s' % (prefix, name) for name in files]
return files
# =============================================================================
def _agent_method(request, context, build_id, method):
"""Check authentification and return Agent ``method`` result.
:param request: (:class:`pyramid.request.Request` instance)
Current request.
:param context: (dictionary)
See :meth:`~.lib.build.front.FrontBuildManager.call`.
:param build_id: (string)
Build ID.
:return: (tuple)
``(<error>, <method_result>)``
"""
if not _authorized(request, context):
return _not_authorized(context)
build_id = _agent_build_id(request, context, build_id)
return '', getattr(request.registry['abuild'], method)(build_id)