root
/
anvil-build
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #8 from Husafan/context 

Fix test failure and restructure the BuildContext execution.
This commit is contained in:
Ben Vanik 2014-12-02 12:24:57 -08:00
parent ed26bd2a83 48755c148e
commit 455444073b
2 changed files with 147 additions and 167 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

170
anvil/context.py
View File

@ -116,6 +116,8 @@ class BuildContext(object):
self.stop_on_error = stop_on_error self.stop_on_error = stop_on_error
self.raise_on_error = raise_on_error self.raise_on_error = raise_on_error
self.error_encountered = False
# Build the rule graph # Build the rule graph
self.rule_graph = graph.RuleGraph(self.project) self.rule_graph = graph.RuleGraph(self.project)
@ -154,9 +156,9 @@ class BuildContext(object):
d = self.execute_async(target_rule_names) d = self.execute_async(target_rule_names)
self.wait(d) self.wait(d)
result = [None] result = [None]
def _callback(): def _callback(*args, **kwargs):
result[0] = True result[0] = True
def _errback(): def _errback(*args, **kwargs):
result[0] = False result[0] = False
d.add_callback_fn(_callback) d.add_callback_fn(_callback)
d.add_errback_fn(_errback) d.add_errback_fn(_errback)
@ -188,126 +190,88 @@ class BuildContext(object):
# Calculate the sequence of rules to execute # Calculate the sequence of rules to execute
rule_sequence = self.rule_graph.calculate_rule_sequence(target_rule_names) rule_sequence = self.rule_graph.calculate_rule_sequence(target_rule_names)
any_failed = [False]
main_deferred = Deferred()
remaining_rules = rule_sequence[:] remaining_rules = rule_sequence[:]
in_flight_rules = []
pumping = [False]
def _issue_rule(rule): def _issue_rule(rule, deferred=None):
"""Issues a single rule into the current execution context. """Issues a single rule into the current execution context.
Updates the in_flight_rules list and pumps when the rule completes. Updates the in_flight_rules list and pumps when the rule completes.
Args: Args:
rule: Rule to issue. rule: Rule to issue.
deferred: Deferred to wait on before executing the rule.
""" """
def _rule_callback(*args, **kwargs): def _rule_callback(*args, **kwargs):
in_flight_rules.remove(rule) remaining_rules.remove(rule)
_pump(previous_succeeded=True)
def _rule_errback(exception=None, *args, **kwargs): def _rule_errback(exception=None, *args, **kwargs):
in_flight_rules.remove(rule) remaining_rules.remove(rule)
if self.stop_on_error:
self.error_encountered = True
# TODO(benvanik): log result/exception/etc? # TODO(benvanik): log result/exception/etc?
if exception: # pragma: no cover if exception: # pragma: no cover
print exception print exception
any_failed[0] = True
_pump(previous_succeeded=False)
in_flight_rules.append(rule) # All RuleContexts should be created by the time this method is called.
rule_deferred = self._execute_rule(rule) assert self.rule_contexts[rule.path]
rule_deferred = self.rule_contexts[rule.path].deferred
rule_deferred.add_callback_fn(_rule_callback) rule_deferred.add_callback_fn(_rule_callback)
rule_deferred.add_errback_fn(_rule_errback) rule_deferred.add_errback_fn(_rule_errback)
def _execute(*args, **kwargs):
self._execute_rule(rule)
def _on_failure(*args, **kwards):
self._execute_rule(rule)
if deferred:
deferred.add_callback_fn(_execute)
deferred.add_errback_fn(_on_failure)
else:
_execute()
return rule_deferred return rule_deferred
def _pump(previous_succeeded=True): def _chain_rule_execution(target_rules):
"""Attempts to run another rule and signals the main_deferred if done. """Given a list of target rules, build them and all dependencies.
This method builds the passed in target rules and all dependencies. It
first assembles a list of the dependencies to target rules orded as:
[dependencies -> target_rules]
It then traverses the list, issuing execute commands for all rules that
do not have dependencies within the list. For all rules that do have
dependencies within the list, a deferred is used to trigger the rule's
exeution once all dependencies have completed executing.
Args: Args:
previous_succeeded: Whether the previous rule succeeded. target_rules: A list of rules to be executed.
Returns:
A deferred that resolves once all target_rules have either executed
successfully or failed.
""" """
# If we're already done, gracefully exit issued_rules = []
if main_deferred.is_done(): all_deferreds = []
return for rule in target_rules:
# Create the RuleContexts here so that failures can cascade and the
# deferred is accessible by any rules that depend on this one.
rule_ctx = rule.create_context(self)
self.rule_contexts[rule.path] = rule_ctx
# If we failed and we are supposed to stop, gracefully stop by # Make the execution of the current rule dependent on the execution
# killing all future rules # of all rules it depends on.
# This is better than terminating immediately, as it allows legit tasks dependent_deferreds = []
# to finish for executable_rule in issued_rules:
if any_failed[0] and self.stop_on_error: if self.rule_graph.has_dependency(rule.path, executable_rule.path):
remaining_rules[:] = [] executable_ctx = self.rule_contexts[executable_rule.path]
# TODO(benvanik): better error message dependent_deferreds.append(executable_ctx.deferred)
main_deferred.errback() if dependent_deferreds:
return dependent_deferred = async.gather_deferreds(
dependent_deferreds, errback_if_any_fail=True)
if pumping[0]: all_deferreds.append(_issue_rule(rule, dependent_deferred))
return
pumping[0] = True
# Scan through all remaining rules - if any are unblocked, issue them
to_issue = []
for i in range(0, len(remaining_rules)):
next_rule = remaining_rules[i]
# Ignore if any dependency on any rule before it in the list
skip_rule = False
if i:
for old_rule in remaining_rules[:i]:
if self.rule_graph.has_dependency(next_rule.path, old_rule.path):
# Blocked on previous rule
skip_rule = True
break
if skip_rule:
continue
# Ignore if any dependency on an in-flight rule
for in_flight_rule in in_flight_rules:
if self.rule_graph.has_dependency(next_rule.path,
in_flight_rule.path):
# Blocked on a previous rule, so pass and wait for the next pump
skip_rule = True
break
if skip_rule:
continue
# If here then we found no conflicting rules, queue for running
to_issue.append(next_rule)
# Run all rules that we can
for rule in to_issue:
remaining_rules.remove(rule)
for rule in to_issue:
_issue_rule(rule)
if (not len(remaining_rules) and
not len(in_flight_rules) and
not main_deferred.is_done()):
assert not len(remaining_rules)
# Done!
# TODO(benvanik): better errbacks? some kind of BuildResults?
if not any_failed[0]:
# Only save the cache when we have succeeded
# This causes some stuff to be rebuilt in failure cases, but prevents
# a lot of weirdness when things are partially broken
self.cache.save()
main_deferred.callback()
else: else:
main_deferred.errback() all_deferreds.append(_issue_rule(rule))
return async.gather_deferreds(all_deferreds, errback_if_any_fail=True)
pumping[0] = False return _chain_rule_execution(rule_sequence)
# Keep the queue pumping
if not len(in_flight_rules) and len(remaining_rules):
_pump()
# Kick off execution (once for each rule as a heuristic for filling the
# pipeline)
for rule in rule_sequence:
_pump()
return main_deferred
def wait(self, deferreds): def wait(self, deferreds):
"""Blocks waiting on a list of deferreds until they all complete. """Blocks waiting on a list of deferreds until they all complete.
@ -330,10 +294,10 @@ class BuildContext(object):
Returns: Returns:
A Deferred that will callback when the rule has completed executing. A Deferred that will callback when the rule has completed executing.
""" """
assert not self.rule_contexts.has_key(rule.path) assert self.rule_contexts.has_key(rule.path)
rule_ctx = rule.create_context(self) rule_ctx = self.rule_contexts[rule.path]
self.rule_contexts[rule.path] = rule_ctx if (rule_ctx.check_predecessor_failures() or
if rule_ctx.check_predecessor_failures(): self.stop_on_error and self.error_encountered):
return rule_ctx.cascade_failure() return rule_ctx.cascade_failure()
else: else:
rule_ctx.begin() rule_ctx.begin()
@ -647,7 +611,7 @@ class RuleContext(object):
"""Checks all dependencies for failure. """Checks all dependencies for failure.
Returns: Returns:
True if any dependency has failed. True if any dependency has failed or been interrupted.
""" """
for dep in self.rule.get_dependent_paths(): for dep in self.rule.get_dependent_paths():
if util.is_rule_path(dep): if util.is_rule_path(dep):
@ -655,7 +619,7 @@ class RuleContext(object):
dep, requesting_module=self.rule.parent_module) dep, requesting_module=self.rule.parent_module)
other_rule_ctx = self.build_context.rule_contexts.get( other_rule_ctx = self.build_context.rule_contexts.get(
other_rule.path, None) other_rule.path, None)
if other_rule_ctx.status == Status.FAILED: if (other_rule_ctx.status == Status.FAILED):
return True return True
return False return False

144
anvil/context_test.py
View File

@ -153,19 +153,31 @@ class BuildContextTest(FixtureTestCase):
results = ctx.get_rule_results('m:b') results = ctx.get_rule_results('m:b')
self.assertEqual(results[0], Status.FAILED) self.assertEqual(results[0], Status.FAILED)
print '*******************************************************'
project = Project(modules=[Module('m', rules=[ project = Project(modules=[Module('m', rules=[
FailRule('a'), FailRule('a'),
SucceedRule('b', deps=[':a'])])]) SucceedRule('b', deps=[':a']),
SucceedRule('c'),
SucceedRule('d', deps=[':c']),
SucceedRule('e', deps=[':c']),
SucceedRule('f', deps=[':d', ':e'])])])
with BuildContext(self.build_env, project, stop_on_error=True) as ctx: with BuildContext(self.build_env, project, stop_on_error=True) as ctx:
d = ctx.execute_async(['m:b']) d = ctx.execute_async(['m:b', 'm:f'])
ctx.wait(d) ctx.wait(d)
self.assertErrback(d) self.assertErrback(d)
results = ctx.get_rule_results('m:a') results = ctx.get_rule_results('m:a')
self.assertEqual(results[0], Status.FAILED) self.assertEqual(results[0], Status.FAILED)
results = ctx.get_rule_results('m:b') results = ctx.get_rule_results('m:b')
self.assertEqual(results[0], Status.FAILED) self.assertEqual(results[0], Status.FAILED)
# Because m:a failed and stop_on_error is true, even though m:c is a
# succed rule, m:d, m:e and m:f should all be FAILED as well.
results = ctx.get_rule_results('m:d')
self.assertEqual(results[0], Status.FAILED)
results = ctx.get_rule_results('m:e')
self.assertEqual(results[0], Status.FAILED)
results = ctx.get_rule_results('m:f')
self.assertEqual(results[0], Status.FAILED)
# TODO(benvanik): test stop_on_error
# TODO(benvanik): test raise_on_error # TODO(benvanik): test raise_on_error
def testCaching(self): def testCaching(self):
@ -338,26 +350,29 @@ class RuleContextTest(FixtureTestCase):
project = Project(module_resolver=FileModuleResolver(self.root_path)) project = Project(module_resolver=FileModuleResolver(self.root_path))
build_ctx = BuildContext(self.build_env, project) build_ctx = BuildContext(self.build_env, project)
rule = project.resolve_rule(':file_input') rule = ':file_input'
d = build_ctx._execute_rule(rule) resolved = project.resolve_rule(rule)
self.assertTrue(d.is_done()) success = build_ctx.execute_sync([rule])
rule_outputs = build_ctx.get_rule_outputs(rule) self.assertTrue(success)
rule_outputs = build_ctx.get_rule_outputs(resolved)
self.assertEqual( self.assertEqual(
set([os.path.basename(f) for f in rule_outputs]), set([os.path.basename(f) for f in rule_outputs]),
set(['a.txt'])) set(['a.txt']))
rule = project.resolve_rule(':local_txt') rule = ':local_txt'
d = build_ctx._execute_rule(rule) resolved = project.resolve_rule(rule)
self.assertTrue(d.is_done()) success = build_ctx.execute_sync([rule])
rule_outputs = build_ctx.get_rule_outputs(rule) self.assertTrue(success)
rule_outputs = build_ctx.get_rule_outputs(resolved)
self.assertEqual( self.assertEqual(
set([os.path.basename(f) for f in rule_outputs]), set([os.path.basename(f) for f in rule_outputs]),
set(['a.txt', 'b.txt', 'c.txt'])) set(['a.txt', 'b.txt', 'c.txt']))
rule = project.resolve_rule(':recursive_txt') rule = ':recursive_txt'
d = build_ctx._execute_rule(rule) resolved = project.resolve_rule(rule)
self.assertTrue(d.is_done()) success = build_ctx.execute_sync([rule])
rule_outputs = build_ctx.get_rule_outputs(rule) self.assertTrue(success)
rule_outputs = build_ctx.get_rule_outputs(resolved)
self.assertEqual( self.assertEqual(
set([os.path.basename(f) for f in rule_outputs]), set([os.path.basename(f) for f in rule_outputs]),
set(['a.txt', 'b.txt', 'c.txt', 'd.txt', 'e.txt'])) set(['a.txt', 'b.txt', 'c.txt', 'd.txt', 'e.txt']))
@ -366,63 +381,67 @@ class RuleContextTest(FixtureTestCase):
project = Project(module_resolver=FileModuleResolver(self.root_path)) project = Project(module_resolver=FileModuleResolver(self.root_path))
build_ctx = BuildContext(self.build_env, project) build_ctx = BuildContext(self.build_env, project)
rule = project.resolve_rule(':missing_txt') rule = ':missing_txt'
with self.assertRaises(OSError): with self.assertRaises(OSError):
build_ctx._execute_rule(rule) build_ctx.execute_sync([rule])
rule = project.resolve_rule(':missing_glob_txt') rule = ':missing_glob_txt'
d = build_ctx._execute_rule(rule) resolved = project.resolve_rule(rule)
self.assertTrue(d.is_done()) success = build_ctx.execute_sync([rule])
rule_outputs = build_ctx.get_rule_outputs(rule) self.assertTrue(success)
rule_outputs = build_ctx.get_rule_outputs(resolved)
self.assertEqual(len(rule_outputs), 0) self.assertEqual(len(rule_outputs), 0)
rule = project.resolve_rule(':local_txt_filter') rule = ':local_txt_filter'
d = build_ctx._execute_rule(rule) resolved = project.resolve_rule(rule)
self.assertTrue(d.is_done()) success = build_ctx.execute_sync([rule])
rule_outputs = build_ctx.get_rule_outputs(rule) self.assertTrue(success)
rule_outputs = build_ctx.get_rule_outputs(resolved)
self.assertEqual( self.assertEqual(
set([os.path.basename(f) for f in rule_outputs]), set([os.path.basename(f) for f in rule_outputs]),
set(['a.txt', 'b.txt', 'c.txt'])) set(['a.txt', 'b.txt', 'c.txt']))
rule = project.resolve_rule(':recursive_txt_filter') rule = ':recursive_txt_filter'
d = build_ctx._execute_rule(rule) resolved = project.resolve_rule(rule)
self.assertTrue(d.is_done()) success = build_ctx.execute_sync([rule])
rule_outputs = build_ctx.get_rule_outputs(rule) self.assertTrue(success)
rule_outputs = build_ctx.get_rule_outputs(resolved)
self.assertEqual( self.assertEqual(
set([os.path.basename(f) for f in rule_outputs]), set([os.path.basename(f) for f in rule_outputs]),
set(['a.txt', 'b.txt', 'c.txt', 'd.txt', 'e.txt'])) set(['a.txt', 'b.txt', 'c.txt', 'd.txt', 'e.txt']))
rule = project.resolve_rule(':exclude_txt_filter') rule = ':exclude_txt_filter'
d = build_ctx._execute_rule(rule) resolved = project.resolve_rule(rule)
self.assertTrue(d.is_done()) success = build_ctx.execute_sync([rule])
rule_outputs = build_ctx.get_rule_outputs(rule) self.assertTrue(success)
rule_outputs = build_ctx.get_rule_outputs(resolved)
self.assertEqual( self.assertEqual(
set([os.path.basename(f) for f in rule_outputs]), set([os.path.basename(f) for f in rule_outputs]),
set(['dir_2', 'a.txt-a', 'b.txt-b', 'c.txt-c', 'g.not-txt', 'BUILD'])) set(['dir_2', 'a.txt-a', 'b.txt-b', 'c.txt-c', 'g.not-txt', 'BUILD']))
rule = project.resolve_rule(':include_exclude_filter') rule = ':include_exclude_filter'
d = build_ctx._execute_rule(rule) resolved = project.resolve_rule(rule)
self.assertTrue(d.is_done()) success = build_ctx.execute_sync([rule])
rule_outputs = build_ctx.get_rule_outputs(rule) self.assertTrue(success)
rule_outputs = build_ctx.get_rule_outputs(resolved)
self.assertEqual( self.assertEqual(
set([os.path.basename(f) for f in rule_outputs]), set([os.path.basename(f) for f in rule_outputs]),
set(['a.txt-a', 'b.txt-b'])) set(['a.txt-a', 'b.txt-b']))
rule = project.resolve_rule(':multi_exts') rule = ':only_a'
build_ctx._execute_rule(rule) resolved = project.resolve_rule(rule)
success = build_ctx.execute_sync([rule])
rule = project.resolve_rule(':only_a') self.assertTrue(success)
d = build_ctx._execute_rule(rule) rule_outputs = build_ctx.get_rule_outputs(resolved)
self.assertTrue(d.is_done())
rule_outputs = build_ctx.get_rule_outputs(rule)
self.assertEqual( self.assertEqual(
set([os.path.basename(f) for f in rule_outputs]), set([os.path.basename(f) for f in rule_outputs]),
set(['a.txt-a'])) set(['a.txt-a']))
rule = project.resolve_rule(':only_ab') rule = ':only_ab'
d = build_ctx._execute_rule(rule) resolved = project.resolve_rule(rule)
self.assertTrue(d.is_done()) success = build_ctx.execute_sync([rule])
rule_outputs = build_ctx.get_rule_outputs(rule) self.assertTrue(success)
rule_outputs = build_ctx.get_rule_outputs(resolved)
self.assertEqual( self.assertEqual(
set([os.path.basename(f) for f in rule_outputs]), set([os.path.basename(f) for f in rule_outputs]),
set(['a.txt-a', 'b.txt-b'])) set(['a.txt-a', 'b.txt-b']))
@ -431,36 +450,33 @@ class RuleContextTest(FixtureTestCase):
project = Project(module_resolver=FileModuleResolver(self.root_path)) project = Project(module_resolver=FileModuleResolver(self.root_path))
build_ctx = BuildContext(self.build_env, project) build_ctx = BuildContext(self.build_env, project)
rule = project.resolve_rule(':file_input') rule = ':file_input'
d = build_ctx._execute_rule(rule) resolved = project.resolve_rule(rule)
self.assertTrue(d.is_done()) success = build_ctx.execute_sync([rule])
self.assertTrue(success)
rule_outputs = build_ctx.get_rule_outputs(rule) rule_outputs = build_ctx.get_rule_outputs(rule)
self.assertNotEqual(len(rule_outputs), 0) self.assertNotEqual(len(rule_outputs), 0)
rule = project.resolve_rule(':rule_input') rule = ':rule_input'
d = build_ctx._execute_rule(rule) resolved = project.resolve_rule(rule)
self.assertTrue(d.is_done()) success = build_ctx.execute_sync([rule])
self.assertTrue(success)
rule_outputs = build_ctx.get_rule_outputs(rule) rule_outputs = build_ctx.get_rule_outputs(rule)
self.assertEqual( self.assertEqual(
set([os.path.basename(f) for f in rule_outputs]), set([os.path.basename(f) for f in rule_outputs]),
set(['a.txt'])) set(['a.txt']))
rule = project.resolve_rule(':mixed_input') rule = ':mixed_input'
d = build_ctx._execute_rule(rule) resolved = project.resolve_rule(rule)
self.assertTrue(d.is_done()) success = build_ctx.execute_sync([rule])
self.assertTrue(success)
rule_outputs = build_ctx.get_rule_outputs(rule) rule_outputs = build_ctx.get_rule_outputs(rule)
self.assertEqual( self.assertEqual(
set([os.path.basename(f) for f in rule_outputs]), set([os.path.basename(f) for f in rule_outputs]),
set(['a.txt', 'b.txt'])) set(['a.txt', 'b.txt']))
rule = project.resolve_rule(':missing_input')
with self.assertRaises(KeyError): with self.assertRaises(KeyError):
build_ctx._execute_rule(rule) build_ctx.execute_sync([':missing_input'])
build_ctx = BuildContext(self.build_env, project)
rule = project.resolve_rule(':rule_input')
with self.assertRaises(RuntimeError):
build_ctx._execute_rule(rule)
def _compare_path(self, result, expected): def _compare_path(self, result, expected):
result = os.path.relpath(result, self.root_path) result = os.path.relpath(result, self.root_path)