Starting caching work

TODO.md

@@ -4,7 +4,7 @@ foo#*.ext for src_filter on input side
 Caching
 ==============================================================
 
-
+copy_files/etc need to preserve times
 
 Stages
 ==============================================================

anvil/cache.py

@@ -6,6 +6,7 @@
 __author__ = 'benvanik@google.com (Ben Vanik)'
 
 
+import base64
 import cPickle
 import os
 
@@ -24,10 +25,12 @@ class RuleCache(object):
     """
     pass
 
-  def compute_delta(self, src_paths):
+  def compute_delta(self, rule_path, mode, src_paths):
     """Computes a file delta for the given source paths.
 
     Args:
+      rule_path: Full path to the rule.
+      mode: Mode indicating which type of set to use.
       src_paths: A list of fully-resolved source file paths.
 
     Returns:
@@ -54,7 +57,7 @@ class FileRuleCache(RuleCache):
     self.data = dict()
     self._dirty = False
 
-    if os.access(self.cache_path, os.R_OK):
+    if os.path.exists(self.cache_path):
       with open(self.cache_path, 'rb') as file_obj:
         self.data.update(cPickle.load(file_obj))
 
@@ -62,13 +65,39 @@ class FileRuleCache(RuleCache):
     if not self._dirty:
       return
     self._dirty = False
+    try:
+      os.makedirs(os.path.split(self.cache_path)[0])
+    except:
+      pass
     with open(self.cache_path, 'wb') as file_obj:
       cPickle.dump(self.data, file_obj, 2)
 
-  def compute_delta(self, src_paths):
+  def compute_delta(self, rule_path, mode, src_paths):
     file_delta = FileDelta()
     file_delta.all_files.extend(src_paths)
 
+    # Scan all files - we need this to compare regardless of whether we have
+    # data from the cache
+    # TODO(benvanik): make this parallel
+    new_data = dict()
+    for src_path in src_paths:
+      file_time = os.path.getmtime(src_path)
+      file_size = os.path.getsize(src_path)
+      new_data[src_path] = '%s-%s' % (file_time, file_size)
+
+    # Always swap for new data
+    key = base64.b64encode('%s->%s' % (rule_path, mode))
+    old_data = self.data.get(key, None)
+    self.data[key] = new_data
+
+    # No previous data - ignore
+    if not old_data:
+      self._dirty = True
+      file_delta.changed_files.extend(src_paths)
+      return file_delta
+
+    # Compare data
+
     # TODO(benvanik): work
     self._dirty = True
     file_delta.changed_files.extend(src_paths)

anvil/context.py

@@ -600,23 +600,6 @@ class RuleContext(object):
           return True
     return False
 
-  def _check_if_cached(self):
-    """Checks if all inputs and outputs match their expected values.
-
-    Returns:
-      True if no inputs or outputs have changed.
-    """
-    # If any input changed...
-    if self.file_delta.any_changes():
-      return False
-
-    # If any output is changed...
-    output_delta = self.build_context.cache.compute_delta(self.all_output_files)
-    if output_delta.any_changes():
-      return False
-
-    return True
-
   def begin(self):
     """Begins asynchronous rule execution.
     Custom RuleContext implementations should override this method to perform
@@ -642,10 +625,29 @@ class RuleContext(object):
 
     # Compute file delta
     # Note that this could be done async (somehow)
-    self.file_delta = self.build_context.cache.compute_delta(self.src_paths)
+    self.file_delta = self.build_context.cache.compute_delta(
+        self.rule.path, 'src', self.src_paths)
 
     return self.deferred
 
+  def _check_if_cached(self):
+    """Checks if all inputs and outputs match their expected values.
+
+    Returns:
+      True if no inputs or outputs have changed.
+    """
+    # If any input changed...
+    if self.file_delta.any_changes():
+      return False
+
+    # If any output is changed...
+    output_delta = self.build_context.cache.compute_delta(
+        self.rule.path, 'out', self.all_output_files)
+    if output_delta.any_changes():
+      return False
+
+    return True
+
   def cascade_failure(self):
     """Instantly fails a rule, signaling that a rule prior to it has failed
     and it should not be run.