Merge branch 'release/0.13.0'

.travis.yml

@@ -1,14 +1,12 @@
 language: python
 python:
   - 2.7
-  - 3.3
+  - 3.4
 
 notifications:
   email: false
 
 # Install stuff
-virtualenv:
-  system_site_packages: true
 before_install:
   - travis/install_sge.sh
   - export SGE_ROOT=/var/lib/gridengine

README.rst

@@ -27,11 +27,11 @@ A package to allow you to easily create jobs on the cluster directly from
 Python. You can directly map Python functions onto the cluster without needing
 to write any wrapper code yourself.
 
-This is the ETS fork of an older project called Python Grid. Unlike the older
-version,  it is Python 2/3 compatible. Another major difference is that you can
-change the  configuration via environment variables instead of having to modify
-a Python file in your ``site-packages`` directory. We've also removed some cruft
-and improved the reliability of some of the features.
+This is the ETS fork of an older project called `Python Grid <https://github.com/cwidmer/pythongrid>`__. Unlike the older
+version, it is Python 2/3 compatible. Another major difference is that you can
+change the configuration via environment variables instead of having to modify
+a Python file in your ``site-packages`` directory. We've also removed some
+cruft and improved the reliability of some of the features.
 
 For some examples of how to use it, check out ``map_reduce.py`` (for a simple
 example of how you can map a function onto the cluster) and ``manual.py`` (for
@@ -46,6 +46,7 @@ Requirements
 ~~~~~~~~~~~~
 
 -  `drmaa <https://github.com/drmaa-python/drmaa-python>`__
+-  `psutil <https://github.com/giampaolo/psutil>`__
 -  `pyzmq <https://github.com/zeromq/pyzmq>`__
 -  Python 2.7+
 

docs/conf.py

@@ -67,7 +67,7 @@ def __getattr__(cls, name):
 
 # General information about the project.
 project = u'GridMap'
-copyright = u'2008-2012 Max-Planck-Society, 2012-2013 ETS'
+copyright = u'2008-2012 Max-Planck-Society, 2012-2014 ETS'
 
 # The version info for the project you're documenting, acts as replacement for
 # |version| and |release|, also used in various other places throughout the
@@ -276,7 +276,7 @@ def __getattr__(cls, name):
 epub_title = u'GridMap'
 epub_author = u'Daniel Blanchard, Cheng Soon Ong, Christian Widmer'
 epub_publisher = u'Educational Testing Service'
-epub_copyright = u'2008-2012 Max-Planck-Society, 2012-2013 Educational Testing Service'
+epub_copyright = u'2008-2012 Max-Planck-Society, 2012-2014 Educational Testing Service'
 
 # The language of the text. It defaults to the language option
 # or en if the language is not set.

examples/manual.py

@@ -2,8 +2,8 @@
 
 # Written (W) 2008-2012 Christian Widmer
 # Written (W) 2008-2010 Cheng Soon Ong
-# Written (W) 2012-2013 Daniel Blanchard, dblanchard@ets.org
-# Copyright (C) 2008-2012 Max-Planck-Society, 2012-2013 ETS
+# Written (W) 2012-2014 Daniel Blanchard, dblanchard@ets.org
+# Copyright (C) 2008-2012 Max-Planck-Society, 2012-2014 ETS
 
 # This file is part of GridMap.
 
@@ -28,16 +28,29 @@
 
 from __future__ import print_function, unicode_literals
 
-import time
+import logging
+from datetime import datetime
 
 from gridmap import Job, process_jobs
 
 
+def sleep_walk(secs):
+    '''
+    Pass the time by adding numbers until the specified number of seconds has
+    elapsed. Intended as a replacement for ``time.sleep`` that doesn't leave the
+    CPU idle (which will make the job seem like it's stalled).
+    '''
+    start_time = datetime.now()
+    num = 0
+    while (datetime.now() - start_time).seconds < secs:
+        num = num + 1
+
+
 def compute_factorial(n):
     """
     computes factorial of n
     """
-    time.sleep(10)
+    sleep_walk(10)
     ret = 1
     for i in range(n):
         ret = ret * (i + 1)
@@ -62,7 +75,9 @@ def make_jobs():
 
     # create job objects
     for arg in inputvec:
-        job = Job(compute_factorial, arg)
+        # The default queue used by the Job class is all.q. You must specify
+        # the `queue` keyword argument if that is not the name of your queue.
+        job = Job(compute_factorial, arg, queue='all.q')
         jobs.append(job)
 
     return jobs
@@ -73,6 +88,10 @@ def main():
     run a set of jobs on cluster
     """
 
+    logging.captureWarnings(True)
+    logging.basicConfig(format=('%(asctime)s - %(name)s - %(levelname)s - ' +
+                                '%(message)s'), level=logging.INFO)
+
     print("=====================================")
     print("========   Submit and Wait   ========")
     print("=====================================")
@@ -83,7 +102,7 @@ def main():
     print("sending function jobs to cluster")
     print("")
 
-    job_outputs = process_jobs(functionJobs)
+    job_outputs = process_jobs(functionJobs, max_processes=4)
 
     print("results from each job")
     for (i, result) in enumerate(job_outputs):

examples/map_reduce.py

@@ -2,8 +2,8 @@
 
 # Written (W) 2008-2012 Christian Widmer
 # Written (W) 2008-2010 Cheng Soon Ong
-# Written (W) 2012-2013 Daniel Blanchard, dblanchard@ets.org
-# Copyright (C) 2008-2012 Max-Planck-Society, 2012-2013 ETS
+# Written (W) 2012-2014 Daniel Blanchard, dblanchard@ets.org
+# Copyright (C) 2008-2012 Max-Planck-Society, 2012-2014 ETS
 
 # This file is part of GridMap.
 
@@ -30,13 +30,29 @@
 
 from __future__ import print_function, unicode_literals
 
+import logging
+from datetime import datetime
+
 from gridmap import grid_map
 
 
+def sleep_walk(secs):
+    '''
+    Pass the time by adding numbers until the specified number of seconds has
+    elapsed. Intended as a replacement for ``time.sleep`` that doesn't leave the
+    CPU idle (which will make the job seem like it's stalled).
+    '''
+    start_time = datetime.now()
+    num = 0
+    while (datetime.now() - start_time).seconds < secs:
+        num = num + 1
+
+
 def computeFactorial(n):
     """
     computes factorial of n
     """
+    sleep_walk(10)
     ret = 1
     for i in range(n):
         ret = ret * (i + 1)
@@ -48,10 +64,18 @@ def main():
     execute map example
     """
 
-    args = [1, 2, 4, 8, 16]
+    logging.captureWarnings(True)
+    logging.basicConfig(format=('%(asctime)s - %(name)s - %(levelname)s - ' +
+                                '%(message)s'), level=logging.INFO)
+
+    args = [3, 5, 10, 20]
 
-    intermediate_results = grid_map(computeFactorial, args, quiet=False)
+    # The default queue used by grid_map is all.q. You must specify
+    # the `queue` keyword argument if that is not the name of your queue.
+    intermediate_results = grid_map(computeFactorial, args, quiet=False,
+                                    max_processes=4, queue='all.q')
 
+    # Just print the items instead of really reducing. We could always sum them.
     print("reducing result")
     for i, ret in enumerate(intermediate_results):
         print("f({0}) = {1}".format(args[i], ret))

gridmap/__init__.py

@@ -2,8 +2,8 @@
 
 # Written (W) 2008-2012 Christian Widmer
 # Written (W) 2008-2010 Cheng Soon Ong
-# Written (W) 2012-2013 Daniel Blanchard, dblanchard@ets.org
-# Copyright (C) 2008-2012 Max-Planck-Society, 2012-2013 ETS
+# Written (W) 2012-2014 Daniel Blanchard, dblanchard@ets.org
+# Copyright (C) 2008-2012 Max-Planck-Society, 2012-2014 ETS
 
 # This file is part of GridMap.
 

gridmap/conf.py

@@ -2,8 +2,8 @@
 
 # Written (W) 2008-2012 Christian Widmer
 # Written (W) 2008-2010 Cheng Soon Ong
-# Written (W) 2012-2013 Daniel Blanchard, dblanchard@ets.org
-# Copyright (C) 2008-2012 Max-Planck-Society, 2012-2013 ETS
+# Written (W) 2012-2014 Daniel Blanchard, dblanchard@ets.org
+# Copyright (C) 2008-2012 Max-Planck-Society, 2012-2014 ETS
 
 # This file is part of GridMap.
 

gridmap/data.py

@@ -2,8 +2,8 @@
 
 # Written (W) 2008-2012 Christian Widmer
 # Written (W) 2008-2010 Cheng Soon Ong
-# Written (W) 2012-2013 Daniel Blanchard, dblanchard@ets.org
-# Copyright (C) 2008-2012 Max-Planck-Society, 2012-2013 ETS
+# Written (W) 2012-2014 Daniel Blanchard, dblanchard@ets.org
+# Copyright (C) 2008-2012 Max-Planck-Society, 2012-2014 ETS
 
 # This file is part of GridMap.
 
@@ -39,19 +39,6 @@
 import re
 
 
-def clean_path(path):
-    '''
-    Replace all weird SAN paths with normal paths. This is really
-    ETS-specific, but shouldn't harm anyone else.
-    '''
-
-    path = re.sub(r'/\.automount/\w+/SAN/NLP/(\w+)-(dynamic|static)',
-                  r'/home/nlp-\1/\2', path)
-    path = re.sub(r'/\.automount/[^/]+/SAN/Research/HomeResearch',
-                  '/home/research', path)
-    return path
-
-
 def zdumps(obj):
     """
     dumps pickleable object into bz2 compressed string