py-why / py-why/EconML

Bootstrap and DeepIV does not work on Multiple Workers

Open
#51 1 comment 0 reactions 1 assignee View on GitHub

@kbattocchi is already working on this.

Since May 2, 2019.

bug
Dominant language
Jupyter Notebook
Stars
4.8k
Forks
827
PR merge metrics
No merged PRs in 30d

Description

I tried adding the bootstrap wrapper to the file Deep IV Examples.ipynb with n_jobs more than 1, but it gives me some errors:

deepIvEst = DeepIVEstimator(n_components = 10, # number of gaussians in our mixture density network
                            m = lambda z, x : treatment_model(keras.layers.concatenate([z,x])), # treatment model
                            h = lambda t, x : response_model(keras.layers.concatenate([t,x])),  # response model
                            n_samples = 1, # number of samples to use to estimate the response
                            use_upper_bound_loss = False, # whether to use an approximation to the true loss
                            n_gradient_samples = 1, # number of samples to use in second estimate of the response (to make loss estimate unbiased)
                            optimizer='adam', # Keras optimizer to use for training - see https://keras.io/optimizers/ 
                            s1=100, # number of epochs to train treatment model
                            s2=100) # number of epochs to train response model
boot_est = BootstrapEstimator(deepIvEst, n_bootstrap_samples=100, n_jobs=4)
boot_est.fit(Y=y,T=t,X=x,Z=z)
[Parallel(n_jobs=4)]: Using backend ThreadingBackend with 4 concurrent workers.
---------------------------------------------------------------------------
ValueError                                Traceback (most recent call last)
<ipython-input-14-2528bf95d3ca> in <module>
----> 1 boot_est.fit(Y=y,T=t,X=x,Z=z)

~/Dropbox (Cambridge University)/R301/EconML/econml/bootstrap.py in fit(self, *args, **named_args)
     56         Parallel(n_jobs=self._n_jobs, prefer='threads', verbose=3)(
     57             (obj.fit, [arg[inds] for arg in args], {arg: named_args[arg][inds] for arg in named_args})
---> 58             for obj, inds in zip(self._instances, indices)
     59         )
     60         return self

~/anaconda3/lib/python3.6/site-packages/joblib-0.13.2-py3.6.egg/joblib/parallel.py in __call__(self, iterable)
    932 
    933             with self._backend.retrieval_context():
--> 934                 self.retrieve()
    935             # Make sure that we get a last message telling us we are done
    936             elapsed_time = time.time() - self._start_time

~/anaconda3/lib/python3.6/site-packages/joblib-0.13.2-py3.6.egg/joblib/parallel.py in retrieve(self)
    831             try:
    832                 if getattr(self._backend, 'supports_timeout', False):
--> 833                     self._output.extend(job.get(timeout=self.timeout))
    834                 else:
    835                     self._output.extend(job.get())

~/anaconda3/lib/python3.6/multiprocessing/pool.py in get(self, timeout)
    642             return self._value
    643         else:
--> 644             raise self._value
    645 
    646     def _set(self, i, obj):

~/anaconda3/lib/python3.6/multiprocessing/pool.py in worker(inqueue, outqueue, initializer, initargs, maxtasks, wrap_exception)
    117         job, i, func, args, kwds = task
    118         try:
--> 119             result = (True, func(*args, **kwds))
    120         except Exception as e:
    121             if wrap_exception and func is not _helper_reraises_exception:

~/anaconda3/lib/python3.6/site-packages/joblib-0.13.2-py3.6.egg/joblib/_parallel_backends.py in __call__(self, *args, **kwargs)
    565     def __call__(self, *args, **kwargs):
    566         try:
--> 567             return self.func(*args, **kwargs)
    568         except KeyboardInterrupt:
    569             # We capture the KeyboardInterrupt and reraise it as

~/anaconda3/lib/python3.6/site-packages/joblib-0.13.2-py3.6.egg/joblib/parallel.py in __call__(self)
    223         with parallel_backend(self._backend, n_jobs=self._n_jobs):
    224             return [func(*args, **kwargs)
--> 225                     for func, args, kwargs in self.items]
    226 
    227     def __len__(self):

~/anaconda3/lib/python3.6/site-packages/joblib-0.13.2-py3.6.egg/joblib/parallel.py in <listcomp>(.0)
    223         with parallel_backend(self._backend, n_jobs=self._n_jobs):
    224             return [func(*args, **kwargs)
--> 225                     for func, args, kwargs in self.items]
    226 
    227     def __len__(self):

~/Dropbox (Cambridge University)/R301/EconML/econml/deepiv.py in fit(self, Y, T, X, Z)
    313         n_components = self._n_components
    314 
--> 315         treatment_network = self._m(z_in, x_in)
    316 
    317         # the dimensionality of the output of the network

<ipython-input-10-df8c54d69989> in <lambda>(z, x)
      1 deepIvEst = DeepIVEstimator(n_components = 10, # number of gaussians in our mixture density network
----> 2                             m = lambda z, x : treatment_model(keras.layers.concatenate([z,x])), # treatment model
      3                             h = lambda t, x : response_model(keras.layers.concatenate([t,x])),  # response model
      4                             n_samples = 1, # number of samples to use to estimate the response
      5                             use_upper_bound_loss = False, # whether to use an approximation to the true loss

~/anaconda3/lib/python3.6/site-packages/keras/engine/base_layer.py in __call__(self, inputs, **kwargs)
    455             # Actually call the layer,
    456             # collecting output(s), mask(s), and shape(s).
--> 457             output = self.call(inputs, **kwargs)
    458             output_mask = self.compute_mask(inputs, previous_mask)
    459 

~/anaconda3/lib/python3.6/site-packages/keras/engine/network.py in call(self, inputs, mask)
    562             return self._output_tensor_cache[cache_key]
    563         else:
--> 564             output_tensors, _, _ = self.run_internal_graph(inputs, masks)
    565             return output_tensors
    566 

~/anaconda3/lib/python3.6/site-packages/keras/engine/network.py in run_internal_graph(self, inputs, masks)
    719                                     kwargs['mask'] = computed_mask
    720                             output_tensors = to_list(
--> 721                                 layer.call(computed_tensor, **kwargs))
    722                             output_masks = layer.compute_mask(computed_tensor,
    723                                                               computed_mask)

~/anaconda3/lib/python3.6/site-packages/keras/layers/core.py in call(self, inputs)
    877 
    878     def call(self, inputs):
--> 879         output = K.dot(inputs, self.kernel)
    880         if self.use_bias:
    881             output = K.bias_add(output, self.bias, data_format='channels_last')

~/anaconda3/lib/python3.6/site-packages/keras/backend/tensorflow_backend.py in dot(x, y)
   1083         out = tf.sparse_tensor_dense_matmul(x, y)
   1084     else:
-> 1085         out = tf.matmul(x, y)
   1086     return out
   1087 

~/anaconda3/lib/python3.6/site-packages/tensorflow/python/ops/math_ops.py in matmul(a, b, transpose_a, transpose_b, adjoint_a, adjoint_b, a_is_sparse, b_is_sparse, name)
   2385       are both set to True.
   2386   """
-> 2387   with ops.name_scope(name, "MatMul", [a, b]) as name:
   2388     if transpose_a and adjoint_a:
   2389       raise ValueError("Only one of transpose_a and adjoint_a can be True.")

~/anaconda3/lib/python3.6/site-packages/tensorflow/python/framework/ops.py in __enter__(self)
   6081       if self._values is None:
   6082         self._values = []
-> 6083       g = _get_graph_from_inputs(self._values)
   6084       self._g_manager = g.as_default()
   6085       self._g_manager.__enter__()

~/anaconda3/lib/python3.6/site-packages/tensorflow/python/framework/ops.py in _get_graph_from_inputs(op_input_list, graph)
   5711         graph = graph_element.graph
   5712       elif original_graph_element is not None:
-> 5713         _assert_same_graph(original_graph_element, graph_element)
   5714       elif graph_element.graph is not graph:
   5715         raise ValueError("%s is not from the passed-in graph." % graph_element)

~/anaconda3/lib/python3.6/site-packages/tensorflow/python/framework/ops.py in _assert_same_graph(original_item, item)
   5647   if original_item.graph is not item.graph:
   5648     raise ValueError("%s must be from the same graph as %s." % (item,
-> 5649                                                                 original_item))
   5650 
   5651 

ValueError: Tensor("dense_1/kernel:0", shape=(2, 128), dtype=float32_ref) must be from the same graph as Tensor("concatenate_1/concat:0", shape=(?, 2), dtype=float32).

However, the problem is solved when I change n_jobs=1:

boot_est = BootstrapEstimator(deepIvEst, n_bootstrap_samples=10, n_jobs=1)

Is this a currency issue (i.e. DeepIV can only be trained on one job)? But it shouldn't be because in Bootstrapping, you should be able to do parallel processing ...

Contributor guide

No contributing guide indexed for this repository

First steps

  1. Read the whole issue, then the project's contributing guide.
  2. Comment on the issue to say you are picking it up — it saves two people doing the same work.
  3. Fork the repository and make your change on a branch.
  4. Open a pull request that references the issue number.

Assessment

This issue has not been assessed yet.

Get new issues in your inbox

A short digest of beginner-friendly GitHub issues.